Merge branch 'master' into examples/SK-855

.devcontainer/Dockerfile

@@ -1,14 +1,10 @@
-ARG BASE_IMG
-FROM $BASE_IMG
+FROM python:3.10-slim
 
 # Non-root user with sudo access
 ARG USERNAME=default
 ARG USER_UID=1000
 ARG USER_GID=$USER_UID
 
-# Versioning
-ARG DOCKER_VERSION=19.03.9
-ARG COMPOSE_VERSION=1.29.2
 
 # Avoid warnings by switching to noninteractive
 ENV DEBIAN_FRONTEND=noninteractive
@@ -27,13 +23,7 @@ RUN apt-get --allow-releaseinfo-change update \
   curl \
   git \
   vim \
-  #
-  # Install docker binaries
-  && curl -L https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}.tgz | tar xvz docker/docker \
-  && cp docker/docker /usr/local/bin \
-  && rm -R docker \
-  && curl -L https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m) -o /usr/local/bin/docker-compose \
-  && chmod +x /usr/local/bin/docker-compose \
+  ssh \
   #
   # Create a non-root user to use if preferred
   && groupadd --gid $USER_GID $USERNAME \

.dockerignore

@@ -4,6 +4,7 @@
 tmp/*
 test/*
 docs/*
+examples/*
 **/.venv
 **/.mnist-keras
 **/.mnist-pytorch

.github/workflows/integration-tests.yaml

@@ -1,14 +1,14 @@
 name: "integration tests"
 
-on: 
+on:
   push:
     branches:
-        - master
-        - develop
-        - 'release/**'
+      - master
+      - develop
+      - "release/**"
   pull_request:
     branches:
-      - '**'
+      - "**"
 
 jobs:
   integration-tests:
@@ -17,7 +17,7 @@ jobs:
         to_test:
           - "mnist-keras numpyhelper"
           - "mnist-pytorch numpyhelper"
-        python_version: ["3.8","3.9","3.10", "3.11"]
+        python_version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         os:
           - ubuntu-22.04
     runs-on: ${{ matrix.os }}
@@ -28,7 +28,7 @@ jobs:
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python_version }}
-      
+
       - name: run ${{ matrix.to_test }}
         run: .ci/tests/examples/run.sh ${{ matrix.to_test }}
 

README.rst

@@ -50,7 +50,13 @@ FEDn Studio - From development to FL in production:
 -  Features for the trusted-third party: Manage access to the FL network, FL clients and training progress.
 -  REST API for handling experiments/jobs. 
 -  View and export logging and tracing information. 
--  Public cloud, dedicated cloud and on-premise deployment options. 
+-  Public cloud, dedicated cloud and on-premise deployment options.
+
+Available clients:
+
+- Python client (this repository)
+- C++ client (`FEDn C++ client <https://github.com/scaleoutsystems/fedn-cpp-client>`__)
+- Android Kotlin client (`FEDn Kotlin client <https://github.com/scaleoutsystems/fedn-android-client>`__)
 
 
 Getting started

docker-compose.yaml

@@ -107,7 +107,7 @@ services:
           "/bin/grpc_health_probe",
           "-addr=localhost:12080"
         ]
-      interval: 2s
+      interval: 20s
       timeout: 10s
       retries: 5
     depends_on:

docs/conf.py

@@ -12,7 +12,7 @@
 author = "Scaleout Systems AB"
 
 # The full version, including alpha/beta/rc tags
-release = "0.9.2"
+release = "0.9.6"
 
 # Add any Sphinx extension module names here, as strings
 extensions = [

docs/quickstart.rst

@@ -123,7 +123,7 @@ For example, to split the data in 10 parts and start a client using the 8th part
          $env:FEDN_PACKAGE_EXTRACT_DIR="package"
          $env:FEDN_NUM_DATA_SPLITS=10
          $env:FEDN_DATA_PATH="./data/clients/8/mnist.pt"
-         fedn run client -in client.yaml --secure=True --force-ssl
+         fedn client start -in client.yaml --secure=True --force-ssl
 
 
 Start a training session

examples/FedSimSiam/client/python_env.yaml

@@ -6,4 +6,4 @@ build_dependencies:
 dependencies:
   - torch==2.2.0
   - torchvision==0.17.0
-  - fedn==0.9.0
+  - fedn==0.9.0

examples/huggingface/client/python_env.yaml

@@ -8,4 +8,4 @@ dependencies:
   - torchvision==0.17.1
   - fedn==0.9.0
   - transformers==4.39.3
-  - datasets==2.19.0
+  - datasets==2.19.0

examples/mnist-keras/client/python_env.yaml

@@ -2,8 +2,8 @@ name: mnist-keras
 build_dependencies:
   - pip
   - setuptools
-  - wheel==0.37.1
+  - wheel
 dependencies:
-  - tensorflow==2.13.1
+  - tensorflow>=2.13.1
   - fire==0.3.1
-  - fedn==0.9.0
+  - fedn

examples/mnist-keras/client/python_env_macosx.yaml

@@ -2,9 +2,9 @@ name: mnist-keras
 build_dependencies:
   - pip
   - setuptools
-  - wheel==0.37.1
+  - wheel
 dependencies:
   - tensorflow-macos
-  - tensorflow-metal 
+  - tensorflow-metal
   - fire==0.3.1
-  - fedn==0.9.0b2
+  - fedn

examples/mnist-keras/requirements.txt

@@ -1,3 +1,3 @@
-tensorflow==2.13.1
+tensorflow>=2.13.1
 fire==0.3.1
 docker==6.1.1

examples/mnist-pytorch/client/python_env.yaml

@@ -2,8 +2,8 @@ name: mnist-pytorch
 build_dependencies:
   - pip
   - setuptools
-  - wheel==0.37.1
+  - wheel
 dependencies:
-  - torch==2.2.1
-  - torchvision==0.17.1
-  - fedn==0.9.0
+  - torch==2.3.1
+  - torchvision==0.18.1
+  - fedn

fedn/network/api/client.py

@@ -571,7 +571,7 @@ def start_session(
         round_buffer_size: int = -1,
         delete_models: bool = True,
         validate: bool = True,
-        helper: str = "numpyhelper",
+        helper: str = "",
         min_clients: int = 1,
         requested_clients: int = 8,
     ):

fedn/network/api/interface.py

@@ -963,7 +963,7 @@ def start_session(
         round_buffer_size=-1,
         delete_models=True,
         validate=True,
-        helper="numpyhelper",
+        helper="",
         min_clients=1,
         requested_clients=8,
     ):
@@ -1002,13 +1002,17 @@ def start_session(
             return jsonify({"success": False, "message": "A session is already running."})
 
         # Check if compute package is set
-        if not self.statestore.get_compute_package():
+        package = self.statestore.get_compute_package()
+        if not package:
             return jsonify(
                 {
                     "success": False,
                     "message": "No compute package set. Set compute package before starting session.",
                 }
             )
+        if not helper:
+            # get helper from compute package
+            helper = package["helper"]
 
         # Check that initial (seed) model is set
         if not self.statestore.get_initial_model():

fedn/network/clients/client.py

@@ -10,8 +10,8 @@
 import time
 import uuid
 from datetime import datetime
-from distutils.dir_util import copy_tree
 from io import BytesIO
+from shutil import copytree
 
 import grpc
 from cryptography.hazmat.primitives.serialization import Encoding
@@ -22,11 +22,13 @@
 import fedn.network.grpc.fedn_pb2 as fedn
 import fedn.network.grpc.fedn_pb2_grpc as rpc
 from fedn.common.config import FEDN_AUTH_SCHEME, FEDN_PACKAGE_EXTRACT_DIR
-from fedn.common.log_config import logger, set_log_level_from_string, set_log_stream
+from fedn.common.log_config import (logger, set_log_level_from_string,
+                                    set_log_stream)
 from fedn.network.clients.connect import ConnectorClient, Status
 from fedn.network.clients.package import PackageRuntime
 from fedn.network.clients.state import ClientState, ClientStateToString
-from fedn.network.combiner.modelservice import get_tmp_path, upload_request_generator
+from fedn.network.combiner.modelservice import (get_tmp_path,
+                                                upload_request_generator)
 from fedn.utils.dispatcher import Dispatcher
 from fedn.utils.helpers.helpers import get_helper
 
@@ -340,7 +342,7 @@ def _initialize_dispatcher(self, config):
             }
             from_path = os.path.join(os.getcwd(), "client")
 
-            copy_tree(from_path, self.run_path)
+            copytree(from_path, self.run_path)
             self.dispatcher = Dispatcher(dispatch_config, self.run_path)
         # Get or create python environment
         activate_cmd = self.dispatcher._get_or_create_python_env()
@@ -709,6 +711,7 @@ def _send_heartbeat(self, update_frequency=2.0):
                     )
                     if self._missed_heartbeat > self.config["reconnect_after_missed_heartbeat"]:
                         self.disconnect()
+                        self._missed_heartbeat = 0
                 if status_code == grpc.StatusCode.UNAUTHENTICATED:
                     details = e.details()
                     if details == "Token expired":

fedn/network/clients/package.py

@@ -129,6 +129,7 @@ def unpack(self):
                 logger.info("Successfully extracted compute package content in {}".format(self.pkg_path))
                 # delete the tarball
                 logger.info("Deleting temporary package tarball file.")
+                f.close()
                 os.remove(os.path.join(self.pkg_path, self.pkg_name))
                 # search for file fedn.yaml in extracted package
                 for root, dirs, files in os.walk(self.pkg_path):

pyproject.toml

@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "fedn"
-version = "0.9.2"
+version = "0.9.6"
 description = "Scaleout Federated Learning"
 authors = [{ name = "Scaleout Systems AB", email = "[email protected]" }]
 readme = "README.rst"
@@ -24,9 +24,10 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
 ]
 
-requires-python = '>=3.8,<3.12'
+requires-python = '>=3.8,<3.13'
 dependencies = [
   "requests",
   "urllib3>=1.26.4",