Feature/SK-746 | Add python_env support in fedn.yaml (#556)

.ci/tests/examples/configure.sh

@@ -10,13 +10,12 @@ example="$1"
 
 >&2 echo "Configuring $example environment"
 pushd "examples/$example"
-bin/init_venv.sh
 
->&2 echo "Download and prepare data"
-bin/get_data
-bin/split_data
-
->&2 echo "Build compute package and seed"
-bin/build.sh
+# If example equals "mnist-keras"
+if [ "$example" == "mnist-keras" ]; then
 
+    >&2 echo "Download and prepare data"
+    bin/get_data
+    bin/split_data
+fi
 popd

.ci/tests/examples/print_logs.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
 echo "Minio logs"
-docker logs "$(basename $PWD)_minio_1"
+docker logs "$(basename $PWD)-minio-1"
 
 echo "Mongo logs"
-docker logs "$(basename $PWD)_mongo_1"
+docker logs "$(basename $PWD)-mongo-1"
 
 echo "API-Server logs"
-docker logs "$(basename $PWD)_api-server_1"
+docker logs "$(basename $PWD)-api-server-1"
 
 echo "Combiner logs"
-docker logs "$(basename $PWD)_combiner_1"
+docker logs "$(basename $PWD)-combiner-1"
 
 echo "Client 1 logs"
-docker logs "$(basename $PWD)_client_1"
+docker logs "$(basename $PWD)-client-1"
 
 echo "Client 2 logs"
-docker logs "$(basename $PWD)_client_2"
+docker logs "$(basename $PWD)-client-2"

.ci/tests/examples/run.sh

@@ -9,38 +9,45 @@ fi
 example="$1"
 helper="$2"
 
+python -m venv ".$example"
+source ".$example/bin/activate"
+".$example/bin/pip" install ./fedn/ fire
+
 >&2 echo "Start FEDn"
 pushd "examples/$example"
 
+"../../.$example/bin/fedn" package create --path client
+"../../.$example/bin/fedn" run build --path client
+
 docker compose \
     -f ../../docker-compose.yaml \
     -f docker-compose.override.yaml \
     up -d --build
 
 >&2 echo "Wait for reducer to start"
-".$example/bin/python" ../../.ci/tests/examples/wait_for.py reducer
+python ../../.ci/tests/examples/wait_for.py reducer
 
 >&2 echo "Wait for combiners to connect"
-".$example/bin/python" ../../.ci/tests/examples/wait_for.py combiners
+python ../../.ci/tests/examples/wait_for.py combiners
 
 >&2 echo "Upload compute package"
-".$example/bin/python" ../../.ci/tests/examples/api_test.py set_package --path package.tgz --helper "$helper"
-
->&2 echo "Upload seed"
-".$example/bin/python" ../../.ci/tests/examples/api_test.py set_seed --path seed.npz
+python ../../.ci/tests/examples/api_test.py set_package --path package.tgz --helper "$helper"
 
 >&2 echo "Wait for clients to connect"
-".$example/bin/python" ../../.ci/tests/examples/wait_for.py clients
+python ../../.ci/tests/examples/wait_for.py clients
+
+>&2 echo "Upload seed"
+python ../../.ci/tests/examples/api_test.py set_seed --path seed.npz
 
 >&2 echo "Start session"
-".$example/bin/python" ../../.ci/tests/examples/api_test.py start_session --rounds 3 --helper "$helper"
+python ../../.ci/tests/examples/api_test.py start_session --rounds 3 --helper "$helper"
 
 >&2 echo "Checking rounds success"
-".$example/bin/python" ../../.ci/tests/examples/wait_for.py rounds
+python ../../.ci/tests/examples/wait_for.py rounds
 
 >&2 echo "Test client connection with dowloaded settings"
 # Get config
-".$example/bin/python" ../../.ci/tests/examples/api_test.py get_client_config --output ../../client.yaml
+python ../../.ci/tests/examples/api_test.py get_client_config --output ../../client.yaml
 
 # Redeploy clients with config
 docker compose \
@@ -50,10 +57,10 @@ docker compose \
     up -d
 
 >&2 echo "Wait for clients to reconnect"
-".$example/bin/python" ../../.ci/tests/examples/wait_for.py clients
+python ../../.ci/tests/examples/wait_for.py clients
 
 >&2 echo "Test API GET requests"
-".$example/bin/python" ../../.ci/tests/examples/api_test.py test_api_get_methods
+python ../../.ci/tests/examples/api_test.py test_api_get_methods
 
 popd
 >&2 echo "Test completed successfully"

.ci/tests/examples/wait_for.py

@@ -6,8 +6,8 @@
 import pymongo
 import requests
 
-RETRIES = 18
-SLEEP = 10
+RETRIES = 30
+SLEEP = 20
 
 
 def _eprint(*args, **kwargs):

.github/workflows/branch-name-check.yaml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: branch name check
         run: |

.github/workflows/build-containers.yaml

@@ -21,7 +21,7 @@ jobs:
       contents: read
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Docker meta fedn
         id: meta1
@@ -95,5 +95,3 @@ jobs:
           tags: ${{ steps.meta3.outputs.tags }}
           labels: ${{ steps.meta3.outputs.labels }}
           file: Dockerfile
-          build-args: |
-            REQUIREMENTS=examples/mnist-pytorch/requirements.txt

.github/workflows/code-checks.yaml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: init venv
         run: .devcontainer/bin/init_venv.sh

.github/workflows/integration-tests.yaml

@@ -17,23 +17,17 @@ jobs:
         to_test:
           - "mnist-keras numpyhelper"
           - "mnist-pytorch numpyhelper"
-        python_version: ["3.8", "3.9","3.10","3.11"]
+        python_version: ["3.9","3.10"]
         os:
-          - ubuntu-20.04
           - ubuntu-22.04
     runs-on: ${{ matrix.os }}
     steps:
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python_version }}
-
-      - name: configure ${{ matrix.to_test }}
-        run: |
-          echo 'BASE_IMG=python:${{ matrix.python_version }}-slim' > .env 
-          .ci/tests/examples/configure.sh ${{ matrix.to_test }}
 
       - name: run ${{ matrix.to_test }}
         run: .ci/tests/examples/run.sh ${{ matrix.to_test }}

.gitignore

@@ -111,6 +111,8 @@ celerybeat-schedule
 # Environments
 .env
 .venv
+.mnist-pytorch
+.mnist-keras
 env/
 venv/
 ENV/

docker-compose.yaml

@@ -117,6 +117,7 @@ services:
   client:
     environment:
       - GET_HOSTS_FROM=dns
+      - FEDN_PACKAGE_EXTRACT_DIR=package
     build:
       context: .
       args:

docs/distributed.rst

@@ -1,8 +1,8 @@
-Distributed Deployment
+Distributed deployment
 ===================================
 
-This tutorial outlines the steps for deploying the FEDn framework over a **local network**, using a workstation as 
-the host and different devices as clients. For general steps on how to run FEDn, see one of the quickstart tutorials. 
+This tutorial outlines the steps for deploying the FEDn framework over a **local network**, using a single workstation or laptop as 
+the host, and different devices as clients. For general steps on how to run FEDn, see one of the quickstart tutorials. 
 
 
 .. note::
@@ -16,23 +16,27 @@ the host and different devices as clients. For general steps on how to run FEDn,
 Prerequisites
 -------------
 -  `One host workstation and atleast one client device`
--  `Python 3.8, 3.9 or 3.10 <https://www.python.org/downloads>`__
+-  `Python 3.8, 3.9, 3.10 or 3.11 <https://www.python.org/downloads>`__
 -  `Docker <https://docs.docker.com/get-docker>`__
 -  `Docker Compose <https://docs.docker.com/compose/install>`__
 
 Launch a distributed FEDn Network 
--------------
+---------------------------------------
 
 
 Start by noting your host's local IP address, used within your network. Discover it by running ifconfig on UNIX or 
 ipconfig on Windows, typically listed under inet for Unix and IPv4 for Windows.
 
-Continue with following the standard procedure to initiate a FEDn network, for example using by docker-compose. 
+Continue by following the standard procedure to initiate a FEDn network, for example using the provided docker-compose template. 
 Once the network is active, upload your compute package and seed (for comprehensive details, see the quickstart tutorials).
 
+.. note::
+   This guide covers general local networks where server and client may be on different hosts but able to communicate on their private IPs. 
+   A common scenario is also to run fedn and the clients on **localhost** on a single machine. In that case, you can replace <host local ip>
+   by "127.0.0.1" below.   
 
 Configuring and Attaching Clients
--------------
+---------------------------------------
 
 On your client device, continue with initializing your client. To connect to the host machine we need to ensure we are 
 routing the correct DNS to our hosts local IP address. We can do this using the standard FEDn `client.yaml`:
@@ -44,7 +48,7 @@ routing the correct DNS to our hosts local IP address. We can do this using the
    discover_port: 8092
 
 
-We can then run using docker by adding the hosts in the docker run command:
+We can then run a client using docker by adding the hostname:ip mapping in the docker run command:
 
 .. code-block::
 

docs/index.rst

@@ -12,7 +12,7 @@
    studio
    distributed
    apiclient
-   tutorial
+   projects
    architecture
    aggregators
    helpers

docs/introduction.rst

@@ -1,4 +1,4 @@
-Introduction to Federated Learning
+What is FEDn? 
 ==================================
 
 Federated Learning offers a novel approach to address challenges related to data privacy, security,