From d59d6d84e0d0cb19edb38c708a4af96150e8ace8 Mon Sep 17 00:00:00 2001 From: Andreas Hellander Date: Thu, 18 Apr 2024 14:31:38 +0200 Subject: [PATCH] Docs/update (#581) --- .ci/tests/examples/print_logs.sh | 9 +- .ci/tests/examples/run.sh | 15 +- .github/workflows/build-containers.yaml | 46 +--- .github/workflows/code-checks.yaml | 1 + .github/workflows/integration-tests.yaml | 2 +- README.rst | 8 +- docker-compose.yaml | 2 +- docs/_static/css/text.css | 1 + docs/conf.py | 3 +- docs/distributed.rst | 8 +- docs/faq.rst | 29 +-- docs/helpers.rst | 4 +- docs/introduction.rst | 62 +++-- docs/quickstart.rst | 232 +++++++++++------- docs/requirements.txt | 3 +- docs/studio.rst | 4 +- examples/README.md | 17 +- examples/mnist-keras/README.md | 45 ---- examples/mnist-keras/README.rst | 54 ++++ .../mnist-keras/docker-compose.override.yaml | 2 +- examples/mnist-pytorch/README.rst | 178 ++++++-------- examples/mnist-pytorch/client/data.py | 5 +- examples/mnist-pytorch/client/python_env.yaml | 2 +- .../docker-compose.override.yaml | 32 ++- 24 files changed, 381 insertions(+), 383 deletions(-) delete mode 100644 examples/mnist-keras/README.md create mode 100644 examples/mnist-keras/README.rst diff --git a/.ci/tests/examples/print_logs.sh b/.ci/tests/examples/print_logs.sh index 5b53aa8bb..e948195ba 100755 --- a/.ci/tests/examples/print_logs.sh +++ b/.ci/tests/examples/print_logs.sh @@ -12,7 +12,8 @@ echo "Combiner logs" docker logs "$(basename $PWD)-combiner-1" echo "Client 1 logs" -docker logs "$(basename $PWD)-client-1" - -echo "Client 2 logs" -docker logs "$(basename $PWD)-client-2" \ No newline at end of file +if [ "$example" == "mnist-keras" ]; then + docker logs "$(basename $PWD)-client-1" +else + docker logs "$(basename $PWD)-client1-1" +fi diff --git a/.ci/tests/examples/run.sh b/.ci/tests/examples/run.sh index 4f032a84f..e21ed2b2c 100755 --- a/.ci/tests/examples/run.sh +++ b/.ci/tests/examples/run.sh @@ -19,10 +19,17 @@ pushd "examples/$example" "../../.$example/bin/fedn" package create --path client "../../.$example/bin/fedn" run build --path client -docker compose \ - -f ../../docker-compose.yaml \ - -f docker-compose.override.yaml \ - up -d --build --scale client=1 +if [ "$example" == "mnist-keras" ]; then + docker compose \ + -f ../../docker-compose.yaml \ + -f docker-compose.override.yaml \ + up -d --build --scale client=1 +else + docker compose \ + -f ../../docker-compose.yaml \ + -f docker-compose.override.yaml \ + up -d --build combiner api-server mongo minio client1 +fi >&2 echo "Wait for reducer to start" python ../../.ci/tests/examples/wait_for.py reducer diff --git a/.github/workflows/build-containers.yaml b/.github/workflows/build-containers.yaml index 530beed09..8dd2f561a 100644 --- a/.github/workflows/build-containers.yaml +++ b/.github/workflows/build-containers.yaml @@ -35,33 +35,6 @@ jobs: type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} type=sha - - - name: Docker meta mnist-keras - id: meta2 - uses: docker/metadata-action@v4 - with: - images: | - docker.pkg.github.com/${{ github.repository }}/fedn - tags: | - type=ref,event=branch,suffix=-mnist-keras - type=ref,event=pr,suffix=-mnist-keras - type=semver,pattern={{version}},suffix=-mnist-keras - type=semver,pattern={{major}}.{{minor}},suffix=-mnist-keras - type=sha,suffix=-mnist-keras - - - name: Docker meta mnist-pytorch - id: meta3 - uses: docker/metadata-action@v4 - with: - images: | - docker.pkg.github.com/${{ github.repository }}/fedn - tags: | - type=ref,event=branch,suffix=-mnist-pytorch - type=ref,event=pr,suffix=-mnist-pytorch - type=semver,pattern={{version}},suffix=-mnist-pytorch - type=semver,pattern={{major}}.{{minor}},suffix=-mnist-pytorch - type=sha,suffix=-mnist-pytorch - - name: Log in to GitHub Container Registry uses: docker/login-action@v2 @@ -69,6 +42,7 @@ jobs: registry: docker.pkg.github.com username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push uses: docker/build-push-action@v4 @@ -77,21 +51,3 @@ jobs: tags: ${{ steps.meta1.outputs.tags }} labels: ${{ steps.meta1.outputs.labels }} file: Dockerfile - - - name: Build and push (mnist-keras) - uses: docker/build-push-action@v4 - with: - push: "${{ github.event_name != 'pull_request' }}" - tags: ${{ steps.meta2.outputs.tags }} - labels: ${{ steps.meta2.outputs.labels }} - file: Dockerfile - build-args: | - REQUIREMENTS=examples/mnist-keras/requirements.txt - - - name: Build and push (mnist-pytorch) - uses: docker/build-push-action@v4 - with: - push: "${{ github.event_name != 'pull_request' }}" - tags: ${{ steps.meta3.outputs.tags }} - labels: ${{ steps.meta3.outputs.labels }} - file: Dockerfile diff --git a/.github/workflows/code-checks.yaml b/.github/workflows/code-checks.yaml index 497f99a65..939be75e7 100644 --- a/.github/workflows/code-checks.yaml +++ b/.github/workflows/code-checks.yaml @@ -45,6 +45,7 @@ jobs: --exclude-dir='docs' --exclude-dir='flower-client' --exclude='tests.py' + --exclude='README.rst' '^[ \t]+(import|from) ' -I . # TODO: add linting/formatting for all file types \ No newline at end of file diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index 802bfc5a6..44bb3d008 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -17,7 +17,7 @@ jobs: to_test: - "mnist-keras numpyhelper" - "mnist-pytorch numpyhelper" - python_version: ["3.9","3.10", "3.11"] + python_version: ["3.8","3.9","3.10", "3.11"] os: - ubuntu-22.04 runs-on: ${{ matrix.os }} diff --git a/README.rst b/README.rst index d7113daf4..a5e73cdd5 100644 --- a/README.rst +++ b/README.rst @@ -57,7 +57,7 @@ Getting started The best way to get started is to take the quickstart tutorial: -- `Quickstart `__ +- `Quickstart `__ Documentation ============= @@ -72,8 +72,8 @@ Running your project in FEDn Studio (SaaS or on-premise) The FEDn Studio SaaS is free for development, testing and research (one project per user, backend compute resources sized for dev/test): -- `Register for a free account in FEDn Studio `__ -- `Take the tutorial to deploy your project on FEDn Studio `__ +- `Register for a free account in FEDn Studio `__ +- `Take the tutorial to deploy your project on FEDn Studio `__ Scaleout can also support users to scale up experiments and demonstrators on Studio, by granting custom resource quotas. Additonally, charts are available for self-managed deployment on-premise or in your cloud VPC (all major cloud providers). Contact the Scaleout team for more information. @@ -91,7 +91,7 @@ Making contributions All pull requests will be considered and are much appreciated. For more details please refer to our `contribution -guidelines `__. +guidelines `__. Citation ======== diff --git a/docker-compose.yaml b/docker-compose.yaml index ca6045301..3f2f3c2be 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,5 @@ # Compose schema version -version: '3.3' +version: '3.4' # Setup network networks: diff --git a/docs/_static/css/text.css b/docs/_static/css/text.css index 83f04637d..2008de521 100644 --- a/docs/_static/css/text.css +++ b/docs/_static/css/text.css @@ -34,6 +34,7 @@ body { a { color: var(--scaleout-black); + font-weight: bold; text-decoration: none; display: inline-block; } diff --git a/docs/conf.py b/docs/conf.py index 026266359..8b9d898a6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,7 +23,8 @@ 'sphinx.ext.mathjax', 'sphinx.ext.ifconfig', 'sphinx.ext.viewcode', - 'sphinx_rtd_theme' + 'sphinx_rtd_theme', + 'sphinx_code_tabs' ] # The master toctree document. diff --git a/docs/distributed.rst b/docs/distributed.rst index 608f89f18..13803dd3f 100644 --- a/docs/distributed.rst +++ b/docs/distributed.rst @@ -1,13 +1,13 @@ -Distributed deployment -====================== +Self-managed distributed deployment +=================================== This tutorial outlines the steps for deploying the FEDn framework over a **local network**, using a single workstation or laptop as -the host, and different devices as clients. For general steps on how to run FEDn, see one of the quickstart tutorials. +the host for the servier-side components, and other hosts or devices as clients. For general steps on how to run FEDn, see the quickstart tutorials. .. note:: For a secure and production-grade deployment solution over **public networks**, explore the FEDn Studio service at - **studio.scaleoutsystems.com**. + **fedn.scaleoutsystems.com**. Alternatively follow this tutorial substituting the hosts local IP with your public IP, open the neccesary ports (see which ports are used in docker-compose.yaml), and ensure you have taken additional neccesary security diff --git a/docs/faq.rst b/docs/faq.rst index e7a0ed28b..f817a0dee 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -19,17 +19,6 @@ However, during development of a new model it will be necessary to reinitialize. 2. Restart the clients. -Q: Can I skip fetching the remote package and instead use a local folder when developing the compute package ------------------------------------------------------------------------------------------------------------- - -Yes, to facilitate interactive development of the compute package you can start a client that uses a local folder 'client' in your current working directory by: - -.. code-block:: bash - - fedn run client --remote=False -in client.yaml - - -Note that in production federations this options should in most cases be disallowed. Q: How can other aggregation algorithms can be defined? ------------------------------------------------------- @@ -39,10 +28,10 @@ There is a plugin interface for extending the framework with new aggregators. Se :ref:`agg-label` -Q: What is needed to include other ML frameworks in FEDn like sklearn, xgboost, etc.? +Q: What is needed to include additional ML frameworks in FEDn? ------------------------------------------------------------------------------------- -You need to make sure that FEDn knows how to serialize and deseralize the model object into paramters. If you can +You need to make sure that FEDn knows how to serialize and deserialize the model object. If you can serialize to a list of numpy ndarrays in your compute package entrypoint (see the Quickstart Tutorial code), you can use the built in "numpyhelper". If this is not possible, you can extend the framework with a custom helper, see the section about model marshaling: @@ -62,21 +51,21 @@ Yes! You can toggle which message streams a client subscibes to when starting th Q: How do you approach the question of output privacy? ---------------------------------------------------------------------------------- -We take security in (federated) machine learning very seriously. Federated learning is a foundational technology that impoves input privacy +We take security in (federated) machine learning seriously. Federated learning is a foundational technology that impoves input privacy in machine learning by allowing datasets to stay local and private, and not copied to a server. FEDn is designed to provide an industry grade implementation of the core communication and aggregration layers of federated learning, as well as configurable modules for traceability, logging etc, to allow the developer balance between privacy and auditability. With `FEDn Studio `__ we add functionality for user authentication, authorization, and federated client identity management. As such, The FEDn Framework provides a comprehensive software suite for implemeting secure federated learning following industry best-practices. -Going beyond input privacy, there are several additional considerations relating to output privacy and potential attacks on (federated) machine learning systems. For an -introduction to the topic, see this blog post: +Going beyond input privacy, there are several additional considerations relating to output privacy and potential attacks on (federated) machine learning systems. +For an introduction to the topic, see this blog post: - `Output Privacy and Federated Machine Learning `__ -Striking the appropriate balance between system complexity and secturity becomes a use-case dependent endeavor, and we are happy to -engage in detailed conversations about this. As an example, one might consider layering differential privacy on top of the aggregation -to protect against a honest-but-curious server, at the price of a loss of accuracy for the global model. Depending on the privacy requirements, +Striking the appropriate balance between system complexity and security becomes a use-case dependent endeavor, and we are happy to +support projects with guidance on these matters. For an example, one might consider layering differential privacy on top of the aggregation +to protect against an honest-but-curious server, at the price of a reduced accuracy for the global model. Depending on the privacy requirements, the model type, the amount of data, the number of local updates possible during training etc, this may or may not be necessary. We are engaged in several cybersecurity projects focused on federated machine learning, do not hesitate to reach out to discuss further @@ -84,5 +73,5 @@ with the Scaleout team. - `LEAKPRO: Leakage Profiling and Risk Oversight for Machine Learning Models `__ - `Validating a System Development Kit for edge federated learning `__ -- `Truseted Execution Environments for Federated Learning: `__ +- `Trusted Execution Environments for Federated Learning: `__ - `Robust IoT Security: Intrusion Detection Leveraging Contributions from Multiple Systems `__ \ No newline at end of file diff --git a/docs/helpers.rst b/docs/helpers.rst index 9adcc94b7..8f89a8317 100644 --- a/docs/helpers.rst +++ b/docs/helpers.rst @@ -1,7 +1,7 @@ .. _helper-label: -Model Serialization/Deserialization - Helpers -============================================= +Model Serialization/Deserialization +=================================== In federated learning, model updates need to be serialized and deserialized in order to be transferred between clients and server/combiner. There is also a need to write and load models diff --git a/docs/introduction.rst b/docs/introduction.rst index 9dd2d87e7..0a6a59879 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -7,53 +7,51 @@ Federated Learning allows for collaborative model training while keeping data lo scenarios where data cannot be easily shared due to privacy regulations, network limitations, or ownership concerns. At its core, Federated Learning orchestrates model training across distributed devices or servers, referred to as clients or participants. -These participants could be diverse endpoints such as mobile devices, IoT gadgets, or remote servers. Rather than transmitting raw data to a central location, +These participants could be diverse endpoints such as mobile devices, IoT gateways, or remote servers. Rather than transmitting raw data to a central location, each participant computes gradients locally based on its data. These gradients are then communicated to a server, often called the aggregator. The server aggregates and combines the gradients from multiple participants to update a global model. This iterative process allows the global model to improve without the need to share the raw data. -**FEDn: the SDK for scalable federated learning** +FEDn empowers users to create federated learning applications that seamlessly transition from local proofs-of-concept to secure distributed deployments. +We develop the FEDn framework following these core design principles: -FEDn serves as a System Development Kit (SDK) enabling scalable federated learning. -It is used to implement the core server side logic (including model aggregation) and the client side integrations. -Developers and ML engineers can use FEDn to build custom federated learning systems and bespoke deployments. +- **Seamless transition from proof-of-concepts to real-world FL**. FEDn has been designed to make the journey from R&D to real-world deployments as smooth as possibe. Develop your federated learning use case in a pseudo-local environment, then deploy it to FEDn Studio (cloud or on-premise) for real-world scenarios. No code change is required to go from development and testing to production. +- **Designed for scalability and resilience.** FEDn enables model aggregation through multiple aggregation servers sharing the workload. A hierarchical architecture makes the framework well suited borh for cross-silo and cross-device use-cases. FEDn seamlessly recover from failures in all critical components, and manages intermittent client-connections, ensuring robust deployment in production environments. -One of the standout features of FEDn is its ability to deploy and scale the server-side in geographically distributed setups, -adapting to varying project needs and geographical considerations. +- **Secure by design.** FL clients do not need to open any ingress ports, facilitating distributed deployments across a wide variety of settings. Additionally, FEDn utilizes secure, industry-standard communication protocols and supports token-based authentication and RBAC for FL clients (JWT), providing flexible integration in production environments. +- **Developer and data scientist friendly.** Extensive event logging and distributed tracing enables developers to monitor experiments in real-time, simplifying troubleshooting and auditing. Machine learning metrics can be accessed via both a Python API and visualized in an intuitive UI that helps the data scientists analyze and communicate ML-model training progress. -**Scalable and Resilient** -FEDn exhibits scalability and resilience, thanks to its tiered architecture. Multiple aggregation servers, in FEDn called combiners, -form a network to divide the workload of coordinating clients and aggregating models. -This architecture allows for high performance in various settings, from thousands of clients in a cross-device environment to -large model updates in a cross-silo scenario. Importantly, FEDn has built-in recovery capabilities for all critical components, enhancing system reliability. +Features +========= -**ML-Framework Agnostic** +Federated machine learning: -With FEDn, model updates are treated as black-box computations, meaning it can support any ML model type or framework. -This flexibility allows for out-of-the-box support for popular frameworks like Keras and PyTorch, making it a versatile tool for any machine learning project. +- Support for any ML framework (e.g. PyTorch, Tensforflow/Keras and Scikit-learn) +- Extendable via a plug-in architecture (aggregators, load balancers, object storage backends, databases etc.) +- Built-in federated algorithms (FedAvg, FedAdam, FedYogi, FedAdaGrad, etc.) +- CLI and Python API client for running FEDn networks and coordinating experiments. +- Implement clients in any language (Python, C++, Kotlin etc.) +- No open ports needed client-side. -**Security** -A key security feature of FEDn is its client protection capabilities - clients do not need to expose any ingress ports, -thus reducing potential security vulnerabilities. +FEDn Studio - From development to FL in production: -**Event Tracking and Training progress** +- Leverage Scaleout's free managed service for development and testing in real-world scenarios (SaaS). +- Token-based authentication (JWT) and role-based access control (RBAC) for FL clients. +- REST API and UI. +- Data science dashboard for orchestrating experiments and visualizing results. +- Admin dashboard for managing the FEDn network and users/clients. +- View extensive logging and tracing information. +- Collaborate with other data-scientists on the project specification in a shared workspace. +- Cloud or on-premise deployment (cloud-native design, deploy to any Kubernetes cluster) -To ensure transparency and control over the training process, as well as to provide means to troubleshoot distributed deployments, -FEDn logs events and does real-time tracking of training progress. A flexible API lets the user define validation strategies locally on clients. -Data is logged as JSON to MongoDB, enabling users to create custom dashboards and visualizations easily. +Support +========= -**REST-API and Python API Client and CLI** +Community support in available in our `Discord +server `__. -FEDn comes with an REST-API, a CLI and a Python API Client for programmatic interaction with a FEDn network. This allows for flexible automation of experiments, for integration with -other systems, and for easy integration with external dashboards and visualization tools. - -FEDn Studio ------------ - -FEDn Studio is a web-based tool for managing and monitoring federated learning experiments. It provides the FEDn network as a managed service, as well as a user-friendly interface for monitoring the progress of training and visualizing the results. FEDn Studio is available as a SaaS at fedn.scaleoutsystems.com . It is free for development, testing and research (one project per user, backend compute resources sized for dev/test). - -Scaleout can also support users to scale up experiments and demonstrators on Studio, by granting custom resource quotas. Additonally, charts are available for self-managed deployment on-premise or in your cloud VPC (all major cloud providers). Contact the Scaleout team for more information. \ No newline at end of file +Options are available for `Enterprise support `__. \ No newline at end of file diff --git a/docs/quickstart.rst b/docs/quickstart.rst index f6e17c8fd..571d4cd08 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -2,47 +2,19 @@ Getting started with FEDn ========================= .. note:: - This tutorial is a quickstart guide to FEDn based on a pre-made FEDn Project. It is desinged to serve as a minimalistic starting point for developers. - To learn how to develop your own federated machine learning projects with FEDn, see :ref:`projects-label`. - -This classic example of hand-written text recognition is well suited as a lightweight test when developing on FEDn in pseudo-distributed mode or in FEDn Studio. -A normal high-end laptop or a workstation should be able to sustain a few clients. -The example automates the partitioning of data and deployment of a variable number of clients on a single host. -We here assume working experience with containers, Docker and docker-compose if you are running in pseudo-distributed mode. -For a details on FEDn Studio UI, see :ref:`studio`. + This tutorial is a quickstart guide to FEDn based on a pre-made FEDn Project. It is designed to serve as a minimalistic starting point for developers. + To learn about FEDn Projects in order to develop your own federated machine learning projects, see :ref:`projects-label`. -**Prerequisites using FEDn Studio (recommended)** +**Prerequisites** - `Python >=3.8, <=3.11 `__ +- `A FEDn Studio account `__ -**Prerequisites for pseudo-distributed mode** -- `Docker `__ -- `Docker Compose `__ - - -In pseudo-distributed mode --------------------------- - -.. note:: - This is not required if you are using FEDn Studio! - -In pseudo-distributed mode, you can start a FEDn network using docker-compose. -Clone this repository, locate into it and start a pseudo-distributed FEDn network using docker-compose: - -.. code-block:: - - docker-compose up - -This starts up the needed backend services MongoDB and Minio, the API Server and one Combiner. -You can verify the deployment using these urls: - -- API Server: http://localhost:8092/get_controller_status -- Minio: http://localhost:9000 -- Mongo Express: http://localhost:8081 +Set up a FEDn Studio Project +---------------------------- -.. warning:: - The FEDn network is configured to use a local Minio and MongoDB instances for storage. This is not suitable for production, but is fine for testing. +Start by creating an account in FEDn Studio and set up a project by following the instruction here: :ref:`studio`. Install FEDn ------------ @@ -69,14 +41,17 @@ It is recommended to use a virtual environment when installing FEDn. .. _package-creation: -Prepare the package and seed model ----------------------------------- +Initialize FEDn with the client code bundle and seed model +---------------------------------------------------------- -Next, we will prepare the client. A key concept in FEDn is the package - -a code bundle that contains entrypoints for training and (optionally) validating a model update on the client. +Next, we will prepare the client. The key part of a FEDn Project is the client definition - +code that contains entrypoints for training and (optionally) validating a model update on the client. -Locate into ``examples/mnist-pytorch`` and familiarize yourself with the project structure. The dependencies needed in the client environment are specified in -in ``client/python_env.yaml``. +Locate into ``examples/mnist-pytorch`` and familiarize yourself with the project structure. The dependencies needed in the client environment are specified +in ``client/python_env.yaml``. + +In order to train a federated model using FEDn, your Studio project needs to be initialized with a compute package and a seed model. The compute package is a bundle +of the client specification, and the seed model is a first version of the global model. Create a package of the fedn project (assumes your current working directory is in the root of the project /examples/mnist-pytorch): @@ -92,63 +67,67 @@ Next, run the build entrypoint defined in ``client/fedn.yaml`` to build the mode fedn run build --path client -This will create a seed model called 'seed.npz' in the root of the project. +This will create a seed model called 'seed.npz' in the root of the project. We will now upload these to your Studio project using the FEDn APIClient. **Upload the package and seed model** .. note:: - If you are using FEDn Studio, you need to create an admin token and use the token to authenticate the APIClient. - Do this by going to the 'Settings' tab in FEDn Studio and click 'Generate token'. Copy the access token and use it in the APIClient. - The controller host can be found on the dashboard in FEDn Studio. + You need to create an API admin token and use the token to authenticate the APIClient. + Do this by going to the 'Settings' tab in FEDn Studio and click 'Generate token'. Copy the access token and use it in the APIClient below. + The controller host can be found on the main Dashboard in FEDn Studio. You can also upload the file via the FEDn Studio UI. Please see :ref:`studio-upload-files` for more details. -Upload the package and seed model to FEDn controller using the APIClient: +Upload the package and seed model using the APIClient: .. code:: python >>> from fedn import APIClient - >>> client = APIClient(host="localhost", port=8092) + >>> client = APIClient(host="", token="", secure=True, verify=True) >>> client.set_active_package("package.tgz", helper="numpyhelper") >>> client.set_active_model("seed.npz") -.. note:: - If you are using FEDn Studio, you need to authenticate the APIClient by setting the access token: - - .. code:: python - - client = APIClient(host=, token=, secure=True, verify=True) Configure and attach clients ---------------------------- -**Pseudo-distributed mode** +Each local client needs an access token in order to connect. These tokens are issued from your Studio Project. Go to the 'Clients' tab and click 'Connect client'. +Download a client configuration file and save it to the root of the examples/mnist-pytorch folder. Rename the file to 'client.yaml'. +Then start the client by running the following command in the root of the project: -In pseudo-distributed mode, you can start a client using the provided docker compose template in the root of the project. .. code-block:: - docker-compose -f ../../docker-compose.yaml -f docker-compose.override.yaml up --scale client=2 + fedn run client -in client.yaml --secure=True --force-ssl +Repeat the above for the number of clients you want to use. A normal laptop should be able to handle several clients for this example. -This will build a container image for the client, start two clients and connect them to local API server. +**Modifing the data split:** -.. note:: +The default traning and test data for this example (MNIST) is for convenience downloaded and split by the client when it starts up (see 'startup' entrypoint). +The number of splits and which split used by a client can be controlled via the environment variables ``FEDN_NUM_DATA_SPLITS`` and ``FEDN_DATA_PATH``. +For example, to split the data in 10 parts and start a client using the 8th partiton: - In FEDn Studio, you can configure and attach clients to the network. Go to the 'Clients' tab and click 'Connect client'. - Download the client configuration file and save it to the root of the examples/mnist-pytorch folder. Rename the file to 'client.yaml'. - Then start the client by running the following command in the root of the project: +.. tabs:: - .. code-block:: + .. code-tab:: bash + :caption: Unix/MacOS - export FEDN_AUTH_SCHEME=Bearer - fedn run client -in client.yaml --secure=True --force-ssl + export FEDN_PACKAGE_EXTRACT_DIR=package + export FEDN_NUM_DATA_SPLITS=10 + export FEDN_DATA_PATH=package/data/clients/8/mnist.pt + fedn run client -in client.yaml --secure=True --force-ssl -Start a training session ------------------------- + .. code-tab:: bash + :caption: Windows (Powershell) -.. note:: + $env:FEDN_PACKAGE_EXTRACT_DIR="package" + $env:FEDN_NUM_DATA_SPLITS=10 + $env:FEDN_DATA_PATH="package/data/clients/8/mnist.pt" + fedn run client -in client.yaml --secure=True --force-ssl - In FEDn Studio, you can start a training session by going to the 'Sessions' tab and click 'Start session'. + +Start a training session +------------------------ You are now ready to start training the model using the APIClient: @@ -163,51 +142,126 @@ You are now ready to start training the model using the APIClient: # Show model performance: >>> client.get_validations() -Please see :py:mod:`fedn.network.api` for more details on the APIClient. - -There is also a Jupyter `Notebook `_ version of this tutorial including examples of how to fetch and visualize model validations. -Access logs and validation data from MongoDB --------------------------------------------- -You can access and download event logs and validation data via the API. If your are running in pseudo-distributed mode, you can access the MongoDB backend directly. -Either using pymongo (or other mongo clients) or via the MongoExpress interface: +Please see :py:mod:`fedn.network.api` for more details on the APIClient. -- http://localhost:8081/db/fedn-network/ +.. note:: -The credentials are as set in docker-compose.yaml in the root of the repository. + In FEDn Studio, you can start a training session by going to the 'Sessions' tab and click 'Start session'. See :ref:`studio` for a + step-by-step guide for how to control experiments using the UI. Access model updates -------------------- - .. note:: - In FEDn Studio, you can access model updates by going to the 'Models' or 'Sessions' tab. Here you can download model updates, metrics (as csv) and view the model trail. + In FEDn Studio, you can access global model updates by going to the 'Models' or 'Sessions' tab. Here you can download model updates, metrics (as csv) and view the model trail. -You can access model updates via the APIClient: +You can access global model updates via the APIClient: .. code:: python >>> ... >>> client.download_model("", path="model.npz") -.. note:: - If running in pseudo-distributed mode, you can access model updates via the Minio interface. - You can obtain model updates from the 'fedn-models' bucket: - - - http://localhost:9000 +**Connecting clients using Docker** -**Clean up** -If you are running in pseudo-distributed mode, you can stop the network using docker-compose: +You can also use Docker to containerize the client. +For convenience, there is a Docker image hosted on ghrc.io with fedn preinstalled. +To start a client using Docker: .. code-block:: - docker-compose down + docker run \ + -v $PWD/client.yaml:/app/client.yaml \ + -e FEDN_PACKAGE_EXTRACT_DIR=package \ + -e FEDN_NUM_DATA_SPLITS=2 \ + -e FEDN_DATA_PATH=/app/package/data/clients/1/mnist.pt \ + ghcr.io/scaleoutsystems/fedn/fedn:0.9.0 run client -in client.yaml --force-ssl --secure=True + **Where to go from here?** -With you first FEDn federation deployed, we suggest that you take a close look at how a FEDn project is structured +With you first FEDn federation set up, we suggest that you take a close look at how a FEDn project is structured and how you develop your own FEDn projects: - :ref:`projects-label` + + +Local development deployment (using docker compose) +---------------------------------------------------------- + +.. note:: + These instructions are for users wanting to set up a local development deployment of FEDn (wihout Studio). + This requires basic knowledge of Docker and docker-compose. + The main use-case for this is rapid iteration while developing the FEDn Project, + development of aggregator plugins, etc. + +Follow the steps above to install FEDn, generate 'package.tgz' and 'seed.tgz'. Then, instead of +using a Studio project for a managed FEDn server-side, start a local FEDn network +using docker-compose: + +.. code-block:: + + docker compose \ + -f ../../docker-compose.yaml \ + -f docker-compose.override.yaml \ + up + +This starts up local services for MongoDB, Minio, the API Server, one Combiner and two clients. +You can verify the deployment using these urls: + +- API Server: http://localhost:8092/get_controller_status +- Minio: http://localhost:9000 +- Mongo Express: http://localhost:8081 + +Upload the package and seed model to FEDn controller using the APIClient. In Python: + +.. code-block:: + + from fedn import APIClient + client = APIClient(host="localhost", port=8092) + client.set_active_package("package.tgz", helper="numpyhelper") + client.set_active_model("seed.npz") + +You can now start a training session with 5 rounds (default): + +.. code-block:: + + client.start_session() + +**Automate experimentation with several clients** + +If you want to scale the number of clients, you can do so by modifying ``docker-compose.override.yaml``. For example, +in order to run with 3 clients, change the environment variable ``FEDN_NUM_DATA_SPLITS`` to 3, and add one more client +by copying ``client1`` and setting ``FEDN_DATA_PATH`` to ``/app/package/data/clients/3/mnist.pt`` + + +**Access message logs and validation data from MongoDB** + +You can access and download event logs and validation data via the API, and you can also as a developer obtain +the MongoDB backend data using pymongo or via the MongoExpress interface: + +- http://localhost:8081/db/fedn-network/ + +The credentials are as set in docker-compose.yaml in the root of the repository. + +**Access global models** + +You can obtain global model updates from the 'fedn-models' bucket in Minio: + +- http://localhost:9000 + +**Reset the FEDn deployment** + +To purge all data from a deployment incuding all session and round data, access the MongoExpress UI interface and +delete the entire ``fedn-network`` collection. Then restart all services. + +**Clean up** + +You can clean up by running + +.. code-block:: + + docker-compose -f ../../docker-compose.yaml -f docker-compose.override.yaml down -v diff --git a/docs/requirements.txt b/docs/requirements.txt index 4170c03ef..46a0b5e64 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1 +1,2 @@ -sphinx-rtd-theme \ No newline at end of file +sphinx-rtd-theme +sphinx_code_tabs diff --git a/docs/studio.rst b/docs/studio.rst index 39a3cfa5c..26e4615a0 100644 --- a/docs/studio.rst +++ b/docs/studio.rst @@ -3,14 +3,14 @@ Studio =============== -FEDn Studio is a web-based tool for managing and monitoring federated learning experiments. It provides the FEDn network as a managed service, as well as a user-friendly interface for monitoring the progress of training and visualizing the results. FEDn Studio is available as a SaaS at `studio.scaleoutsystems.com `_ . It is free for development, testing and research (one project per user, backend compute resources sized for dev/test). +FEDn Studio is a web-based tool for managing and monitoring federated learning experiments. It provides the FEDn network as a managed service, as well as a user-friendly interface for monitoring the progress of training and visualizing the results. FEDn Studio is available as a SaaS at `fedn.scaleoutsystems.com `_ . It is free for development, testing and research (one project per user, backend compute resources sized for dev/test). Scaleout can also support users to scale up experiments and demonstrators on Studio, by granting custom resource quotas. Additonally, charts are available for self-managed deployment on-premise or in your cloud VPC (all major cloud providers). Contact the Scaleout team for more information. Getting started --------------- -Before you can start using Studio, you will need an account. Head over to `studio.scaleoutsystems.com/signup `_ and sign up. +Before you can start using Studio, you will need an account. Head over to `fedn.scaleoutsystems.com/signup `_ and sign up. **Create a project** diff --git a/examples/README.md b/examples/README.md index 27f3dce63..ae4a89c75 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,18 +1,3 @@ ## Examples -The examples distributed here in this folder are maintained by Scaleout. -We recommend all new users to start with the [Quickstart Tutorial (PyTorch)](https://github.com/scaleoutsystems/fedn/tree/master/examples/mnist-pytorch). - -### External examples -Below we maintain a list of examples provided both by the Scaleout core team and users. They may or may not be tested with the latest release of FEDn, please refer to the README of each specific project/example for detail. -If you have a project that you want to include in this list, talk to a core developer in [Discord](https://discord.gg/CCRgjpMsVA). - -- [C++ version of the MNIST example](https://github.com/scaleoutsystems/examples) Also executes in Intel SGX TEE via Gramine. -- [Human activity recognition use case from the FEDn paper](https://github.com/scaleoutsystems/examples) IoT/cross-device. -- [Sentiment analysis use case from the FEDn paper](https://github.com/scaleoutsystems/examples) NLP. -- [FedBird - Federated object detection for Baltic Seabirds](https://github.com/aidotse/fedbird) Project by AI Sweden and Zenseact. -- [FEDn in AI Sweden EdgeLab](https://github.com/aidotse/EdgeLab/tree/main/tutorials/fedn) Instructions for running fully distributed FEDn in [EdgeLab](https://www.ai.se/en/data-factory/edge-lab). -- [FedLM - Federatad Electra.](https://github.com/Kungbib/fedLM) NLP project by the Swedish Royal Library. -- [Fraud detection with a Keras auto-encoder (ANN encoder)](https://github.com/scaleoutsystems/FEDn-client-fraud_keras) -- [VGG16 trained on cifar-10 with a PyTorch client (cross-silo)](https://github.com/scaleoutsystems/FEDn-client-cifar10-pytorch) -- [Sentiment analysis with a PyTorch CNN trained on the IMDB dataset (cross-silo)](https://github.com/scaleoutsystems/FEDn-client-imdb-pytorch.git) +This folder contains examples of [FEDn Projects](https://fedn.readthedocs.io/en/latest/projects.html). diff --git a/examples/mnist-keras/README.md b/examples/mnist-keras/README.md deleted file mode 100644 index ced812c21..000000000 --- a/examples/mnist-keras/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# MNIST (TensorFlow/Keras version) - -This is a mimimalistic TF/Keras version of the Quickstart Tutorial (PyTorch). For more detailed explaination including a Jupyter Notebook with -examples of API usage for starting and interacting with federated experiments, refer to that tutorial. - -## Prerequisites -- [Python 3.8, 3.9, 3.10 or 3.11](https://www.python.org/downloads) -- [Docker](https://docs.docker.com/get-docker) -- [Docker Compose](https://docs.docker.com/compose/install) - -## Running the example (pseudo-distributed) -Clone FEDn and locate into this directory. -```sh -git clone https://github.com/scaleoutsystems/fedn.git -cd fedn/examples/mnist-keras -``` - -### Build the compute package and the seed model (model to initalize the global model trail) - -```sh -fedn package create --path client -``` - -```sh -fedn run build --path client -``` - -> You will now have two files, `package.tgz` and `seed.npz`. - -### Deploy FEDn -Now we are ready to deploy FEDn and two clients with `docker-compose`. - -``` -docker-compose -f ../../docker-compose.yaml -f docker-compose.override.yaml up -``` - -> **Note**: run with `--scale client=N` to start *N* clients. - -### Run federated training -Refer to this notebook to upload the package and seed model and run experiments: - - https://github.com/scaleoutsystems/fedn/blob/master/examples/mnist-pytorch/API_Example.ipynb - -## Clean up -You can clean up by running `docker-compose down`. diff --git a/examples/mnist-keras/README.rst b/examples/mnist-keras/README.rst new file mode 100644 index 000000000..0f2ec9cf1 --- /dev/null +++ b/examples/mnist-keras/README.rst @@ -0,0 +1,54 @@ +FEDn Project: Keras/Tensorflow (MNIST) +------------------------------------------- + +This is a TF/Keras version of the Quickstart Tutorial (PyTorch) FEDn Project. For a step-by-step guide, refer to that tutorial. + + **Note: We recommend all new users to start by following the Quickstart Tutorial: https://fedn.readthedocs.io/en/latest/quickstart.html** + +Prerequisites +------------------------------------------- + +- `Python 3.8, 3.9, 3.10 or 3.11 `__ + +Creating the compute package and seed model +------------------------------------------- + +Install fedn: + +.. code-block:: + + pip install fedn + +Clone this repository, then locate into this directory: + +.. code-block:: + + git clone https://github.com/scaleoutsystems/fedn.git + cd fedn/examples/mnist-pytorch + +Create the compute package: + +.. code-block:: + + fedn package create --path client + +This should create a file 'package.tgz' in the project folder. + +Next, generate a seed model (the first model in a global model trail): + +.. code-block:: + + fedn run build --path client + +This step will take a few minutes, depending on hardware and internet connection (builds a virtualenv). + +Using FEDn Studio +------------------ + +To set up your FEDn Studio project and learn how to connect clients (using JWT token authentication), follow this guide: https://fedn.readthedocs.io/en/latest/studio.html. On the +step "Upload Files", upload 'package.tgz' and 'seed.npz' created above. + +Using pseudo-distributed mode (local development) +------------------------------------------------- + +See the PyTorch version of this example for detailed instructions on how to deploy FEDn in `local development mode <../mnist-pytorch/README.rst>`_. diff --git a/examples/mnist-keras/docker-compose.override.yaml b/examples/mnist-keras/docker-compose.override.yaml index a7bcaa5f4..3a9dccb3c 100644 --- a/examples/mnist-keras/docker-compose.override.yaml +++ b/examples/mnist-keras/docker-compose.override.yaml @@ -1,5 +1,5 @@ # Compose schema version -version: '3.3' +version: '3.4' # Overriding requirements services: diff --git a/examples/mnist-pytorch/README.rst b/examples/mnist-pytorch/README.rst index eaa09c435..4fa022bfd 100644 --- a/examples/mnist-pytorch/README.rst +++ b/examples/mnist-pytorch/README.rst @@ -1,26 +1,29 @@ -Quickstart Tutorial PyTorch (MNIST) ------------------------------------ - -This is an example FEDn Project based on the classic hand-written text recognition dataset MNIST. -The project is well suited as a lightweight test when developing on FEDn in pseudo-distributed mode. -A normal high-end laptop or a workstation (CPU only or GPU) should be able to sustain a few clients. -The example automates the partitioning of data and deployment of a variable number of clients on a single host. -We here assume working experience with containers, Docker and docker-compose. - +FEDn Project: MNIST (PyTorch) +----------------------------- + +This is an example FEDn Project based on the classic hand-written text recognition dataset MNIST. +The example is intented as a minimalistic quickstart and automates the handling of training data +by letting the client download and create its partition of the dataset as it starts up. + + **Note: These instructions are geared towards users seeking to learn how to work + with FEDn in local development mode using Docker/docker-compose. We recommend all new users + to start by following the Quickstart Tutorial: https://fedn.readthedocs.io/en/stable/quickstart.html** + Prerequisites ------------- Using FEDn Studio: - `Python 3.8, 3.9, 3.10 or 3.11 `__ +- `A FEDn Studio account `__ -If using self-managed with docker-compose: +If using pseudo-distributed mode with docker-compose: - `Docker `__ - `Docker Compose `__ -Quick start with FEDn Studio ----------------------------- +Creating the compute package and seed model +------------------------------------------- Install fedn: @@ -35,7 +38,7 @@ Clone this repository, then locate into this directory: git clone https://github.com/scaleoutsystems/fedn.git cd fedn/examples/mnist-pytorch -Create the compute package (compress the 'client' folder): +Create the compute package: .. code-block:: @@ -43,42 +46,57 @@ Create the compute package (compress the 'client' folder): This should create a file 'package.tgz' in the project folder. -Next, generate a seed model (the first model in the global model trail): +Next, generate a seed model (the first model in a global model trail): .. code-block:: fedn run build --path client -Upload the package and seed model to the FEDn Studio project on the "Session" page. +This will create a seed model called 'seed.npz' in the root of the project. This step will take a few minutes, depending on hardware and internet connection (builds a virtualenv). + +Using FEDn Studio +----------------- + +Follow the guide here to set up your FEDn Studio project and learn how to connect clients (using token authentication): `Studio guide `__. +On the step "Upload Files", upload 'package.tgz' and 'seed.npz' created above. + + +Modifing the data split: +======================== -In Studio, go to "Clients" and download a new client configuration file (contains the access token). -Then, start the client using the client.yaml file: +The default traning and test data for this example is downloaded and split direcly by the client when it starts up (see 'startup' entrypoint). +The number of splits and which split used by a client can be controlled via the environment variables ``FEDN_NUM_DATA_SPLITS`` and ``FEDN_DATA_PATH``. +For example, to split the data in 10 parts and start a client using the 8th partiton: .. code-block:: - export FEDN_AUTH_SCHEME=Bearer export FEDN_PACKAGE_EXTRACT_DIR=package + export FEDN_NUM_DATA_SPLITS=10 + export FEDN_DATA_PATH=package/data/clients/8/mnist.pt fedn run client -in client.yaml --secure=True --force-ssl -The default traning and test data is for this example downloaded and split direcly by the client when it starts up. -The data will be found in package/data/clients/1/mnist.pt and can be changed to other partitions by exporting the environment variable FEDN_DATA_PATH. -For example, to use the second partiton: +The default is to split the data into 2 partitions and use the first partition. -.. code-block:: - export FEDN_DATA_PATH=data/clients/2/mnist.pt +Connecting clients using Docker: +================================ -The default split into 2 partitions can be changed in client/data.py. +For convenience, there is a Docker image hosted on ghrc.io with fedn preinstalled. To start a client using Docker: -Quick start with docker-compose -------------------------------- +.. code-block:: -Clone this repository, locate into this directory: + docker run \ + -v $PWD/client.yaml:/app/client.yaml \ + -e FEDN_PACKAGE_EXTRACT_DIR=package \ + -e FEDN_NUM_DATA_SPLITS=2 \ + -e FEDN_DATA_PATH=/app/package/data/clients/1/mnist.pt \ + ghcr.io/scaleoutsystems/fedn/fedn:0.9.0 run client -in client.yaml --force-ssl --secure=True -.. code-block:: - git clone https://github.com/scaleoutsystems/fedn.git - cd fedn/examples/mnist-pytorch +Local development mode using Docker/docker compose +-------------------------------------------------- + +Follow the steps above to install FEDn, generate 'package.tgz' and 'seed.tgz'. Start a pseudo-distributed FEDn network using docker-compose: @@ -87,98 +105,41 @@ Start a pseudo-distributed FEDn network using docker-compose: docker compose \ -f ../../docker-compose.yaml \ -f docker-compose.override.yaml \ - up -d + up -This starts up the needed backend services MongoDB and Minio, the API Server and one Combiner. As well as two clients. +This starts up local services for MongoDB, Minio, the API Server, one Combiner and two clients. You can verify the deployment using these urls: - API Server: http://localhost:8092/get_controller_status - Minio: http://localhost:9000 - Mongo Express: http://localhost:8081 -To check the client output logs, run: - -.. code-block:: - - docker logs fedn-client-1 - -It should be waiting for the configuration of the package. - -Create the package (compress client folder): - -.. code-block:: - - tar -czvf package.tgz client - -You should now have a file 'package.tgz'. - -You are now ready to use the API to initialize the system with the package. - -Obs - After you have uploaded the package, you need to fetch the initial model (seed.npz) from client container: +Upload the package and seed model to FEDn controller using the APIClient. In Python: .. code-block:: - bin/get_data - - -Split the data in 10 partitions: - -.. code-block:: - - bin/split_data --n_splits=10 - -Data partitions will be generated in the folder 'data/clients'. + from fedn import APIClient + client = APIClient(host="localhost", port=8092) + client.set_active_package("package.tgz", helper="numpyhelper") + client.set_active_model("seed.npz") -FEDn relies on a configuration file for the client to connect to the server. Create a file called 'client.yaml' with the follwing content: +You can now start a training session with 5 rounds (default): .. code-block:: - network_id: fedn-network - discover_host: api-server - discover_port: 8092 - -Make sure to move the file ``client.yaml`` to the root of the examples/mnist-pytorch folder. -To connect a client that uses the data partition ``data/clients/1/mnist.pt`` and the config file ``client.yaml`` to the network, run the following docker command: - -.. code-block:: - - docker run \ - -v $PWD/client.yaml:/app/client.yaml \ - -v $PWD/data/clients/1:/var/data \ - -e ENTRYPOINT_OPTS=--data_path=/var/data/mnist.pt \ - --network=fedn_default \ - ghcr.io/scaleoutsystems/fedn/fedn:master-mnist-pytorch run client -in client.yaml --name client1 - -Observe the API Server logs and combiner logs, you should see the client connecting and entering into a state asking for a compute package. - -In a separate terminal, start a second client using the data partition 'data/clients/2/mnist.pt': - -.. code-block:: - - docker run \ - -v $PWD/client.yaml:/app/client.yaml \ - -v $PWD/data/clients/2:/var/data \ - -e ENTRYPOINT_OPTS=--data_path=/var/data/mnist.pt \ - --network=fedn_default \ - ghcr.io/scaleoutsystems/fedn/fedn:master-mnist-pytorch run client -in client.yaml --name client2 - -You are now ready to use the API to initialize the system with the compute package and seed model, and to start federated training. - -- Follow the example in the `Jupyter Notebook `__ - + client.start_session() Automate experimentation with several clients ------------------------------------------------ +============================================= -If you want to scale the number of clients, you can do so by running the following command: +If you want to scale the number of clients, you can do so by modifying ``docker-compose.override.yaml``. For example, +in order to run with 3 clients, change the environment variable ``FEDN_NUM_DATA_SPLITS`` to 3, and add one more client +by copying ``client1`` and setting ``FEDN_DATA_PATH`` to ``/app/package/data/clients/3/mnist.pt`` -.. code-block:: - - docker-compose -f ../../docker-compose.yaml -f docker-compose.override.yaml up --scale client=4 +Access message logs and validation data from MongoDB +==================================================== -Access logs and validation data from MongoDB ---------------------------------------------- You can access and download event logs and validation data via the API, and you can also as a developer obtain the MongoDB backend data using pymongo or via the MongoExpress interface: @@ -186,16 +147,21 @@ the MongoDB backend data using pymongo or via the MongoExpress interface: The credentials are as set in docker-compose.yaml in the root of the repository. -Access model updates ---------------------- +Access global models +==================== -You can obtain model updates from the 'fedn-models' bucket in Minio: +You can obtain global model updates from the 'fedn-models' bucket in Minio: - http://localhost:9000 +Reset the FEDn deployment +========================= + +To purge all data from a deployment incuding all session and round data, access the MongoExpress UI interface and +delete the entire ``fedn-network`` collection. Then restart all services. Clean up --------- +======== You can clean up by running .. code-block:: diff --git a/examples/mnist-pytorch/client/data.py b/examples/mnist-pytorch/client/data.py index 4e3858550..d67274548 100644 --- a/examples/mnist-pytorch/client/data.py +++ b/examples/mnist-pytorch/client/data.py @@ -59,7 +59,10 @@ def splitset(dataset, parts): return result -def split(out_dir='data', n_splits=2): +def split(out_dir='data'): + + n_splits = int(os.environ.get("FEDN_NUM_DATA_SPLITS", 2)) + # Make dir if not os.path.exists(f'{out_dir}/clients'): os.mkdir(f'{out_dir}/clients') diff --git a/examples/mnist-pytorch/client/python_env.yaml b/examples/mnist-pytorch/client/python_env.yaml index 241466d69..442a7de1d 100644 --- a/examples/mnist-pytorch/client/python_env.yaml +++ b/examples/mnist-pytorch/client/python_env.yaml @@ -6,4 +6,4 @@ build_dependencies: dependencies: - torch==2.2.1 - torchvision==0.17.1 - - fedn==0.9.0b2 + - fedn==0.9.0b2 \ No newline at end of file diff --git a/examples/mnist-pytorch/docker-compose.override.yaml b/examples/mnist-pytorch/docker-compose.override.yaml index 71ee384d5..eb7aedc47 100644 --- a/examples/mnist-pytorch/docker-compose.override.yaml +++ b/examples/mnist-pytorch/docker-compose.override.yaml @@ -1,10 +1,36 @@ # Compose schema version -version: '3.3' +version: '3.4' # Overriding requirements + +x-env: + &defaults + GET_HOSTS_FROM: dns + FEDN_PACKAGE_EXTRACT_DIR: package + FEDN_NUM_DATA_SPLITS: 2 + services: - client: + + client1: + extends: + file: ${HOST_REPO_DIR:-.}/docker-compose.yaml + service: client + environment: + <<: *defaults + FEDN_DATA_PATH: /app/package/data/clients/1/mnist.pt + deploy: + replicas: 1 + volumes: + - ${HOST_REPO_DIR:-.}/fedn:/app/fedn + + client2: + extends: + file: ${HOST_REPO_DIR:-.}/docker-compose.yaml + service: client + environment: + <<: *defaults + FEDN_DATA_PATH: /app/package/data/clients/2/mnist.pt deploy: - replicas: 2 + replicas: 1 volumes: - ${HOST_REPO_DIR:-.}/fedn:/app/fedn