diff --git a/.github/workflows/build-containers.yaml b/.github/workflows/build-containers.yaml index 8dd2f561a..89648764e 100644 --- a/.github/workflows/build-containers.yaml +++ b/.github/workflows/build-containers.yaml @@ -1,6 +1,7 @@ name: "build containers" on: + workflow_dispatch: push: branches: - master diff --git a/.github/workflows/code-checks.yaml b/.github/workflows/code-checks.yaml index dbe378c63..8c48a3015 100644 --- a/.github/workflows/code-checks.yaml +++ b/.github/workflows/code-checks.yaml @@ -25,6 +25,7 @@ jobs: --exclude-dir='docs' --exclude-dir='flower-client' --exclude='tests.py' + --exclude='controller_cmd.py' --exclude='README.rst' '^[ \t]+(import|from) ' -I . diff --git a/.github/workflows/push-to-pypi.yaml b/.github/workflows/push-to-pypi.yaml index 1b59835ad..1e184c17f 100644 --- a/.github/workflows/push-to-pypi.yaml +++ b/.github/workflows/push-to-pypi.yaml @@ -1,8 +1,9 @@ name: Publish Python distribution to PyPI on: + workflow_dispatch: release: - types: [created] + types: published jobs: build-and-publish: diff --git a/docker-compose.yaml b/docker-compose.yaml index 85386c6da..c3620e79d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -78,7 +78,7 @@ services: - mongo entrypoint: [ "sh", "-c" ] command: - - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/python fedn/network/api/server.py" + - "/venv/bin/pip install --no-cache-dir -e . && /venv/bin/fedn controller start" ports: - 8092:8092 diff --git a/docs/conf.py b/docs/conf.py index bebc3a80e..c45e90846 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,7 +12,7 @@ author = "Scaleout Systems AB" # The full version, including alpha/beta/rc tags -release = "0.11.0" +release = "0.11.1" # Add any Sphinx extension module names here, as strings extensions = [ diff --git a/fedn/cli/__init__.py b/fedn/cli/__init__.py index 137fc9b9c..bcd27dc53 100644 --- a/fedn/cli/__init__.py +++ b/fedn/cli/__init__.py @@ -9,3 +9,4 @@ from .session_cmd import session_cmd # noqa: F401 from .status_cmd import status_cmd # noqa: F401 from .validation_cmd import validation_cmd # noqa: F401 +from .controller_cmd import controller_cmd # noqa: F401 diff --git a/fedn/cli/controller_cmd.py b/fedn/cli/controller_cmd.py new file mode 100644 index 000000000..ab8727b27 --- /dev/null +++ b/fedn/cli/controller_cmd.py @@ -0,0 +1,18 @@ +import click + +from .main import main + + +@main.group("controller") +@click.pass_context +def controller_cmd(ctx): + """:param ctx:""" + pass + + +@controller_cmd.command("start") +@click.pass_context +def controller_cmd(ctx): + from fedn.network.api.server import start_server_api + + start_server_api() diff --git a/fedn/cli/main.py b/fedn/cli/main.py index 0d5660c0b..ab1dd448e 100644 --- a/fedn/cli/main.py +++ b/fedn/cli/main.py @@ -1,5 +1,4 @@ -import importlib.metadata - +from fedn.utils.dist import get_version import click CONTEXT_SETTINGS = dict( @@ -7,11 +6,7 @@ help_option_names=["-h", "--help"], ) -# Dynamically get the version of the package -try: - version = importlib.metadata.version("fedn") -except importlib.metadata.PackageNotFoundError: - version = "unknown" +version=get_version("fedn") @click.group(context_settings=CONTEXT_SETTINGS) diff --git a/fedn/common/config.py b/fedn/common/config.py index 94b346d65..23d873ff7 100644 --- a/fedn/common/config.py +++ b/fedn/common/config.py @@ -2,6 +2,8 @@ import yaml +from fedn.utils.dist import get_package_path + SECRET_KEY = os.environ.get("FEDN_JWT_SECRET_KEY", False) FEDN_JWT_CUSTOM_CLAIM_KEY = os.environ.get("FEDN_JWT_CUSTOM_CLAIM_KEY", False) FEDN_JWT_CUSTOM_CLAIM_VALUE = os.environ.get("FEDN_JWT_CUSTOM_CLAIM_VALUE", False) @@ -20,9 +22,15 @@ def get_environment_config(): """Get the configuration from environment variables.""" global STATESTORE_CONFIG global MODELSTORAGE_CONFIG - - STATESTORE_CONFIG = os.environ.get("STATESTORE_CONFIG", "/workspaces/fedn/config/settings-reducer.yaml.template") - MODELSTORAGE_CONFIG = os.environ.get("MODELSTORAGE_CONFIG", "/workspaces/fedn/config/settings-reducer.yaml.template") + if not os.environ.get("STATESTORE_CONFIG", False): + STATESTORE_CONFIG = get_package_path() + "/common/settings-controller.yaml.template" + else: + STATESTORE_CONFIG = os.environ.get("STATESTORE_CONFIG") + + if not os.environ.get("MODELSTORAGE_CONFIG", False): + MODELSTORAGE_CONFIG = get_package_path() + "/common/settings-controller.yaml.template" + else: + MODELSTORAGE_CONFIG = os.environ.get("MODELSTORAGE_CONFIG") def get_statestore_config(file=None): diff --git a/fedn/common/settings-controller.yaml.template b/fedn/common/settings-controller.yaml.template new file mode 100644 index 000000000..a5266a38b --- /dev/null +++ b/fedn/common/settings-controller.yaml.template @@ -0,0 +1,24 @@ +network_id: fedn-network +controller: + host: localhost + port: 8092 + debug: True + +statestore: + type: MongoDB + mongo_config: + username: fedn_admin + password: password + host: localhost + port: 6534 + +storage: + storage_type: S3 + storage_config: + storage_hostname: localhost + storage_port: 9000 + storage_access_key: fedn_admin + storage_secret_key: password + storage_bucket: fedn-models + context_bucket: fedn-context + storage_secure_mode: False diff --git a/fedn/network/api/server.py b/fedn/network/api/server.py index c9e54ff87..d56c3ab0b 100644 --- a/fedn/network/api/server.py +++ b/fedn/network/api/server.py @@ -9,6 +9,7 @@ from fedn.network.api.v1 import _routes custom_url_prefix = os.environ.get("FEDN_CUSTOM_URL_PREFIX", False) +# statestore_config,modelstorage_config,network_id,control=set_statestore_config() api = API(statestore, control) app = Flask(__name__) for bp in _routes: @@ -625,8 +626,14 @@ def list_combiners_data(): if custom_url_prefix: app.add_url_rule(f"{custom_url_prefix}/list_combiners_data", view_func=list_combiners_data, methods=["POST"]) -if __name__ == "__main__": + +def start_server_api(): config = get_controller_config() port = config["port"] debug = config["debug"] - app.run(debug=debug, port=port, host="0.0.0.0") + host = "0.0.0.0" + app.run(debug=debug, port=port, host=host) + + +if __name__ == "__main__": + start_server_api() diff --git a/fedn/network/api/shared.py b/fedn/network/api/shared.py index fc8d4ae57..9e0e5acbd 100644 --- a/fedn/network/api/shared.py +++ b/fedn/network/api/shared.py @@ -5,7 +5,6 @@ statestore_config = get_statestore_config() modelstorage_config = get_modelstorage_config() network_id = get_network_config() - statestore = MongoStateStore(network_id, statestore_config["mongo_config"]) statestore.set_storage_backend(modelstorage_config) control = Control(statestore=statestore) diff --git a/fedn/network/api/v1/shared.py b/fedn/network/api/v1/shared.py index b7ae170af..a27a6f637 100644 --- a/fedn/network/api/v1/shared.py +++ b/fedn/network/api/v1/shared.py @@ -3,10 +3,9 @@ import pymongo from pymongo.database import Database -from fedn.network.api.shared import statestore_config, network_id +from fedn.network.api.shared import statestore_config,network_id api_version = "v1" - mc = pymongo.MongoClient(**statestore_config["mongo_config"]) mc.server_info() mdb: Database = mc[network_id] diff --git a/fedn/network/combiner/combiner.py b/fedn/network/combiner/combiner.py index 7f7b93548..2c59991f9 100644 --- a/fedn/network/combiner/combiner.py +++ b/fedn/network/combiner/combiner.py @@ -127,7 +127,10 @@ def __init__(self, config): # Set the status to offline for previous clients. previous_clients = self.statestore.clients.find({"combiner": config["name"]}) for client in previous_clients: - self.statestore.set_client({"name": client["name"], "status": "offline", "client_id": client["client_id"]}) + try: + self.statestore.set_client({"name": client["name"], "status": "offline", "client_id": client["client_id"]}) + except KeyError: + self.statestore.set_client({"name": client["name"], "status": "offline"}) self.modelservice = ModelService() diff --git a/fedn/network/storage/statestore/mongostatestore.py b/fedn/network/storage/statestore/mongostatestore.py index 7262e5554..7ef22a795 100644 --- a/fedn/network/storage/statestore/mongostatestore.py +++ b/fedn/network/storage/statestore/mongostatestore.py @@ -738,7 +738,13 @@ def set_client(self, client_data): :return: """ client_data["updated_at"] = str(datetime.now()) - self.clients.update_one({"client_id": client_data["client_id"]}, {"$set": client_data}, True) + try: + self.clients.update_one({"client_id": client_data["client_id"]}, {"$set": client_data}, True) + except KeyError: + # If client_id is not present, use name as identifier, for backwards compatibility + id = str(uuid.uuid4()) + client_data["client_id"] = id + self.clients.update_one({"name": client_data["name"]}, {"$set": client_data}, True) def get_client(self, client_id): """Get client by client_id. diff --git a/fedn/utils/dist.py b/fedn/utils/dist.py new file mode 100644 index 000000000..e5fa7192b --- /dev/null +++ b/fedn/utils/dist.py @@ -0,0 +1,17 @@ +import importlib.metadata + +import fedn + + +def get_version(pacakge): + # Dynamically get the version of the package + try: + version = importlib.metadata.version("fedn") + except importlib.metadata.PackageNotFoundError: + version = "unknown" + return version + + +def get_package_path(): + # Get the path of the package + return fedn.__path__[0] diff --git a/pyproject.toml b/pyproject.toml index 5773a38a5..3806fee56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta" [project] name = "fedn" -version = "0.11.0" +version = "0.11.1" description = "Scaleout Federated Learning" authors = [{ name = "Scaleout Systems AB", email = "contact@scaleoutsystems.com" }] readme = "README.rst"