From 972ebfce24bdf7efd9891d31165770a1c08c4b9b Mon Sep 17 00:00:00 2001 From: Niklas Date: Tue, 17 Oct 2023 10:52:10 +0200 Subject: [PATCH 01/23] Pagination added for sessions and events --- fedn/fedn/network/api/interface.py | 513 +++++++++++------- fedn/fedn/network/api/server.py | 165 +++--- .../network/statestore/mongostatestore.py | 265 +++++---- 3 files changed, 570 insertions(+), 373 deletions(-) diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index f05222e87..f886261d0 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -10,35 +10,35 @@ from werkzeug.utils import secure_filename from fedn.common.config import get_controller_config, get_network_config -from fedn.network.combiner.interfaces import (CombinerInterface, - CombinerUnavailableError) +from fedn.network.combiner.interfaces import ( + CombinerInterface, + CombinerUnavailableError, +) from fedn.network.state import ReducerState, ReducerStateToString from fedn.utils.checksum import sha -__all__ = 'API', +__all__ = ("API",) class API: - """ The API class is a wrapper for the statestore. It is used to expose the statestore to the network API. """ + """The API class is a wrapper for the statestore. It is used to expose the statestore to the network API.""" def __init__(self, statestore, control): self.statestore = statestore self.control = control - self.name = 'api' + self.name = "api" def _to_dict(self): - """ Convert the object to a dict. + """Convert the object to a dict. ::return: The object as a dict. ::rtype: dict """ - data = { - 'name': self.name - } + data = {"name": self.name} return data def _get_combiner_report(self, combiner_id): - """ Get report response from combiner. + """Get report response from combiner. :param combiner_id: The combiner id to get report response from. :type combiner_id: str @@ -50,8 +50,10 @@ def _get_combiner_report(self, combiner_id): report = combiner.report return report - def _allowed_file_extension(self, filename, ALLOWED_EXTENSIONS={'gz', 'bz2', 'tar', 'zip', 'tgz'}): - """ Check if file extension is allowed. + def _allowed_file_extension( + self, filename, ALLOWED_EXTENSIONS={"gz", "bz2", "tar", "zip", "tgz"} + ): + """Check if file extension is allowed. :param filename: The filename to check. :type filename: str @@ -59,11 +61,13 @@ def _allowed_file_extension(self, filename, ALLOWED_EXTENSIONS={'gz', 'bz2', 'ta :rtype: bool """ - return '.' in filename and \ - filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS + return ( + "." in filename + and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS + ) def get_all_clients(self): - """ Get all clients from the statestore. + """Get all clients from the statestore. :return: All clients as a json response. :rtype: :class:`flask.Response` @@ -72,19 +76,20 @@ def get_all_clients(self): clients_objects = self.statestore.list_clients() payload = {} for object in clients_objects: - id = object['name'] - info = {"combiner": object['combiner'], - "combiner_preferred": object['combiner_preferred'], - "ip": object['ip'], - "updated_at": object['updated_at'], - "status": object['status'], - } + id = object["name"] + info = { + "combiner": object["combiner"], + "combiner_preferred": object["combiner_preferred"], + "ip": object["ip"], + "updated_at": object["updated_at"], + "status": object["status"], + } payload[id] = info return jsonify(payload) def get_active_clients(self, combiner_id): - """ Get all active clients, i.e that are assigned to a combiner. + """Get all active clients, i.e that are assigned to a combiner. A report request to the combiner is neccessary to determine if a client is active or not. :param combiner_id: The combiner id to get active clients for. @@ -95,12 +100,20 @@ def get_active_clients(self, combiner_id): # Get combiner interface object combiner = self.control.network.get_combiner(combiner_id) if combiner is None: - return jsonify({"success": False, "message": f"Combiner {combiner_id} not found."}), 404 + return ( + jsonify( + { + "success": False, + "message": f"Combiner {combiner_id} not found.", + } + ), + 404, + ) response = combiner.list_active_clients() return response def get_all_combiners(self): - """ Get all combiners from the statestore. + """Get all combiners from the statestore. :return: All combiners as a json response. :rtype: :class:`flask.Response` @@ -109,20 +122,21 @@ def get_all_combiners(self): combiner_objects = self.statestore.get_combiners() payload = {} for object in combiner_objects: - id = object['name'] - info = {"address": object['address'], - "fqdn": object['fqdn'], - "parent_reducer": object['parent']["name"], - "port": object['port'], - "report": object['report'], - "updated_at": object['updated_at'], - } + id = object["name"] + info = { + "address": object["address"], + "fqdn": object["fqdn"], + "parent_reducer": object["parent"]["name"], + "port": object["port"], + "report": object["report"], + "updated_at": object["updated_at"], + } payload[id] = info return jsonify(payload) def get_combiner(self, combiner_id): - """ Get a combiner from the statestore. + """Get a combiner from the statestore. :param combiner_id: The combiner id to get. :type combiner_id: str @@ -132,36 +146,40 @@ def get_combiner(self, combiner_id): # Will return ObjectId object = self.statestore.get_combiner(combiner_id) payload = {} - id = object['name'] - info = {"address": object['address'], - "fqdn": object['fqdn'], - "parent_reducer": object['parent']["name"], - "port": object['port'], - "report": object['report'], - "updated_at": object['updated_at'], - } + id = object["name"] + info = { + "address": object["address"], + "fqdn": object["fqdn"], + "parent_reducer": object["parent"]["name"], + "port": object["port"], + "report": object["report"], + "updated_at": object["updated_at"], + } payload[id] = info return jsonify(payload) - def get_all_sessions(self): - """ Get all sessions from the statestore. + def get_all_sessions(self, limit=None, skip=None): + """Get all sessions from the statestore. :return: All sessions as a json response. :rtype: :class:`flask.Response` """ - sessions_objects = self.statestore.get_sessions() + sessions_objects = self.statestore.get_sessions(limit, skip) if sessions_objects is None: - return jsonify({"success": False, "message": "No sessions found."}), 404 + return ( + jsonify({"success": False, "message": "No sessions found."}), + 404, + ) payload = {} for object in sessions_objects: - id = object['session_id'] - info = object['session_config'][0] + id = object["session_id"] + info = object["session_config"][0] payload[id] = info return jsonify(payload) def get_session(self, session_id): - """ Get a session from the statestore. + """Get a session from the statestore. :param session_id: The session id to get. :type session_id: str @@ -170,15 +188,23 @@ def get_session(self, session_id): """ session_object = self.statestore.get_session(session_id) if session_object is None: - return jsonify({"success": False, "message": f"Session {session_id} not found."}), 404 + return ( + jsonify( + { + "success": False, + "message": f"Session {session_id} not found.", + } + ), + 404, + ) payload = {} - id = session_object['session_id'] - info = session_object['session_config'][0] + id = session_object["session_id"] + info = session_object["session_config"][0] payload[id] = info return jsonify(payload) def set_compute_package(self, file, helper_type): - """ Set the compute package in the statestore. + """Set the compute package in the statestore. :param file: The compute package to set. :type file: file @@ -189,24 +215,42 @@ def set_compute_package(self, file, helper_type): if file and self._allowed_file_extension(file.filename): filename = secure_filename(file.filename) # TODO: make configurable, perhaps in config.py or package.py - file_path = os.path.join( - '/app/client/package/', filename) + file_path = os.path.join("/app/client/package/", filename) file.save(file_path) - if self.control.state() == ReducerState.instructing or self.control.state() == ReducerState.monitoring: - return jsonify({"success": False, "message": "Reducer is in instructing or monitoring state." - "Cannot set compute package."}), 400 + if ( + self.control.state() == ReducerState.instructing + or self.control.state() == ReducerState.monitoring + ): + return ( + jsonify( + { + "success": False, + "message": "Reducer is in instructing or monitoring state." + "Cannot set compute package.", + } + ), + 400, + ) self.control.set_compute_package(filename, file_path) self.statestore.set_helper(helper_type) success = self.statestore.set_compute_package(filename) if not success: - return jsonify({"success": False, "message": "Failed to set compute package."}), 400 + return ( + jsonify( + { + "success": False, + "message": "Failed to set compute package.", + } + ), + 400, + ) return jsonify({"success": True, "message": "Compute package set."}) def _get_compute_package_name(self): - """ Get the compute package name from the statestore. + """Get the compute package name from the statestore. :return: The compute package name. :rtype: str @@ -217,32 +261,38 @@ def _get_compute_package_name(self): return None, message else: try: - name = package_objects['filename'] + name = package_objects["filename"] except KeyError as e: message = "No compute package found. Key error." print(e) return None, message - return name, 'success' + return name, "success" def get_compute_package(self): - """ Get the compute package from the statestore. + """Get the compute package from the statestore. :return: The compute package as a json response. :rtype: :class:`flask.Response` """ package_object = self.statestore.get_compute_package() if package_object is None: - return jsonify({"success": False, "message": "No compute package found."}), 404 + return ( + jsonify( + {"success": False, "message": "No compute package found."} + ), + 404, + ) payload = {} - id = str(package_object['_id']) - info = {"filename": package_object['filename'], - "helper": package_object['helper'], - } + id = str(package_object["_id"]) + info = { + "filename": package_object["filename"], + "helper": package_object["helper"], + } payload[id] = info return jsonify(payload) def download_compute_package(self, name): - """ Download the compute package. + """Download the compute package. :return: The compute package as a json object. :rtype: :class:`flask.Response` @@ -255,23 +305,27 @@ def download_compute_package(self, name): mutex = threading.Lock() mutex.acquire() # TODO: make configurable, perhaps in config.py or package.py - return send_from_directory('/app/client/package/', name, as_attachment=True) + return send_from_directory( + "/app/client/package/", name, as_attachment=True + ) except Exception: try: data = self.control.get_compute_package(name) # TODO: make configurable, perhaps in config.py or package.py - file_path = os.path.join('/app/client/package/', name) - with open(file_path, 'wb') as fh: + file_path = os.path.join("/app/client/package/", name) + with open(file_path, "wb") as fh: fh.write(data) # TODO: make configurable, perhaps in config.py or package.py - return send_from_directory('/app/client/package/', name, as_attachment=True) + return send_from_directory( + "/app/client/package/", name, as_attachment=True + ) except Exception: raise finally: mutex.release() def _create_checksum(self, name=None): - """ Create the checksum of the compute package. + """Create the checksum of the compute package. :param name: The name of the compute package. :type name: str @@ -282,17 +336,19 @@ def _create_checksum(self, name=None): if name is None: name, message = self._get_compute_package_name() if name is None: - return False, message, '' - file_path = os.path.join('/app/client/package/', name) # TODO: make configurable, perhaps in config.py or package.py + return False, message, "" + file_path = os.path.join( + "/app/client/package/", name + ) # TODO: make configurable, perhaps in config.py or package.py try: sum = str(sha(file_path)) except FileNotFoundError: - sum = '' - message = 'File not found.' + sum = "" + message = "File not found." return True, message, sum def get_checksum(self, name): - """ Get the checksum of the compute package. + """Get the checksum of the compute package. :param name: The name of the compute package. :type name: str @@ -303,66 +359,74 @@ def get_checksum(self, name): success, message, sum = self._create_checksum(name) if not success: return jsonify({"success": False, "message": message}), 404 - payload = {'checksum': sum} + payload = {"checksum": sum} return jsonify(payload) def get_controller_status(self): - """ Get the status of the controller. + """Get the status of the controller. :return: The status of the controller as a json object. :rtype: :py:class:`flask.Response` """ - return jsonify({'state': ReducerStateToString(self.control.state())}) + return jsonify({"state": ReducerStateToString(self.control.state())}) def get_events(self, **kwargs): - """ Get the events of the federated network. + """Get the events of the federated network. :return: The events as a json object. :rtype: :py:class:`flask.Response` """ event_objects = self.statestore.get_events(**kwargs) if event_objects is None: - return jsonify({"success": False, "message": "No events found."}), 404 + return ( + jsonify({"success": False, "message": "No events found."}), + 404, + ) json_docs = [] for doc in self.statestore.get_events(**kwargs): json_doc = json.dumps(doc, default=json_util.default) json_docs.append(json_doc) json_docs.reverse() - return jsonify({'events': json_docs}) + return jsonify({"events": json_docs}) def get_all_validations(self, **kwargs): - """ Get all validations from the statestore. + """Get all validations from the statestore. :return: All validations as a json response. :rtype: :class:`flask.Response` """ validations_objects = self.statestore.get_validations(**kwargs) if validations_objects is None: - return jsonify( - { - "success": False, - "message": "No validations found.", - "filter_used": kwargs - } - ), 404 + return ( + jsonify( + { + "success": False, + "message": "No validations found.", + "filter_used": kwargs, + } + ), + 404, + ) payload = {} for object in validations_objects: - id = str(object['_id']) + id = str(object["_id"]) info = { - 'model_id': object['modelId'], - 'data': object['data'], - 'timestamp': object['timestamp'], - 'meta': object['meta'], - 'sender': object['sender'], - 'receiver': object['receiver'], + "model_id": object["modelId"], + "data": object["data"], + "timestamp": object["timestamp"], + "meta": object["meta"], + "sender": object["sender"], + "receiver": object["receiver"], } payload[id] = info return jsonify(payload) - def add_combiner(self, combiner_id, secure_grpc, address, remote_addr, fqdn, port): - """ Add a combiner to the network. + def add_combiner( + self, combiner_id, secure_grpc, address, remote_addr, fqdn, port + ): + """Add a combiner to the network. :param combiner_id: The combiner id to add. :type combiner_id: str @@ -383,18 +447,20 @@ def add_combiner(self, combiner_id, secure_grpc, address, remote_addr, fqdn, por """ # TODO: Any more required check for config? Formerly based on status: "retry" if not self.control.idle(): - return jsonify({ - 'success': False, - 'status': 'retry', - 'message': 'Conroller is not in idle state, try again later. ' - } + return jsonify( + { + "success": False, + "status": "retry", + "message": "Conroller is not in idle state, try again later. ", + } ) # Check if combiner already exists combiner = self.control.network.get_combiner(combiner_id) if not combiner: - if secure_grpc == 'True': + if secure_grpc == "True": certificate, key = self.certificate_manager.get_or_create( - address).get_keypair_raw() + address + ).get_keypair_raw() _ = base64.b64encode(certificate) _ = base64.b64encode(key) @@ -410,29 +476,32 @@ def add_combiner(self, combiner_id, secure_grpc, address, remote_addr, fqdn, por port=port, certificate=copy.deepcopy(certificate), key=copy.deepcopy(key), - ip=remote_addr) + ip=remote_addr, + ) self.control.network.add_combiner(combiner_interface) # Check combiner now exists combiner = self.control.network.get_combiner(combiner_id) if not combiner: - return jsonify({'success': False, 'message': 'Combiner not added.'}) + return jsonify( + {"success": False, "message": "Combiner not added."} + ) payload = { - 'success': True, - 'message': 'Combiner added successfully.', - 'status': 'added', - 'storage': self.statestore.get_storage_backend(), - 'statestore': self.statestore.get_config(), - 'certificate': combiner.get_certificate(), - 'key': combiner.get_key() + "success": True, + "message": "Combiner added successfully.", + "status": "added", + "storage": self.statestore.get_storage_backend(), + "statestore": self.statestore.get_config(), + "certificate": combiner.get_certificate(), + "key": combiner.get_key(), } return jsonify(payload) def add_client(self, client_id, preferred_combiner, remote_addr): - """ Add a client to the network. + """Add a client to the network. :param client_id: The client id to add. :type client_id: str @@ -444,26 +513,46 @@ def add_client(self, client_id, preferred_combiner, remote_addr): # Check if package has been set package_object = self.statestore.get_compute_package() if package_object is None: - return jsonify({'success': False, 'status': 'retry', 'message': 'No compute package found. Set package in controller.'}), 203 + return ( + jsonify( + { + "success": False, + "status": "retry", + "message": "No compute package found. Set package in controller.", + } + ), + 203, + ) # Assign client to combiner if preferred_combiner: combiner = self.control.network.get_combiner(preferred_combiner) if combiner is None: - return jsonify({'success': False, - 'message': f'Combiner {preferred_combiner} not found or unavailable.'}), 400 + return ( + jsonify( + { + "success": False, + "message": f"Combiner {preferred_combiner} not found or unavailable.", + } + ), + 400, + ) else: combiner = self.control.network.find_available_combiner() if combiner is None: - return jsonify({'success': False, - 'message': 'No combiner available.'}), 400 + return ( + jsonify( + {"success": False, "message": "No combiner available."} + ), + 400, + ) client_config = { - 'name': client_id, - 'combiner_preferred': preferred_combiner, - 'combiner': combiner.name, - 'ip': remote_addr, - 'status': 'available' + "name": client_id, + "combiner_preferred": preferred_combiner, + "combiner": combiner.name, + "ip": remote_addr, + "status": "available", } # Add client to network self.control.network.add_client(client_config) @@ -471,38 +560,36 @@ def add_client(self, client_id, preferred_combiner, remote_addr): # Setup response containing information about the combiner for assinging the client if combiner.certificate: cert_b64 = base64.b64encode(combiner.certificate) - cert = str(cert_b64).split('\'')[1] + cert = str(cert_b64).split("'")[1] else: cert = None payload = { - 'status': 'assigned', - 'host': combiner.address, - 'fqdn': combiner.fqdn, - 'package': 'remote', # TODO: Make this configurable - 'ip': combiner.ip, - 'port': combiner.port, - 'certificate': cert, - 'helper_type': self.control.statestore.get_helper() + "status": "assigned", + "host": combiner.address, + "fqdn": combiner.fqdn, + "package": "remote", # TODO: Make this configurable + "ip": combiner.ip, + "port": combiner.port, + "certificate": cert, + "helper_type": self.control.statestore.get_helper(), } print("Seding payload: ", payload, flush=True) return jsonify(payload) def get_initial_model(self): - """ Get the initial model from the statestore. + """Get the initial model from the statestore. :return: The initial model as a json response. :rtype: :class:`flask.Response` """ model_id = self.statestore.get_initial_model() - payload = { - 'model_id': model_id - } + payload = {"model_id": model_id} return jsonify(payload) def set_initial_model(self, file): - """ Add an initial model to the network. + """Add an initial model to the network. :param file: The initial model to add. :type file: file @@ -520,27 +607,29 @@ def set_initial_model(self, file): self.control.commit(file.filename, model) except Exception as e: print(e, flush=True) - return jsonify({'success': False, 'message': e}) + return jsonify({"success": False, "message": e}) - return jsonify({'success': True, 'message': 'Initial model added successfully.'}) + return jsonify( + {"success": True, "message": "Initial model added successfully."} + ) def get_latest_model(self): - """ Get the latest model from the statestore. + """Get the latest model from the statestore. :return: The initial model as a json response. :rtype: :class:`flask.Response` """ if self.statestore.get_latest_model(): model_id = self.statestore.get_latest_model() - payload = { - 'model_id': model_id - } + payload = {"model_id": model_id} return jsonify(payload) else: - return jsonify({'success': False, 'message': 'No initial model set.'}) + return jsonify( + {"success": False, "message": "No initial model set."} + ) def get_model_trail(self): - """ Get the model trail for a given session. + """Get the model trail for a given session. :param session: The session id to get the model trail for. :type session: str @@ -551,38 +640,41 @@ def get_model_trail(self): if model_info: return jsonify(model_info) else: - return jsonify({'success': False, 'message': 'No model trail available.'}) + return jsonify( + {"success": False, "message": "No model trail available."} + ) def get_all_rounds(self): - """ Get all rounds. + """Get all rounds. :return: The rounds as json response. :rtype: :class:`flask.Response` """ rounds_objects = self.statestore.get_rounds() if rounds_objects is None: - jsonify({'success': False, 'message': 'No rounds available.'}) + jsonify({"success": False, "message": "No rounds available."}) payload = {} for object in rounds_objects: - id = object['round_id'] - if 'reducer' in object.keys(): - reducer = object['reducer'] + id = object["round_id"] + if "reducer" in object.keys(): + reducer = object["reducer"] else: reducer = None - if 'combiners' in object.keys(): - combiners = object['combiners'] + if "combiners" in object.keys(): + combiners = object["combiners"] else: combiners = None - info = {'reducer': reducer, - 'combiners': combiners, - } + info = { + "reducer": reducer, + "combiners": combiners, + } payload[id] = info else: return jsonify(payload) def get_round(self, round_id): - """ Get a round. + """Get a round. :param round_id: The round id to get. :type round_id: str @@ -591,38 +683,48 @@ def get_round(self, round_id): """ round_object = self.statestore.get_round(round_id) if round_object is None: - return jsonify({'success': False, 'message': 'Round not found.'}) + return jsonify({"success": False, "message": "Round not found."}) payload = { - 'round_id': round_object['round_id'], - 'reducer': round_object['reducer'], - 'combiners': round_object['combiners'], + "round_id": round_object["round_id"], + "reducer": round_object["reducer"], + "combiners": round_object["combiners"], } return jsonify(payload) def get_client_config(self, checksum=True): - """ Get the client config. + """Get the client config. :return: The client config as json response. :rtype: :py:class:`flask.Response` """ config = get_controller_config() network_id = get_network_config() - port = config['port'] - host = config['host'] + port = config["port"] + host = config["host"] payload = { - 'network_id': network_id, - 'discover_host': host, - 'discover_port': port, + "network_id": network_id, + "discover_host": host, + "discover_port": port, } if checksum: success, _, checksum_str = self._create_checksum() if success: - payload['checksum'] = checksum_str + payload["checksum"] = checksum_str return jsonify(payload) - def start_session(self, session_id, rounds=5, round_timeout=180, round_buffer_size=-1, delete_models=False, - validate=True, helper='keras', min_clients=1, requested_clients=8): - """ Start a session. + def start_session( + self, + session_id, + rounds=5, + round_timeout=180, + round_buffer_size=-1, + delete_models=False, + validate=True, + helper="keras", + min_clients=1, + requested_clients=8, + ): + """Start a session. :param session_id: The session id to start. :type session_id: str @@ -646,18 +748,22 @@ def start_session(self, session_id, rounds=5, round_timeout=180, round_buffer_si # Check if session already exists session = self.statestore.get_session(session_id) if session: - return jsonify({'success': False, 'message': 'Session already exists.'}) + return jsonify( + {"success": False, "message": "Session already exists."} + ) # Check if session is running if self.control.state() == ReducerState.monitoring: - return jsonify({'success': False, 'message': 'A session is already running.'}) + return jsonify( + {"success": False, "message": "A session is already running."} + ) # Check available clients per combiner clients_available = 0 for combiner in self.control.network.get_combiners(): try: combiner_state = combiner.report() - nr_active_clients = combiner_state['nr_active_clients'] + nr_active_clients = combiner_state["nr_active_clients"] clients_available = clients_available + int(nr_active_clients) except CombinerUnavailableError as e: # TODO: Handle unavailable combiner, stop session or continue? @@ -665,11 +771,16 @@ def start_session(self, session_id, rounds=5, round_timeout=180, round_buffer_si continue if clients_available < min_clients: - return jsonify({'success': False, 'message': 'Not enough clients available to start session.'}) + return jsonify( + { + "success": False, + "message": "Not enough clients available to start session.", + } + ) # Check if validate is string and convert to bool if isinstance(validate, str): - if validate.lower() == 'true': + if validate.lower() == "true": validate = True else: validate = False @@ -678,22 +789,30 @@ def start_session(self, session_id, rounds=5, round_timeout=180, round_buffer_si model_id = self.statestore.get_latest_model() # Setup session config - session_config = {'session_id': session_id, - 'round_timeout': round_timeout, - 'buffer_size': round_buffer_size, - 'model_id': model_id, - 'rounds': rounds, - 'delete_models_storage': delete_models, - 'clients_required': min_clients, - 'clients_requested': requested_clients, - 'task': (''), - 'validate': validate, - 'helper_type': helper - } + session_config = { + "session_id": session_id, + "round_timeout": round_timeout, + "buffer_size": round_buffer_size, + "model_id": model_id, + "rounds": rounds, + "delete_models_storage": delete_models, + "clients_required": min_clients, + "clients_requested": requested_clients, + "task": (""), + "validate": validate, + "helper_type": helper, + } # Start session - threading.Thread(target=self.control.session, - args=(session_config,)).start() + threading.Thread( + target=self.control.session, args=(session_config,) + ).start() # Return success response - return jsonify({'success': True, 'message': 'Session started successfully.', "config": session_config}) + return jsonify( + { + "success": True, + "message": "Session started successfully.", + "config": session_config, + } + ) diff --git a/fedn/fedn/network/api/server.py b/fedn/fedn/network/api/server.py index 4e0e93775..2bdd7aae3 100644 --- a/fedn/fedn/network/api/server.py +++ b/fedn/fedn/network/api/server.py @@ -1,7 +1,11 @@ from flask import Flask, jsonify, request -from fedn.common.config import (get_controller_config, get_modelstorage_config, - get_network_config, get_statestore_config) +from fedn.common.config import ( + get_controller_config, + get_modelstorage_config, + get_network_config, + get_statestore_config, +) from fedn.network.api.interface import API from fedn.network.controller.control import Control from fedn.network.statestore.mongostatestore import MongoStateStore @@ -10,18 +14,16 @@ network_id = get_network_config() modelstorage_config = get_modelstorage_config() statestore = MongoStateStore( - network_id, - statestore_config['mongo_config'], - modelstorage_config + network_id, statestore_config["mongo_config"], modelstorage_config ) control = Control(statestore=statestore) api = API(statestore, control) app = Flask(__name__) -@app.route('/get_model_trail', methods=['GET']) +@app.route("/get_model_trail", methods=["GET"]) def get_model_trail(): - """ Get the model trail for a given session. + """Get the model trail for a given session. param: session: The session id to get the model trail for. type: session: str return: The model trail for the given session as a json object. @@ -30,9 +32,9 @@ def get_model_trail(): return api.get_model_trail() -@app.route('/delete_model_trail', methods=['GET', 'POST']) +@app.route("/delete_model_trail", methods=["GET", "POST"]) def delete_model_trail(): - """ Delete the model trail for a given session. + """Delete the model trail for a given session. param: session: The session id to delete the model trail for. type: session: str return: The response from the statestore. @@ -41,78 +43,84 @@ def delete_model_trail(): return jsonify({"message": "Not implemented"}), 501 -@app.route('/list_clients', methods=['GET']) +@app.route("/list_clients", methods=["GET"]) def list_clients(): - """ Get all clients from the statestore. + """Get all clients from the statestore. return: All clients as a json object. rtype: json """ return api.get_all_clients() -@app.route('/get_active_clients', methods=['GET']) +@app.route("/get_active_clients", methods=["GET"]) def get_active_clients(): - """ Get all active clients from the statestore. + """Get all active clients from the statestore. param: combiner_id: The combiner id to get active clients for. type: combiner_id: str return: All active clients as a json object. rtype: json """ - combiner_id = request.args.get('combiner', None) + combiner_id = request.args.get("combiner", None) if combiner_id is None: - return jsonify({"success": False, "message": "Missing combiner id."}), 400 + return ( + jsonify({"success": False, "message": "Missing combiner id."}), + 400, + ) return api.get_active_clients(combiner_id) -@app.route('/list_combiners', methods=['GET']) +@app.route("/list_combiners", methods=["GET"]) def list_combiners(): - """ Get all combiners in the network. + """Get all combiners in the network. return: All combiners as a json object. rtype: json """ return api.get_all_combiners() -@app.route('/get_combiner', methods=['GET']) +@app.route("/get_combiner", methods=["GET"]) def get_combiner(): - """ Get a combiner from the statestore. + """Get a combiner from the statestore. param: combiner_id: The combiner id to get. type: combiner_id: str return: The combiner as a json object. rtype: json """ - combiner_id = request.args.get('combiner', None) + combiner_id = request.args.get("combiner", None) if combiner_id is None: - return jsonify({"success": False, "message": "Missing combiner id."}), 400 + return ( + jsonify({"success": False, "message": "Missing combiner id."}), + 400, + ) return api.get_combiner(combiner_id) -@app.route('/list_rounds', methods=['GET']) +@app.route("/list_rounds", methods=["GET"]) def list_rounds(): - """ Get all rounds from the statestore. + """Get all rounds from the statestore. return: All rounds as a json object. rtype: json """ return api.get_all_rounds() -@app.route('/get_round', methods=['GET']) +@app.route("/get_round", methods=["GET"]) def get_round(): - """ Get a round from the statestore. + """Get a round from the statestore. param: round_id: The round id to get. type: round_id: str return: The round as a json object. rtype: json """ - round_id = request.args.get('round_id', None) + round_id = request.args.get("round_id", None) if round_id is None: return jsonify({"success": False, "message": "Missing round id."}), 400 return api.get_round(round_id) -@app.route('/start_session', methods=['GET', 'POST']) +@app.route("/start_session", methods=["GET", "POST"]) def start_session(): - """ Start a new session. + """Start a new session. return: The response from control. rtype: json """ @@ -120,30 +128,36 @@ def start_session(): return api.start_session(**json_data) -@app.route('/list_sessions', methods=['GET']) +@app.route("/list_sessions", methods=["GET"]) def list_sessions(): - """ Get all sessions from the statestore. + """Get all sessions from the statestore. return: All sessions as a json object. rtype: json """ - return api.get_all_sessions() + limit = request.args.get("limit", None) + skip = request.args.get("skip", None) + + return api.get_all_sessions(limit, skip) -@app.route('/get_session', methods=['GET']) +@app.route("/get_session", methods=["GET"]) def get_session(): - """ Get a session from the statestore. + """Get a session from the statestore. param: session_id: The session id to get. type: session_id: str return: The session as a json object. rtype: json """ - session_id = request.args.get('session_id', None) + session_id = request.args.get("session_id", None) if session_id is None: - return jsonify({"success": False, "message": "Missing session id."}), 400 + return ( + jsonify({"success": False, "message": "Missing session id."}), + 400, + ) return api.get_session(session_id) -@app.route('/set_package', methods=['POST']) +@app.route("/set_package", methods=["POST"]) def set_package(): """ Set the compute package in the statestore. Usage with curl: @@ -157,64 +171,68 @@ def set_package(): return: The response from the statestore. rtype: json """ - helper_type = request.form.get('helper', None) + helper_type = request.form.get("helper", None) if helper_type is None: - return jsonify({"success": False, "message": "Missing helper type."}), 400 + return ( + jsonify({"success": False, "message": "Missing helper type."}), + 400, + ) try: - file = request.files['file'] + file = request.files["file"] except KeyError: return jsonify({"success": False, "message": "Missing file."}), 400 return api.set_compute_package(file=file, helper_type=helper_type) -@app.route('/get_package', methods=['GET']) +@app.route("/get_package", methods=["GET"]) def get_package(): - """ Get the compute package from the statestore. + """Get the compute package from the statestore. return: The compute package as a json object. rtype: json """ return api.get_compute_package() -@app.route('/download_package', methods=['GET']) +@app.route("/download_package", methods=["GET"]) def download_package(): - """ Download the compute package. + """Download the compute package. return: The compute package as a json object. rtype: json """ - name = request.args.get('name', None) + name = request.args.get("name", None) return api.download_compute_package(name) -@app.route('/get_package_checksum', methods=['GET']) +@app.route("/get_package_checksum", methods=["GET"]) def get_package_checksum(): - name = request.args.get('name', None) + name = request.args.get("name", None) return api.get_checksum(name) -@app.route('/get_latest_model', methods=['GET']) +@app.route("/get_latest_model", methods=["GET"]) def get_latest_model(): - """ Get the latest model from the statestore. + """Get the latest model from the statestore. return: The initial model as a json object. rtype: json """ return api.get_latest_model() + # Get initial model endpoint -@app.route('/get_initial_model', methods=['GET']) +@app.route("/get_initial_model", methods=["GET"]) def get_initial_model(): - """ Get the initial model from the statestore. + """Get the initial model from the statestore. return: The initial model as a json object. rtype: json """ return api.get_initial_model() -@app.route('/set_initial_model', methods=['POST']) +@app.route("/set_initial_model", methods=["POST"]) def set_initial_model(): - """ Set the initial model in the statestore and upload to model repository. + """Set the initial model in the statestore and upload to model repository. Usage with curl: curl -k -X POST -F file=@seed.npz @@ -226,45 +244,48 @@ def set_initial_model(): rtype: json """ try: - file = request.files['file'] + file = request.files["file"] except KeyError: return jsonify({"success": False, "message": "Missing file."}), 400 return api.set_initial_model(file) -@app.route('/get_controller_status', methods=['GET']) +@app.route("/get_controller_status", methods=["GET"]) def get_controller_status(): - """ Get the status of the controller. + """Get the status of the controller. return: The status as a json object. rtype: json """ return api.get_controller_status() -@app.route('/get_client_config', methods=['GET']) +@app.route("/get_client_config", methods=["GET"]) def get_client_config(): - """ Get the client configuration. + """Get the client configuration. return: The client configuration as a json object. rtype: json """ - checksum = request.args.get('checksum', True) + checksum = request.args.get("checksum", True) return api.get_client_config(checksum) -@app.route('/get_events', methods=['GET']) +@app.route("/get_events", methods=["GET"]) def get_events(): - """ Get the events from the statestore. + """Get the events from the statestore. return: The events as a json object. rtype: json """ # TODO: except filter with request.get_json() kwargs = request.args.to_dict() + + print("get_events") + print(kwargs) return api.get_events(**kwargs) -@app.route('/list_validations', methods=['GET']) +@app.route("/list_validations", methods=["GET"]) def list_validations(): - """ Get all validations from the statestore. + """Get all validations from the statestore. return: All validations as a json object. rtype: json """ @@ -273,9 +294,9 @@ def list_validations(): return api.get_all_validations(**kwargs) -@app.route('/add_combiner', methods=['POST']) +@app.route("/add_combiner", methods=["POST"]) def add_combiner(): - """ Add a combiner to the network. + """Add a combiner to the network. return: The response from the statestore. rtype: json """ @@ -284,13 +305,13 @@ def add_combiner(): try: response = api.add_combiner(**json_data, remote_addr=remote_addr) except TypeError as e: - return jsonify({'success': False, 'message': str(e)}), 400 + return jsonify({"success": False, "message": str(e)}), 400 return response -@app.route('/add_client', methods=['POST']) +@app.route("/add_client", methods=["POST"]) def add_client(): - """ Add a client to the network. + """Add a client to the network. return: The response from control. rtype: json """ @@ -300,12 +321,12 @@ def add_client(): try: response = api.add_client(**json_data, remote_addr=remote_addr) except TypeError as e: - return jsonify({'success': False, 'message': str(e)}), 400 + return jsonify({"success": False, "message": str(e)}), 400 return response -if __name__ == '__main__': +if __name__ == "__main__": config = get_controller_config() - port = config['port'] - debug = config['debug'] - app.run(debug=debug, port=port, host='0.0.0.0') + port = config["port"] + debug = config["debug"] + app.run(debug=debug, port=port, host="0.0.0.0") diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index f991701d4..113802a89 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -10,7 +10,7 @@ class MongoStateStore(StateStoreBase): - """ Statestore implementation using MongoDB. + """Statestore implementation using MongoDB. :param network_id: The network id. :type network_id: str @@ -21,7 +21,7 @@ class MongoStateStore(StateStoreBase): """ def __init__(self, network_id, config, model_storage_config): - """ Constructor.""" + """Constructor.""" self.__inited = False try: self.config = config @@ -29,19 +29,19 @@ def __init__(self, network_id, config, model_storage_config): self.mdb = connect_to_mongodb(self.config, self.network_id) # FEDn network - self.network = self.mdb['network'] - self.reducer = self.network['reducer'] - self.combiners = self.network['combiners'] - self.clients = self.network['clients'] - self.storage = self.network['storage'] + self.network = self.mdb["network"] + self.reducer = self.network["reducer"] + self.combiners = self.network["combiners"] + self.clients = self.network["clients"] + self.storage = self.network["storage"] # Control - self.control = self.mdb['control'] - self.package = self.control['package'] - self.state = self.control['state'] - self.model = self.control['model'] - self.sessions = self.control['sessions'] - self.rounds = self.control['rounds'] + self.control = self.mdb["control"] + self.package = self.control["package"] + self.state = self.control["state"] + self.model = self.control["model"] + self.sessions = self.control["sessions"] + self.rounds = self.control["rounds"] # Logging self.status = self.control["status"] @@ -62,7 +62,7 @@ def __init__(self, network_id, config, model_storage_config): self.__inited = True def is_inited(self): - """ Check if the statestore is intialized. + """Check if the statestore is intialized. :return: True if initialized, else False. :rtype: bool @@ -76,105 +76,126 @@ def get_config(self): :rtype: dict """ data = { - 'type': 'MongoDB', - 'mongo_config': self.config, - 'network_id': self.network_id + "type": "MongoDB", + "mongo_config": self.config, + "network_id": self.network_id, } return data def state(self): - """ Get the current state. + """Get the current state. :return: The current state. :rtype: str """ - return StringToReducerState(self.state.find_one()['current_state']) + return StringToReducerState(self.state.find_one()["current_state"]) def transition(self, state): - """ Transition to a new state. + """Transition to a new state. :param state: The new state. :type state: str :return: """ - old_state = self.state.find_one({'state': 'current_state'}) + old_state = self.state.find_one({"state": "current_state"}) if old_state != state: - return self.state.update_one({'state': 'current_state'}, {'$set': {'state': ReducerStateToString(state)}}, True) + return self.state.update_one( + {"state": "current_state"}, + {"$set": {"state": ReducerStateToString(state)}}, + True, + ) else: - print("Not updating state, already in {}".format( - ReducerStateToString(state))) + print( + "Not updating state, already in {}".format( + ReducerStateToString(state) + ) + ) - def get_sessions(self): - """ Get all sessions. + def get_sessions(self, limit=None, skip=None): + """Get all sessions. :return: All sessions. :rtype: ObjectID """ + + if limit is not None and skip is not None: + return self.sessions.find().limit(limit).skip(skip) + return self.sessions.find() def get_session(self, session_id): - """ Get session with id. + """Get session with id. :param session_id: The session id. :type session_id: str :return: The session. :rtype: ObjectID """ - return self.sessions.find_one({'session_id': session_id}) + return self.sessions.find_one({"session_id": session_id}) def set_latest_model(self, model_id): - """ Set the latest model id. + """Set the latest model id. :param model_id: The model id. :type model_id: str :return: """ - self.model.update_one({'key': 'current_model'}, { - '$set': {'model': model_id}}, True) - self.model.update_one({'key': 'model_trail'}, {'$push': {'model': model_id, 'committed_at': str(datetime.now())}}, - True) + self.model.update_one( + {"key": "current_model"}, {"$set": {"model": model_id}}, True + ) + self.model.update_one( + {"key": "model_trail"}, + { + "$push": { + "model": model_id, + "committed_at": str(datetime.now()), + } + }, + True, + ) def get_initial_model(self): - """ Return model_id for the initial model in the model trail + """Return model_id for the initial model in the model trail :return: The initial model id. None if no model is found. :rtype: str """ - result = self.model.find_one({'key': 'model_trail'}, sort=[ - ("committed_at", pymongo.ASCENDING)]) + result = self.model.find_one( + {"key": "model_trail"}, sort=[("committed_at", pymongo.ASCENDING)] + ) if result is None: return None try: - model_id = result['model'] - if model_id == '' or model_id == ' ': + model_id = result["model"] + if model_id == "" or model_id == " ": return None return model_id[0] except (KeyError, IndexError): return None def get_latest_model(self): - """ Return model_id for the latest model in the model_trail + """Return model_id for the latest model in the model_trail :return: The latest model id. None if no model is found. :rtype: str """ - result = self.model.find_one({'key': 'current_model'}) + result = self.model.find_one({"key": "current_model"}) if result is None: return None try: - model_id = result['model'] - if model_id == '' or model_id == ' ': + model_id = result["model"] + if model_id == "" or model_id == " ": return None return model_id except (KeyError, IndexError): return None def get_latest_round(self): - """ Get the id of the most recent round. + """Get the id of the most recent round. :return: The id of the most recent round. :rtype: ObjectId @@ -183,7 +204,7 @@ def get_latest_round(self): return self.rounds.find_one(sort=[("_id", pymongo.DESCENDING)]) def get_round(self, id): - """ Get round with id. + """Get round with id. :param id: id of round to get :type id: int @@ -191,10 +212,10 @@ def get_round(self, id): :rtype: ObjectId """ - return self.rounds.find_one({'round_id': str(id)}) + return self.rounds.find_one({"round_id": str(id)}) def get_rounds(self): - """ Get all rounds. + """Get all rounds. :return: All rounds. :rtype: ObjectId @@ -203,7 +224,7 @@ def get_rounds(self): return self.rounds.find() def get_validations(self, **kwargs): - """ Get validations from the database. + """Get validations from the database. :param kwargs: query to filter validations :type kwargs: dict @@ -215,7 +236,7 @@ def get_validations(self, **kwargs): return result def set_compute_package(self, filename): - """ Set the active compute package in statestore. + """Set the active compute package in statestore. :param filename: The filename of the compute package. :type filename: str @@ -223,66 +244,87 @@ def set_compute_package(self, filename): :rtype: bool """ self.control.package.update_one( - {'key': 'active'}, {'$set': {'filename': filename, 'committed_at': str(datetime.now())}}, True) - self.control.package.update_one({'key': 'package_trail'}, - {'$push': {'filename': filename, 'committed_at': str(datetime.now())}}, True) + {"key": "active"}, + { + "$set": { + "filename": filename, + "committed_at": str(datetime.now()), + } + }, + True, + ) + self.control.package.update_one( + {"key": "package_trail"}, + { + "$push": { + "filename": filename, + "committed_at": str(datetime.now()), + } + }, + True, + ) return True def get_compute_package(self): - """ Get the active compute package. + """Get the active compute package. :return: The active compute package. :rtype: ObjectID """ - ret = self.control.package.find({'key': 'active'}) + ret = self.control.package.find({"key": "active"}) try: retcheck = ret[0] - if retcheck is None or retcheck == '' or retcheck == ' ': # ugly check for empty string + if ( + retcheck is None or retcheck == "" or retcheck == " " + ): # ugly check for empty string return None return retcheck except (KeyError, IndexError): return None def set_helper(self, helper): - """ Set the active helper package in statestore. + """Set the active helper package in statestore. :param helper: The name of the helper package. See helper.py for available helpers. :type helper: str :return: """ - self.control.package.update_one({'key': 'active'}, - {'$set': {'helper': helper}}, True) + self.control.package.update_one( + {"key": "active"}, {"$set": {"helper": helper}}, True + ) def get_helper(self): - """ Get the active helper package. + """Get the active helper package. :return: The active helper set for the package. :rtype: str """ - ret = self.control.package.find_one({'key': 'active'}) + ret = self.control.package.find_one({"key": "active"}) # if local compute package used, then 'package' is None # if not ret: # get framework from round_config instead # ret = self.control.config.find_one({'key': 'round_config'}) try: - retcheck = ret['helper'] - if retcheck == '' or retcheck == ' ': # ugly check for empty string + retcheck = ret["helper"] + if ( + retcheck == "" or retcheck == " " + ): # ugly check for empty string return None return retcheck except (KeyError, IndexError): return None def get_model_trail(self): - """ Get the model trail. + """Get the model trail. :return: dictionary of model_id: committed_at :rtype: dict """ - result = self.model.find_one({'key': 'model_trail'}) + result = self.model.find_one({"key": "model_trail"}) try: if result is not None: - committed_at = result['committed_at'] - model = result['model'] + committed_at = result["committed_at"] + model = result["model"] model_dictionary = dict(zip(model, committed_at)) return model_dictionary else: @@ -291,7 +333,7 @@ def get_model_trail(self): return None def get_events(self, **kwargs): - """ Get events from the database. + """Get events from the database. :param kwargs: query to filter events :type kwargs: dict @@ -300,50 +342,64 @@ def get_events(self, **kwargs): """ # check if kwargs is empty if not kwargs: - return self.control.status.find() + result = self.control.status.find() else: - result = self.control.status.find(kwargs) + limit = kwargs.pop("limit", None) + skip = kwargs.pop("skip", None) + + if limit is not None and skip is not None: + limit = int(limit) + skip = int(skip) + result = ( + self.control.status.find(kwargs).limit(limit).skip(skip) + ) + else: + result = self.control.status.find(kwargs) + return result def get_storage_backend(self): - """ Get the storage backend. + """Get the storage backend. :return: The storage backend. :rtype: ObjectID """ try: ret = self.storage.find( - {'status': 'enabled'}, projection={'_id': False}) + {"status": "enabled"}, projection={"_id": False} + ) return ret[0] except (KeyError, IndexError): return None def set_storage_backend(self, config): - """ Set the storage backend. + """Set the storage backend. :param config: The storage backend configuration. :type config: dict :return: """ config = copy.deepcopy(config) - config['updated_at'] = str(datetime.now()) - config['status'] = 'enabled' + config["updated_at"] = str(datetime.now()) + config["status"] = "enabled" self.storage.update_one( - {'storage_type': config['storage_type']}, {'$set': config}, True) + {"storage_type": config["storage_type"]}, {"$set": config}, True + ) def set_reducer(self, reducer_data): - """ Set the reducer in the statestore. + """Set the reducer in the statestore. :param reducer_data: dictionary of reducer config. :type reducer_data: dict :return: """ - reducer_data['updated_at'] = str(datetime.now()) - self.reducer.update_one({'name': reducer_data['name']}, { - '$set': reducer_data}, True) + reducer_data["updated_at"] = str(datetime.now()) + self.reducer.update_one( + {"name": reducer_data["name"]}, {"$set": reducer_data}, True + ) def get_reducer(self): - """ Get reducer.config. + """Get reducer.config. return: reducer config. rtype: ObjectId @@ -355,19 +411,19 @@ def get_reducer(self): return None def get_combiner(self, name): - """ Get combiner by name. + """Get combiner by name. :return: The combiner. :rtype: ObjectId """ try: - ret = self.combiners.find_one({'name': name}) + ret = self.combiners.find_one({"name": name}) return ret except Exception: return None def get_combiners(self): - """ Get all combiners. + """Get all combiners. :return: list of combiners. :rtype: list @@ -379,43 +435,47 @@ def get_combiners(self): return None def set_combiner(self, combiner_data): - """ Set combiner in statestore. + """Set combiner in statestore. :param combiner_data: dictionary of combiner config :type combiner_data: dict :return: """ - combiner_data['updated_at'] = str(datetime.now()) - self.combiners.update_one({'name': combiner_data['name']}, { - '$set': combiner_data}, True) + combiner_data["updated_at"] = str(datetime.now()) + self.combiners.update_one( + {"name": combiner_data["name"]}, {"$set": combiner_data}, True + ) def delete_combiner(self, combiner): - """ Delete a combiner from statestore. + """Delete a combiner from statestore. :param combiner: name of combiner to delete. :type combiner: str :return: """ try: - self.combiners.delete_one({'name': combiner}) + self.combiners.delete_one({"name": combiner}) except Exception: - print("WARNING, failed to delete combiner: {}".format( - combiner), flush=True) + print( + "WARNING, failed to delete combiner: {}".format(combiner), + flush=True, + ) def set_client(self, client_data): - """ Set client in statestore. + """Set client in statestore. :param client_data: dictionary of client config. :type client_data: dict :return: """ - client_data['updated_at'] = str(datetime.now()) - self.clients.update_one({'name': client_data['name']}, { - '$set': client_data}, True) + client_data["updated_at"] = str(datetime.now()) + self.clients.update_one( + {"name": client_data["name"]}, {"$set": client_data}, True + ) def get_client(self, name): - """ Get client by name. + """Get client by name. :param name: name of client to get. :type name: str @@ -423,7 +483,7 @@ def get_client(self, name): :rtype: ObjectId """ try: - ret = self.clients.find({'key': name}) + ret = self.clients.find({"key": name}) if list(ret) == []: return None else: @@ -444,7 +504,7 @@ def list_clients(self): return None def update_client_status(self, client_data, status, role): - """ Set or update client status. + """Set or update client status. :param client_data: dictionary of client config. :type client_data: dict @@ -454,10 +514,7 @@ def update_client_status(self, client_data, status, role): :type role: str :return: """ - self.clients.update_one({"name": client_data['name']}, - {"$set": - { - "status": status, - "role": role - } - }) + self.clients.update_one( + {"name": client_data["name"]}, + {"$set": {"status": status, "role": role}}, + ) From edf9f575f924ff6af781edec0e35fdad52ffe334 Mon Sep 17 00:00:00 2001 From: Niklas Date: Tue, 17 Oct 2023 13:25:41 +0200 Subject: [PATCH 02/23] get_sessions pagination v1 --- fedn/fedn/network/api/interface.py | 10 +++++++--- fedn/fedn/network/statestore/mongostatestore.py | 16 ++++++++++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index f886261d0..a86f0feb8 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -165,17 +165,21 @@ def get_all_sessions(self, limit=None, skip=None): :return: All sessions as a json response. :rtype: :class:`flask.Response` """ - sessions_objects = self.statestore.get_sessions(limit, skip) - if sessions_objects is None: + sessions_object = self.statestore.get_sessions(limit, skip) + if sessions_object is None: return ( jsonify({"success": False, "message": "No sessions found."}), 404, ) payload = {} - for object in sessions_objects: + for object in sessions_object["result"]: id = object["session_id"] info = object["session_config"][0] payload[id] = info + + result = {"result": payload, "count": sessions_object["count"]} + + return jsonify(result) return jsonify(payload) def get_session(self, session_id): diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index 113802a89..903d1a208 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -118,10 +118,22 @@ def get_sessions(self, limit=None, skip=None): :rtype: ObjectID """ + result = None + if limit is not None and skip is not None: - return self.sessions.find().limit(limit).skip(skip) + limit = int(limit) + skip = int(skip) + + result = self.sessions.find().limit(limit).skip(skip) + else: + result = self.sessions.find() - return self.sessions.find() + count = self.sessions.count_documents({}) + + return { + "result": result, + "count": count, + } def get_session(self, session_id): """Get session with id. From a0645067570744bc839978b8b57044b4310115e8 Mon Sep 17 00:00:00 2001 From: Niklas Date: Wed, 18 Oct 2023 15:17:06 +0200 Subject: [PATCH 03/23] get_models added to api --- fedn/fedn/network/api/interface.py | 19 +- fedn/fedn/network/api/server.py | 20 ++ fedn/fedn/network/controller/control.py | 237 +++++++++++------- fedn/fedn/network/controller/controlbase.py | 124 +++++---- .../network/statestore/mongostatestore.py | 51 +++- 5 files changed, 315 insertions(+), 136 deletions(-) diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index a86f0feb8..40792da81 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -180,7 +180,6 @@ def get_all_sessions(self, limit=None, skip=None): result = {"result": payload, "count": sessions_object["count"]} return jsonify(result) - return jsonify(payload) def get_session(self, session_id): """Get a session from the statestore. @@ -632,6 +631,24 @@ def get_latest_model(self): {"success": False, "message": "No initial model set."} ) + def get_models(self, session_id=None, limit=None, skip=None): + result = self.statestore.list_models(session_id, limit, skip) + + if result is None: + return ( + jsonify({"success": False, "message": "No models found."}), + 404, + ) + + json_docs = [] + for doc in result["result"]: + json_doc = json.dumps(doc, default=json_util.default) + json_docs.append(json_doc) + + json_docs.reverse() + + return jsonify({"result": json_docs, "count": result["count"]}) + def get_model_trail(self): """Get the model trail for a given session. diff --git a/fedn/fedn/network/api/server.py b/fedn/fedn/network/api/server.py index 2bdd7aae3..0c49343f8 100644 --- a/fedn/fedn/network/api/server.py +++ b/fedn/fedn/network/api/server.py @@ -32,6 +32,26 @@ def get_model_trail(): return api.get_model_trail() +@app.route("/list_models", methods=["GET"]) +def list_models(): + """Get models from the statestore. + param: + session_id: The session id to get the model trail for. + limit: The maximum number of models to return. + type: limit: int + param: skip: The number of models to skip. + type: skip: int + Returns: + _type_: json + """ + + session_id = request.args.get("session_id", None) + limit = request.args.get("limit", None) + skip = request.args.get("skip", None) + + return api.get_models(session_id, limit, skip) + + @app.route("/delete_model_trail", methods=["GET", "POST"]) def delete_model_trail(): """Delete the model trail for a given session. diff --git a/fedn/fedn/network/controller/control.py b/fedn/fedn/network/controller/control.py index 5f5bc6634..a8e32333d 100644 --- a/fedn/fedn/network/controller/control.py +++ b/fedn/fedn/network/controller/control.py @@ -9,10 +9,10 @@ class UnsupportedStorageBackend(Exception): - """ Exception class for when storage backend is not supported. Passes """ + """Exception class for when storage backend is not supported. Passes""" def __init__(self, message): - """ Constructor method. + """Constructor method. :param message: The exception message. :type message: str @@ -23,46 +23,46 @@ def __init__(self, message): class MisconfiguredStorageBackend(Exception): - """ Exception class for when storage backend is misconfigured. + """Exception class for when storage backend is misconfigured. :param message: The exception message. :type message: str """ def __init__(self, message): - """ Constructor method.""" + """Constructor method.""" self.message = message super().__init__(self.message) class NoModelException(Exception): - """ Exception class for when model is None + """Exception class for when model is None :param message: The exception message. :type message: str """ def __init__(self, message): - """ Constructor method.""" + """Constructor method.""" self.message = message super().__init__(self.message) class Control(ControlBase): - """ Controller, implementing the overall global training, validation and inference logic. + """Controller, implementing the overall global training, validation and inference logic. :param statestore: A StateStorage instance. :type statestore: class: `fedn.network.statestorebase.StateStorageBase` """ def __init__(self, statestore): - """ Constructor method.""" + """Constructor method.""" super().__init__(statestore) self.name = "DefaultControl" def session(self, config): - """ Execute a new training session. A session consists of one + """Execute a new training session. A session consists of one or several global rounds. All rounds in the same session have the same round_config. @@ -72,7 +72,10 @@ def session(self, config): """ if self._state == ReducerState.instructing: - print("Controller already in INSTRUCTING state. A session is in progress.", flush=True) + print( + "Controller already in INSTRUCTING state. A session is in progress.", + flush=True, + ) return if not self.statestore.get_latest_model(): @@ -82,11 +85,16 @@ def session(self, config): self._state = ReducerState.instructing # Must be called to set info in the db - config['committed_at'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + config["committed_at"] = datetime.datetime.now().strftime( + "%Y-%m-%d %H:%M:%S" + ) self.new_session(config) if not self.statestore.get_latest_model(): - print("No model in model chain, please provide a seed model!", flush=True) + print( + "No model in model chain, please provide a seed model!", + flush=True, + ) self._state = ReducerState.monitoring last_round = int(self.get_latest_round_id()) @@ -96,7 +104,7 @@ def session(self, config): combiner.flush_model_update_queue() # Execute the rounds in this session - for round in range(1, int(config['rounds'] + 1)): + for round in range(1, int(config["rounds"] + 1)): # Increment the round number if last_round: @@ -107,10 +115,17 @@ def session(self, config): try: _, round_data = self.round(config, current_round) except TypeError as e: - print("Could not unpack data from round: {0}".format(e), flush=True) - - print("CONTROL: Round completed with status {}".format( - round_data['status']), flush=True) + print( + "Could not unpack data from round: {0}".format(e), + flush=True, + ) + + print( + "CONTROL: Round completed with status {}".format( + round_data["status"] + ), + flush=True, + ) self.tracer.set_round_data(round_data) @@ -118,7 +133,7 @@ def session(self, config): self._state = ReducerState.idle def round(self, session_config, round_id): - """ Execute a single global round. + """Execute a single global round. :param session_config: The session config. :type session_config: dict @@ -126,35 +141,42 @@ def round(self, session_config, round_id): :type round_id: str(int) """ - round_data = {'round_id': round_id} + round_data = {"round_id": round_id} if len(self.network.get_combiners()) < 1: print("REDUCER: No combiners connected!", flush=True) - round_data['status'] = 'Failed' + round_data["status"] = "Failed" return None, round_data # 1. Assemble round config for this global round, # and check which combiners are able to participate # in the round. round_config = copy.deepcopy(session_config) - round_config['rounds'] = 1 - round_config['round_id'] = round_id - round_config['task'] = 'training' - round_config['model_id'] = self.statestore.get_latest_model() - round_config['helper_type'] = self.statestore.get_helper() + round_config["rounds"] = 1 + round_config["round_id"] = round_id + round_config["task"] = "training" + round_config["model_id"] = self.statestore.get_latest_model() + round_config["helper_type"] = self.statestore.get_helper() combiners = self.get_participating_combiners(round_config) round_start = self.evaluate_round_start_policy(combiners) if round_start: - print("CONTROL: round start policy met, participating combiners {}".format( - combiners), flush=True) + print( + "CONTROL: round start policy met, participating combiners {}".format( + combiners + ), + flush=True, + ) else: - print("CONTROL: Round start policy not met, skipping round!", flush=True) - round_data['status'] = 'Failed' + print( + "CONTROL: Round start policy not met, skipping round!", + flush=True, + ) + round_data["status"] = "Failed" return None - round_data['round_config'] = round_config + round_data["round_config"] = round_config # 2. Ask participating combiners to coordinate model updates _ = self.request_model_updates(combiners) @@ -164,27 +186,37 @@ def round(self, session_config, round_id): # dict to store combiners that have successfully produced an updated model updated = {} # wait until all combiners have produced an updated model or until round timeout - print("CONTROL: Fetching round config (ID: {round_id}) from statestore:".format( - round_id=round_id), flush=True) + print( + "CONTROL: Fetching round config (ID: {round_id}) from statestore:".format( + round_id=round_id + ), + flush=True, + ) while len(updated) < len(combiners): round = self.statestore.get_round(round_id) if round: print("CONTROL: Round found!", flush=True) # For each combiner in the round, check if it has produced an updated model (status == 'Success') - for combiner in round['combiners']: + for combiner in round["combiners"]: print(combiner, flush=True) - if combiner['status'] == 'Success': - if combiner['name'] not in updated.keys(): + if combiner["status"] == "Success": + if combiner["name"] not in updated.keys(): # Add combiner to updated dict - updated[combiner['name']] = combiner['model_id'] + updated[combiner["name"]] = combiner["model_id"] # Print combiner status - print("CONTROL: Combiner {name} status: {status}".format( - name=combiner['name'], status=combiner['status']), flush=True) + print( + "CONTROL: Combiner {name} status: {status}".format( + name=combiner["name"], status=combiner["status"] + ), + flush=True, + ) else: # Print every 10 seconds based on value of wait if wait % 10 == 0: - print("CONTROL: Waiting for round to complete...", flush=True) - if wait >= session_config['round_timeout']: + print( + "CONTROL: Waiting for round to complete...", flush=True + ) + if wait >= session_config["round_timeout"]: print("CONTROL: Round timeout! Exiting round...", flush=True) break # Update wait time used for timeout @@ -194,53 +226,77 @@ def round(self, session_config, round_id): round_valid = self.evaluate_round_validity_policy(updated) if not round_valid: print("REDUCER CONTROL: Round invalid!", flush=True) - round_data['status'] = 'Failed' + round_data["status"] = "Failed" return None, round_data print("CONTROL: Reducing models from combiners...", flush=True) # 3. Reduce combiner models into a global model try: model, data = self.reduce(updated) - round_data['reduce'] = data + round_data["reduce"] = data print("CONTROL: Done reducing models from combiners!", flush=True) except Exception as e: - print("CONTROL: Failed to reduce models from combiners: {}".format( - e), flush=True) - round_data['status'] = 'Failed' + print( + "CONTROL: Failed to reduce models from combiners: {}".format( + e + ), + flush=True, + ) + round_data["status"] = "Failed" return None, round_data # 6. Commit the global model to model trail if model is not None: - print("CONTROL: Committing global model to model trail...", flush=True) + print( + "CONTROL: Committing global model to model trail...", + flush=True, + ) tic = time.time() model_id = uuid.uuid4() - self.commit(model_id, model) - round_data['time_commit'] = time.time() - tic - print("CONTROL: Done committing global model to model trail!", flush=True) + session_id = ( + session_config["session_id"] + if "session_id" in session_config + else None + ) + self.commit(model_id, model, session_id) + round_data["time_commit"] = time.time() - tic + print( + "CONTROL: Done committing global model to model trail!", + flush=True, + ) else: - print("REDUCER: failed to update model in round with config {}".format( - session_config), flush=True) - round_data['status'] = 'Failed' + print( + "REDUCER: failed to update model in round with config {}".format( + session_config + ), + flush=True, + ) + round_data["status"] = "Failed" return None, round_data - round_data['status'] = 'Success' + round_data["status"] = "Success" # 4. Trigger participating combiner nodes to execute a validation round for the current model - validate = session_config['validate'] + validate = session_config["validate"] if validate: combiner_config = copy.deepcopy(session_config) - combiner_config['round_id'] = round_id - combiner_config['model_id'] = self.statestore.get_latest_model() - combiner_config['task'] = 'validation' - combiner_config['helper_type'] = self.statestore.get_helper() + combiner_config["round_id"] = round_id + combiner_config["model_id"] = self.statestore.get_latest_model() + combiner_config["task"] = "validation" + combiner_config["helper_type"] = self.statestore.get_helper() validating_combiners = self._select_participating_combiners( - combiner_config) + combiner_config + ) for combiner, combiner_config in validating_combiners: try: - print("CONTROL: Submitting validation round to combiner {}".format( - combiner), flush=True) + print( + "CONTROL: Submitting validation round to combiner {}".format( + combiner + ), + flush=True, + ) combiner.submit(combiner_config) except CombinerUnavailableError: self._handle_unavailable_combiner(combiner) @@ -249,16 +305,16 @@ def round(self, session_config, round_id): return model_id, round_data def reduce(self, combiners): - """ Combine updated models from Combiner nodes into one global model. + """Combine updated models from Combiner nodes into one global model. :param combiners: dict of combiner names (key) and model IDs (value) to reduce :type combiners: dict """ meta = {} - meta['time_fetch_model'] = 0.0 - meta['time_load_model'] = 0.0 - meta['time_aggregate_model'] = 0.0 + meta["time_fetch_model"] = 0.0 + meta["time_load_model"] = 0.0 + meta["time_aggregate_model"] = 0.0 i = 1 model = None @@ -268,18 +324,25 @@ def reduce(self, combiners): return model, meta for name, model_id in combiners.items(): - # TODO: Handle inactive RPC error in get_model and raise specific error - print("REDUCER: Fetching model ({model_id}) from combiner {name}".format( - model_id=model_id, name=name), flush=True) + print( + "REDUCER: Fetching model ({model_id}) from combiner {name}".format( + model_id=model_id, name=name + ), + flush=True, + ) try: tic = time.time() combiner = self.get_combiner(name) data = combiner.get_model(model_id) - meta['time_fetch_model'] += (time.time() - tic) + meta["time_fetch_model"] += time.time() - tic except Exception as e: - print("REDUCER: Failed to fetch model from combiner {}: {}".format( - name, e), flush=True) + print( + "REDUCER: Failed to fetch model from combiner {}: {}".format( + name, e + ), + flush=True, + ) data = None if data is not None: @@ -288,21 +351,21 @@ def reduce(self, combiners): helper = self.get_helper() data.seek(0) model_next = helper.load(data) - meta['time_load_model'] += (time.time() - tic) + meta["time_load_model"] += time.time() - tic tic = time.time() model = helper.increment_average(model, model_next, i, i) - meta['time_aggregate_model'] += (time.time() - tic) + meta["time_aggregate_model"] += time.time() - tic except Exception: tic = time.time() data.seek(0) model = helper.load(data) - meta['time_aggregate_model'] += (time.time() - tic) + meta["time_aggregate_model"] += time.time() - tic i = i + 1 return model, meta def infer_instruct(self, config): - """ Main entrypoint for executing the inference compute plan. + """Main entrypoint for executing the inference compute plan. :param config: configuration for the inference round """ @@ -330,7 +393,7 @@ def infer_instruct(self, config): self.__state = ReducerState.idle def inference_round(self, config): - """ Execute an inference round. + """Execute an inference round. :param config: configuration for the inference round """ @@ -345,21 +408,27 @@ def inference_round(self, config): # Setup combiner configuration combiner_config = copy.deepcopy(config) - combiner_config['model_id'] = self.statestore.get_latest_model() - combiner_config['task'] = 'inference' - combiner_config['helper_type'] = self.statestore.get_framework() + combiner_config["model_id"] = self.statestore.get_latest_model() + combiner_config["task"] = "inference" + combiner_config["helper_type"] = self.statestore.get_framework() # Select combiners - validating_combiners = self._select_round_combiners( - combiner_config) + validating_combiners = self._select_round_combiners(combiner_config) # Test round start policy round_start = self.check_round_start_policy(validating_combiners) if round_start: - print("CONTROL: round start policy met, participating combiners {}".format( - validating_combiners), flush=True) + print( + "CONTROL: round start policy met, participating combiners {}".format( + validating_combiners + ), + flush=True, + ) else: - print("CONTROL: Round start policy not met, skipping round!", flush=True) + print( + "CONTROL: Round start policy not met, skipping round!", + flush=True, + ) return None # Synch combiners with latest model and trigger inference diff --git a/fedn/fedn/network/controller/controlbase.py b/fedn/fedn/network/controller/controlbase.py index e38d31e38..077620c14 100644 --- a/fedn/fedn/network/controller/controlbase.py +++ b/fedn/fedn/network/controller/controlbase.py @@ -11,7 +11,7 @@ from fedn.network.state import ReducerState # Maximum number of tries to connect to statestore and retrieve storage configuration -MAX_TRIES_BACKEND = os.getenv('MAX_TRIES_BACKEND', 10) +MAX_TRIES_BACKEND = os.getenv("MAX_TRIES_BACKEND", 10) class UnsupportedStorageBackend(Exception): @@ -27,7 +27,7 @@ class MisconfiguredHelper(Exception): class ControlBase(ABC): - """ Base class and interface for a global controller. + """Base class and interface for a global controller. Override this class to implement a global training strategy (control). :param statestore: The statestore object. @@ -36,7 +36,7 @@ class ControlBase(ABC): @abstractmethod def __init__(self, statestore): - """ Constructor. """ + """Constructor.""" self._state = ReducerState.setup self.statestore = statestore @@ -52,26 +52,36 @@ def __init__(self, statestore): not_ready = False else: print( - "REDUCER CONTROL: Storage backend not configured, waiting...", flush=True) + "REDUCER CONTROL: Storage backend not configured, waiting...", + flush=True, + ) sleep(5) tries += 1 if tries > MAX_TRIES_BACKEND: raise Exception except Exception: print( - "REDUCER CONTROL: Failed to retrive storage configuration, exiting.", flush=True) + "REDUCER CONTROL: Failed to retrive storage configuration, exiting.", + flush=True, + ) raise MisconfiguredStorageBackend() - if storage_config['storage_type'] == 'S3': - self.model_repository = S3ModelRepository(storage_config['storage_config']) + if storage_config["storage_type"] == "S3": + self.model_repository = S3ModelRepository( + storage_config["storage_config"] + ) else: - print("REDUCER CONTROL: Unsupported storage backend, exiting.", flush=True) + print( + "REDUCER CONTROL: Unsupported storage backend, exiting.", + flush=True, + ) raise UnsupportedStorageBackend() # The tracer is a helper that manages state in the database backend statestore_config = statestore.get_config() self.tracer = MongoTracer( - statestore_config['mongo_config'], statestore_config['network_id']) + statestore_config["mongo_config"], statestore_config["network_id"] + ) if self.statestore.is_inited(): self._state = ReducerState.idle @@ -89,7 +99,7 @@ def reduce(self, combiners): pass def get_helper(self): - """ Get a helper instance from global config. + """Get a helper instance from global config. :return: Helper instance. :rtype: :class:`fedn.utils.plugins.helperbase.HelperBase` @@ -97,11 +107,15 @@ def get_helper(self): helper_type = self.statestore.get_helper() helper = fedn.utils.helpers.get_helper(helper_type) if not helper: - raise MisconfiguredHelper("Unsupported helper type {}, please configure compute_package.helper !".format(helper_type)) + raise MisconfiguredHelper( + "Unsupported helper type {}, please configure compute_package.helper !".format( + helper_type + ) + ) return helper def get_state(self): - """ Get the current state of the controller. + """Get the current state of the controller. :return: The current state. :rtype: :class:`fedn.network.state.ReducerState` @@ -109,7 +123,7 @@ def get_state(self): return self._state def idle(self): - """ Check if the controller is idle. + """Check if the controller is idle. :return: True if idle, False otherwise. :rtype: bool @@ -139,7 +153,7 @@ def get_latest_round_id(self): if not last_round: return 0 else: - return last_round['round_id'] + return last_round["round_id"] def get_latest_round(self): round = self.statestore.get_latest_round() @@ -153,27 +167,29 @@ def get_compute_package_name(self): definition = self.statestore.get_compute_package() if definition: try: - package_name = definition['filename'] + package_name = definition["filename"] return package_name except (IndexError, KeyError): print( - "No context filename set for compute context definition", flush=True) + "No context filename set for compute context definition", + flush=True, + ) return None else: return None def set_compute_package(self, filename, path): - """ Persist the configuration for the compute package. """ + """Persist the configuration for the compute package.""" self.model_repository.set_compute_package(filename, path) self.statestore.set_compute_package(filename) - def get_compute_package(self, compute_package=''): + def get_compute_package(self, compute_package=""): """ :param compute_package: :return: """ - if compute_package == '': + if compute_package == "": compute_package = self.get_compute_package_name() if compute_package: return self.model_repository.get_compute_package(compute_package) @@ -181,42 +197,49 @@ def get_compute_package(self, compute_package=''): return None def new_session(self, config): - """ Initialize a new session in backend db. """ + """Initialize a new session in backend db.""" if "session_id" not in config.keys(): session_id = uuid.uuid4() - config['session_id'] = str(session_id) + config["session_id"] = str(session_id) else: - session_id = config['session_id'] + session_id = config["session_id"] self.tracer.new_session(id=session_id) self.tracer.set_session_config(session_id, config) def request_model_updates(self, combiners): - """Call Combiner server RPC to get a model update. """ + """Call Combiner server RPC to get a model update.""" cl = [] for combiner, combiner_round_config in combiners: response = combiner.submit(combiner_round_config) cl.append((combiner, response)) return cl - def commit(self, model_id, model=None): - """ Commit a model to the global model trail. The model commited becomes the lastest consensus model. """ + def commit(self, model_id, model=None, session_id=None): + """Commit a model to the global model trail. The model commited becomes the lastest consensus model.""" helper = self.get_helper() if model is not None: - print("CONTROL: Saving model file temporarily to disk...", flush=True) + print( + "CONTROL: Saving model file temporarily to disk...", flush=True + ) outfile_name = helper.save(model) print("CONTROL: Uploading model to Minio...", flush=True) model_id = self.model_repository.set_model( - outfile_name, is_file=True) + outfile_name, is_file=True + ) print("CONTROL: Deleting temporary model file...", flush=True) os.unlink(outfile_name) - print("CONTROL: Committing model {} to global model trail in statestore...".format( - model_id), flush=True) - self.statestore.set_latest_model(model_id) + print( + "CONTROL: Committing model {} to global model trail in statestore...".format( + model_id + ), + flush=True, + ) + self.statestore.set_latest_model(model_id, session_id) def get_combiner(self, name): for combiner in self.network.get_combiners(): @@ -226,7 +249,7 @@ def get_combiner(self, name): def get_participating_combiners(self, combiner_round_config): """Assemble a list of combiners able to participate in a round as - descibed by combiner_round_config. + descibed by combiner_round_config. """ combiners = [] for combiner in self.network.get_combiners(): @@ -238,45 +261,47 @@ def get_participating_combiners(self, combiner_round_config): if combiner_state is not None: is_participating = self.evaluate_round_participation_policy( - combiner_round_config, combiner_state) + combiner_round_config, combiner_state + ) if is_participating: combiners.append((combiner, combiner_round_config)) return combiners - def evaluate_round_participation_policy(self, compute_plan, combiner_state): - """ Evaluate policy for combiner round-participation. - A combiner participates if it is responsive and reports enough - active clients to participate in the round. + def evaluate_round_participation_policy( + self, compute_plan, combiner_state + ): + """Evaluate policy for combiner round-participation. + A combiner participates if it is responsive and reports enough + active clients to participate in the round. """ - if compute_plan['task'] == 'training': - nr_active_clients = int(combiner_state['nr_active_trainers']) - elif compute_plan['task'] == 'validation': - nr_active_clients = int(combiner_state['nr_active_validators']) + if compute_plan["task"] == "training": + nr_active_clients = int(combiner_state["nr_active_trainers"]) + elif compute_plan["task"] == "validation": + nr_active_clients = int(combiner_state["nr_active_validators"]) else: print("Invalid task type!", flush=True) return False - if int(compute_plan['clients_required']) <= nr_active_clients: + if int(compute_plan["clients_required"]) <= nr_active_clients: return True else: return False def evaluate_round_start_policy(self, combiners): - """ Check if the policy to start a round is met. """ + """Check if the policy to start a round is met.""" if len(combiners) > 0: - return True else: return False def evaluate_round_validity_policy(self, combiners): - """ Check if the round should be seen as valid. + """Check if the round should be seen as valid. - At the end of the round, before committing a model to the global model trail, - we check if the round validity policy has been met. This can involve - e.g. asserting that a certain number of combiners have reported in an - updated model, or that criteria on model performance have been met. + At the end of the round, before committing a model to the global model trail, + we check if the round validity policy has been met. This can involve + e.g. asserting that a certain number of combiners have reported in an + updated model, or that criteria on model performance have been met. """ if combiners.keys() == []: return False @@ -294,7 +319,8 @@ def _select_participating_combiners(self, compute_plan): if combiner_state: is_participating = self.evaluate_round_participation_policy( - compute_plan, combiner_state) + compute_plan, combiner_state + ) if is_participating: participating_combiners.append((combiner, compute_plan)) return participating_combiners diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index 903d1a208..9262e9f13 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -145,7 +145,7 @@ def get_session(self, session_id): """ return self.sessions.find_one({"session_id": session_id}) - def set_latest_model(self, model_id): + def set_latest_model(self, model_id, session_id=None): """Set the latest model id. :param model_id: The model id. @@ -153,6 +153,17 @@ def set_latest_model(self, model_id): :return: """ + commited_at = str(datetime.now()) + + self.model.insert_one( + { + "key": "models", + "model": model_id, + "session_id": session_id, + "committed_at": commited_at, + } + ) + self.model.update_one( {"key": "current_model"}, {"$set": {"model": model_id}}, True ) @@ -161,7 +172,7 @@ def set_latest_model(self, model_id): { "$push": { "model": model_id, - "committed_at": str(datetime.now()), + "committed_at": commited_at, } }, True, @@ -326,6 +337,42 @@ def get_helper(self): except (KeyError, IndexError): return None + def list_models(self, session_id=None, limit=None, skip=None): + """List all models in the statestore. + + :param session_id: The session id. + :type session_id: str + :param limit: The maximum number of models to return. + :type limit: int + :param skip: The number of models to skip. + :type skip: int + :return: List of models. + :rtype: list + """ + result = None + + find_option = ( + {"key": "models"} + if session_id is None + else {"key": "models", "session_id": session_id} + ) + + if limit is not None and skip is not None: + limit = int(limit) + skip = int(skip) + + result = self.model.find(find_option).limit(limit).skip(skip) + + else: + result = self.model.find(find_option) + + count = self.model.count_documents({}) + + return { + "result": result, + "count": count, + } + def get_model_trail(self): """Get the model trail. From 7b525e846ca66bd50374b8649ab275681e68575d Mon Sep 17 00:00:00 2001 From: Niklas Date: Wed, 18 Oct 2023 16:57:55 +0200 Subject: [PATCH 04/23] list models update --- fedn/fedn/network/api/interface.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index 40792da81..2c2588149 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -632,22 +632,30 @@ def get_latest_model(self): ) def get_models(self, session_id=None, limit=None, skip=None): - result = self.statestore.list_models(session_id, limit, skip) + models_object = self.statestore.list_models(session_id, limit, skip) - if result is None: + if models_object is None: return ( jsonify({"success": False, "message": "No models found."}), 404, ) - json_docs = [] - for doc in result["result"]: - json_doc = json.dumps(doc, default=json_util.default) - json_docs.append(json_doc) + payload = {} + for object in models_object["result"]: + model = object["model"] + session_id = object["session_id"] + commited_at = object["commited_at"] + + obj = { + "session_id": session_id, + "commited_at": commited_at, + "model": model, + } + payload[model] = obj - json_docs.reverse() + result = {"result": payload, "count": models_object["count"]} - return jsonify({"result": json_docs, "count": result["count"]}) + return jsonify(result) def get_model_trail(self): """Get the model trail for a given session. From ccd1a093a310119bb68c78594f792cea88e3425a Mon Sep 17 00:00:00 2001 From: Niklas Date: Thu, 19 Oct 2023 08:59:57 +0200 Subject: [PATCH 05/23] count only documents with key models --- fedn/fedn/network/statestore/mongostatestore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index 9262e9f13..02c1f760a 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -366,7 +366,7 @@ def list_models(self, session_id=None, limit=None, skip=None): else: result = self.model.find(find_option) - count = self.model.count_documents({}) + count = self.model.count_documents({"key": "models"}) return { "result": result, From e10594b72f8c33cf2b147799ddcd4762d85a6da6 Mon Sep 17 00:00:00 2001 From: Niklas Date: Thu, 19 Oct 2023 11:29:44 +0200 Subject: [PATCH 06/23] list models, correct count --- fedn/fedn/network/statestore/mongostatestore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index 02c1f760a..d2e454a12 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -366,7 +366,7 @@ def list_models(self, session_id=None, limit=None, skip=None): else: result = self.model.find(find_option) - count = self.model.count_documents({"key": "models"}) + count = self.model.count_documents(find_option) return { "result": result, From a8929b10d494dbe8b0f87cba1184581f4c5f3d69 Mon Sep 17 00:00:00 2001 From: Niklas Date: Fri, 20 Oct 2023 10:36:02 +0200 Subject: [PATCH 07/23] list models, sort on committed_at --- fedn/fedn/network/api/interface.py | 22 +++++---------- .../network/statestore/mongostatestore.py | 28 +++++++++++++++---- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index 2c2588149..5fa1cc1dd 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -632,28 +632,20 @@ def get_latest_model(self): ) def get_models(self, session_id=None, limit=None, skip=None): - models_object = self.statestore.list_models(session_id, limit, skip) + result = self.statestore.list_models(session_id, limit, skip) - if models_object is None: + if result is None: return ( jsonify({"success": False, "message": "No models found."}), 404, ) - payload = {} - for object in models_object["result"]: - model = object["model"] - session_id = object["session_id"] - commited_at = object["commited_at"] - - obj = { - "session_id": session_id, - "commited_at": commited_at, - "model": model, - } - payload[model] = obj + arr = [] + + for model in result["result"]: + arr.append(model) - result = {"result": payload, "count": models_object["count"]} + result = {"result": arr, "count": result["count"]} return jsonify(result) diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index d2e454a12..980801569 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -153,14 +153,14 @@ def set_latest_model(self, model_id, session_id=None): :return: """ - commited_at = str(datetime.now()) + committed_at = datetime.now() self.model.insert_one( { "key": "models", "model": model_id, "session_id": session_id, - "committed_at": commited_at, + "committed_at": committed_at, } ) @@ -172,7 +172,7 @@ def set_latest_model(self, model_id, session_id=None): { "$push": { "model": model_id, - "committed_at": commited_at, + "committed_at": str(committed_at), } }, True, @@ -337,7 +337,14 @@ def get_helper(self): except (KeyError, IndexError): return None - def list_models(self, session_id=None, limit=None, skip=None): + def list_models( + self, + session_id=None, + limit=None, + skip=None, + sort_key="committed_at", + sort_order=pymongo.DESCENDING, + ): """List all models in the statestore. :param session_id: The session id. @@ -357,14 +364,23 @@ def list_models(self, session_id=None, limit=None, skip=None): else {"key": "models", "session_id": session_id} ) + projection = {"_id": False, "key": False} + if limit is not None and skip is not None: limit = int(limit) skip = int(skip) - result = self.model.find(find_option).limit(limit).skip(skip) + result = ( + self.model.find(find_option, projection) + .limit(limit) + .skip(skip) + .sort(sort_key, sort_order) + ) else: - result = self.model.find(find_option) + result = self.model.find(find_option, projection).sort( + sort_key, sort_order + ) count = self.model.count_documents(find_option) From b99e593d35fe2eff107de6f75499aeda08efa50a Mon Sep 17 00:00:00 2001 From: Niklas Date: Fri, 20 Oct 2023 15:51:52 +0200 Subject: [PATCH 08/23] pagination added to get events --- fedn/fedn/network/api/interface.py | 20 +- fedn/fedn/network/api/server.py | 2 - fedn/fedn/network/dashboard/restservice.py | 871 +++++++++++------- .../network/dashboard/templates/events.html | 65 +- .../network/statestore/mongostatestore.py | 26 +- 5 files changed, 588 insertions(+), 396 deletions(-) diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index 5fa1cc1dd..e45d15919 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -1,11 +1,10 @@ import base64 import copy -import json + import os import threading from io import BytesIO -from bson import json_util from flask import jsonify, send_from_directory from werkzeug.utils import secure_filename @@ -380,19 +379,20 @@ def get_events(self, **kwargs): :return: The events as a json object. :rtype: :py:class:`flask.Response` """ - event_objects = self.statestore.get_events(**kwargs) - if event_objects is None: + response = self.statestore.get_events(**kwargs) + + result = response["result"] + if result is None: return ( jsonify({"success": False, "message": "No events found."}), 404, ) - json_docs = [] - for doc in self.statestore.get_events(**kwargs): - json_doc = json.dumps(doc, default=json_util.default) - json_docs.append(json_doc) - json_docs.reverse() - return jsonify({"events": json_docs}) + events = [] + for evt in result: + events.append(evt) + + return jsonify({"result": events, "count": response["count"]}) def get_all_validations(self, **kwargs): """Get all validations from the statestore. diff --git a/fedn/fedn/network/api/server.py b/fedn/fedn/network/api/server.py index 0c49343f8..43cf34024 100644 --- a/fedn/fedn/network/api/server.py +++ b/fedn/fedn/network/api/server.py @@ -298,8 +298,6 @@ def get_events(): # TODO: except filter with request.get_json() kwargs = request.args.to_dict() - print("get_events") - print(kwargs) return api.get_events(**kwargs) diff --git a/fedn/fedn/network/dashboard/restservice.py b/fedn/fedn/network/dashboard/restservice.py index 3c272349e..5dfde03c0 100644 --- a/fedn/fedn/network/dashboard/restservice.py +++ b/fedn/fedn/network/dashboard/restservice.py @@ -12,9 +12,19 @@ import pandas as pd from bokeh.embed import json_item from bson import json_util -from flask import (Flask, abort, flash, jsonify, make_response, redirect, - render_template, request, send_file, send_from_directory, - url_for) +from flask import ( + Flask, + abort, + flash, + jsonify, + make_response, + redirect, + render_template, + request, + send_file, + send_from_directory, + url_for, +) from werkzeug.utils import secure_filename from fedn.common.tracer.mongotracer import MongoTracer @@ -23,8 +33,8 @@ from fedn.network.state import ReducerState, ReducerStateToString from fedn.utils.checksum import sha -UPLOAD_FOLDER = '/app/client/package/' -ALLOWED_EXTENSIONS = {'gz', 'bz2', 'tar', 'zip', 'tgz'} +UPLOAD_FOLDER = "/app/client/package/" +ALLOWED_EXTENSIONS = {"gz", "bz2", "tar", "zip", "tgz"} def allowed_file(filename): @@ -33,8 +43,10 @@ def allowed_file(filename): :param filename: :return: """ - return '.' in filename and \ - filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS + return ( + "." in filename + and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS + ) def encode_auth_token(secret_key): @@ -43,16 +55,17 @@ def encode_auth_token(secret_key): """ try: payload = { - 'exp': datetime.datetime.utcnow() + datetime.timedelta(days=90, seconds=0), - 'iat': datetime.datetime.utcnow(), - 'status': 'Success' + "exp": datetime.datetime.utcnow() + + datetime.timedelta(days=90, seconds=0), + "iat": datetime.datetime.utcnow(), + "status": "Success", } - token = jwt.encode( - payload, - secret_key, - algorithm='HS256' + token = jwt.encode(payload, secret_key, algorithm="HS256") + print( + "\n\n\nSECURE MODE ENABLED, USE TOKEN TO ACCESS REDUCER: **** {} ****\n\n\n".format( + token + ) ) - print('\n\n\nSECURE MODE ENABLED, USE TOKEN TO ACCESS REDUCER: **** {} ****\n\n\n'.format(token)) return token except Exception as e: return e @@ -64,56 +77,49 @@ def decode_auth_token(auth_token, secret): :return: string """ try: - payload = jwt.decode( - auth_token, - secret, - algorithms=['HS256'] - ) + payload = jwt.decode(auth_token, secret, algorithms=["HS256"]) return payload["status"] except jwt.ExpiredSignatureError as e: print(e) - return 'Token has expired.' + return "Token has expired." except jwt.InvalidTokenError as e: print(e) - return 'Invalid token.' + return "Invalid token." class ReducerRestService: - """ - - """ + """ """ def __init__(self, config, control, statestore, certificate_manager): - print("config object!: \n\n\n\n{}".format(config)) - if config['host']: - self.host = config['host'] + if config["host"]: + self.host = config["host"] else: self.host = None - self.name = config['name'] + self.name = config["name"] - self.port = config['port'] - self.network_id = config['name'] + '-network' + self.port = config["port"] + self.network_id = config["name"] + "-network" - if 'token' in config.keys(): + if "token" in config.keys(): self.token_auth_enabled = True else: self.token_auth_enabled = False - if 'secret_key' in config.keys(): - self.SECRET_KEY = config['secret_key'] + if "secret_key" in config.keys(): + self.SECRET_KEY = config["secret_key"] else: self.SECRET_KEY = None - if 'use_ssl' in config.keys(): - self.use_ssl = config['use_ssl'] + if "use_ssl" in config.keys(): + self.use_ssl = config["use_ssl"] self.remote_compute_package = config["remote_compute_package"] if self.remote_compute_package: - self.package = 'remote' + self.package = "remote" else: - self.package = 'local' + self.package = "local" self.control = control self.statestore = statestore @@ -125,9 +131,7 @@ def to_dict(self): :return: """ - data = { - 'name': self.name - } + data = {"name": self.name} return data def check_compute_package(self): @@ -165,24 +169,40 @@ def check_configured_response(self): :rtype: json """ if self.control.state() == ReducerState.setup: - return jsonify({'status': 'retry', - 'package': self.package, - 'msg': "Controller is not configured."}) + return jsonify( + { + "status": "retry", + "package": self.package, + "msg": "Controller is not configured.", + } + ) if not self.check_compute_package(): - return jsonify({'status': 'retry', - 'package': self.package, - 'msg': "Compute package is not configured. Please upload the compute package."}) + return jsonify( + { + "status": "retry", + "package": self.package, + "msg": "Compute package is not configured. Please upload the compute package.", + } + ) if not self.check_initial_model(): - return jsonify({'status': 'retry', - 'package': self.package, - 'msg': "Initial model is not configured. Please upload the model."}) + return jsonify( + { + "status": "retry", + "package": self.package, + "msg": "Initial model is not configured. Please upload the model.", + } + ) if not self.control.idle(): - return jsonify({'status': 'retry', - 'package': self.package, - 'msg': "Conroller is not in idle state, try again later. "}) + return jsonify( + { + "status": "retry", + "package": self.package, + "msg": "Conroller is not in idle state, try again later. ", + } + ) return None def check_configured(self): @@ -192,17 +212,29 @@ def check_configured(self): :return: Rendered html template or None """ if not self.check_compute_package(): - return render_template('setup.html', client=self.name, state=ReducerStateToString(self.control.state()), - logs=None, refresh=False, - message='Please set the compute package') + return render_template( + "setup.html", + client=self.name, + state=ReducerStateToString(self.control.state()), + logs=None, + refresh=False, + message="Please set the compute package", + ) if self.control.state() == ReducerState.setup: - return render_template('setup.html', client=self.name, state=ReducerStateToString(self.control.state()), - logs=None, refresh=True, - message='Warning. Reducer is not base-configured. please do so with config file.') + return render_template( + "setup.html", + client=self.name, + state=ReducerStateToString(self.control.state()), + logs=None, + refresh=True, + message="Warning. Reducer is not base-configured. please do so with config file.", + ) if not self.check_initial_model(): - return render_template('setup_model.html', message="Please set the initial model.") + return render_template( + "setup_model.html", message="Please set the initial model." + ) return None @@ -216,31 +248,37 @@ def authorize(self, r, secret): """ try: # Get token - if 'Authorization' in r.headers: # header auth - request_token = r.headers.get('Authorization').split()[1] - elif 'token' in r.args: # args auth - request_token = str(r.args.get('token')) - elif 'fedn_token' in r.cookies: - request_token = r.cookies.get('fedn_token') + if "Authorization" in r.headers: # header auth + request_token = r.headers.get("Authorization").split()[1] + elif "token" in r.args: # args auth + request_token = str(r.args.get("token")) + elif "fedn_token" in r.cookies: + request_token = r.cookies.get("fedn_token") else: # no token provided - print('Authorization failed. No token provided.', flush=True) + print("Authorization failed. No token provided.", flush=True) abort(401) # Log token and secret print( - f'Secret: {secret}. Request token: {request_token}.', flush=True) + f"Secret: {secret}. Request token: {request_token}.", + flush=True, + ) # Authenticate status = decode_auth_token(request_token, secret) - if status == 'Success': + if status == "Success": return True else: - print('Authorization failed. Status: "{}"'.format( - status), flush=True) + print( + 'Authorization failed. Status: "{}"'.format(status), + flush=True, + ) abort(401) except Exception as e: - print('Authorization failed. Expection encountered: "{}".'.format( - e), flush=True) + print( + 'Authorization failed. Expection encountered: "{}".'.format(e), + flush=True, + ) abort(401) def run(self): @@ -250,10 +288,10 @@ def run(self): """ app = Flask(__name__) - app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER - app.config['SECRET_KEY'] = self.SECRET_KEY + app.config["UPLOAD_FOLDER"] = UPLOAD_FOLDER + app.config["SECRET_KEY"] = self.SECRET_KEY - @app.route('/') + @app.route("/") def index(): """ @@ -261,7 +299,7 @@ def index(): """ # Token auth if self.token_auth_enabled: - self.authorize(request, app.config.get('SECRET_KEY')) + self.authorize(request, app.config.get("SECRET_KEY")) # Render template not_configured_template = self.check_configured() @@ -269,29 +307,37 @@ def index(): template = not_configured_template else: events = self.control.get_events() - message = request.args.get('message', None) - message_type = request.args.get('message_type', None) - template = render_template('events.html', client=self.name, state=ReducerStateToString(self.control.state()), - events=events, - logs=None, refresh=True, configured=True, message=message, message_type=message_type) + message = request.args.get("message", None) + message_type = request.args.get("message_type", None) + template = render_template( + "events.html", + client=self.name, + state=ReducerStateToString(self.control.state()), + events=events, + logs=None, + refresh=True, + configured=True, + message=message, + message_type=message_type, + ) # Set token cookie in response if needed response = make_response(template) - if 'token' in request.args: # args auth - response.set_cookie('fedn_token', str(request.args['token'])) + if "token" in request.args: # args auth + response.set_cookie("fedn_token", str(request.args["token"])) # Return response return response - @app.route('/status') + @app.route("/status") def status(): """ :return: """ - return {'state': ReducerStateToString(self.control.state())} + return {"state": ReducerStateToString(self.control.state())} - @app.route('/netgraph') + @app.route("/netgraph") def netgraph(): """ Creates nodes and edges for network graph @@ -299,16 +345,18 @@ def netgraph(): :return: nodes and edges as keys :rtype: dict """ - result = {'nodes': [], 'edges': []} - - result['nodes'].append({ - "id": "reducer", - "label": "Reducer", - "role": 'reducer', - "status": 'active', - "name": 'reducer', # TODO: get real host name - "type": 'reducer', - }) + result = {"nodes": [], "edges": []} + + result["nodes"].append( + { + "id": "reducer", + "label": "Reducer", + "role": "reducer", + "status": "active", + "name": "reducer", # TODO: get real host name + "type": "reducer", + } + ) combiner_info = combiner_status() client_info = client_status() @@ -319,49 +367,55 @@ def netgraph(): for combiner in combiner_info: print("combiner info {}".format(combiner_info), flush=True) try: - result['nodes'].append({ - "id": combiner['name'], # "n{}".format(count), - "label": "Combiner ({} clients)".format(combiner['nr_active_clients']), - "role": 'combiner', - "status": 'active', # TODO: Hard-coded, combiner_info does not contain status - "name": combiner['name'], - "type": 'combiner', - }) + result["nodes"].append( + { + "id": combiner["name"], # "n{}".format(count), + "label": "Combiner ({} clients)".format( + combiner["nr_active_clients"] + ), + "role": "combiner", + "status": "active", # TODO: Hard-coded, combiner_info does not contain status + "name": combiner["name"], + "type": "combiner", + } + ) except Exception as err: print(err) - for client in client_info['active_clients']: + for client in client_info["active_clients"]: try: - if client['status'] != 'offline': - result['nodes'].append({ - "id": str(client['_id']), - "label": "Client", - "role": client['role'], - "status": client['status'], - "name": client['name'], - "combiner": client['combiner'], - "type": 'client', - }) + if client["status"] != "offline": + result["nodes"].append( + { + "id": str(client["_id"]), + "label": "Client", + "role": client["role"], + "status": client["status"], + "name": client["name"], + "combiner": client["combiner"], + "type": "client", + } + ) except Exception as err: print(err) count = 0 - for node in result['nodes']: + for node in result["nodes"]: try: - if node['type'] == 'combiner': - result['edges'].append( + if node["type"] == "combiner": + result["edges"].append( { "id": "e{}".format(count), - "source": node['id'], - "target": 'reducer', + "source": node["id"], + "target": "reducer", } ) - elif node['type'] == 'client': - result['edges'].append( + elif node["type"] == "client": + result["edges"].append( { "id": "e{}".format(count), - "source": node['combiner'], - "target": node['id'], + "source": node["combiner"], + "target": node["id"], } ) except Exception: @@ -369,59 +423,75 @@ def netgraph(): count = count + 1 return result - @app.route('/networkgraph') + @app.route("/networkgraph") def network_graph(): - try: plot = Plot(self.control.statestore) result = netgraph() - df_nodes = pd.DataFrame(result['nodes']) - df_edges = pd.DataFrame(result['edges']) + df_nodes = pd.DataFrame(result["nodes"]) + df_edges = pd.DataFrame(result["edges"]) graph = plot.make_netgraph_plot(df_edges, df_nodes) return json.dumps(json_item(graph, "myplot")) except Exception: raise # return '' - @app.route('/events') + @app.route("/events") def events(): """ :return: """ + response = self.control.get_events() + events = [] + + result = response["result"] + + for evt in result: + events.append(evt) + + return jsonify({"result": events, "count": response["count"]}) + json_docs = [] for doc in self.control.get_events(): json_doc = json.dumps(doc, default=json_util.default) json_docs.append(json_doc) json_docs.reverse() - return {'events': json_docs} - @app.route('/add') + return {"events": json_docs} + + @app.route("/add") def add(): - """ Add a combiner to the network. """ + """Add a combiner to the network.""" print("Adding combiner to network:", flush=True) if self.token_auth_enabled: - self.authorize(request, app.config.get('SECRET_KEY')) + self.authorize(request, app.config.get("SECRET_KEY")) if self.control.state() == ReducerState.setup: - return jsonify({'status': 'retry'}) - - name = request.args.get('name', None) - address = str(request.args.get('address', None)) - fqdn = str(request.args.get('fqdn', None)) - port = request.args.get('port', None) - secure_grpc = request.args.get('secure', None) - - if port is None or address is None or name is None or secure_grpc is None: + return jsonify({"status": "retry"}) + + name = request.args.get("name", None) + address = str(request.args.get("address", None)) + fqdn = str(request.args.get("fqdn", None)) + port = request.args.get("port", None) + secure_grpc = request.args.get("secure", None) + + if ( + port is None + or address is None + or name is None + or secure_grpc is None + ): return "Please specify correct parameters." # Try to retrieve combiner from db combiner = self.control.network.get_combiner(name) if not combiner: - if secure_grpc == 'True': + if secure_grpc == "True": certificate, key = self.certificate_manager.get_or_create( - address).get_keypair_raw() + address + ).get_keypair_raw() _ = base64.b64encode(certificate) _ = base64.b64encode(key) @@ -437,23 +507,24 @@ def add(): port=port, certificate=copy.deepcopy(certificate), key=copy.deepcopy(key), - ip=request.remote_addr) + ip=request.remote_addr, + ) self.control.network.add_combiner(combiner) combiner = self.control.network.get_combiner(name) ret = { - 'status': 'added', - 'storage': self.control.statestore.get_storage_backend(), - 'statestore': self.control.statestore.get_config(), - 'certificate': combiner.get_certificate(), - 'key': combiner.get_key() + "status": "added", + "storage": self.control.statestore.get_storage_backend(), + "statestore": self.control.statestore.get_config(), + "certificate": combiner.get_certificate(), + "key": combiner.get_key(), } return jsonify(ret) - @app.route('/eula', methods=['GET', 'POST']) + @app.route("/eula", methods=["GET", "POST"]) def eula(): """ @@ -462,9 +533,9 @@ def eula(): for r in request.headers: print("header contains: {}".format(r), flush=True) - return render_template('eula.html', configured=True) + return render_template("eula.html", configured=True) - @app.route('/models', methods=['GET', 'POST']) + @app.route("/models", methods=["GET", "POST"]) def models(): """ @@ -472,13 +543,12 @@ def models(): """ # Token auth if self.token_auth_enabled: - self.authorize(request, app.config.get('SECRET_KEY')) + self.authorize(request, app.config.get("SECRET_KEY")) - if request.method == 'POST': + if request.method == "POST": # upload seed file - uploaded_seed = request.files['seed'] + uploaded_seed = request.files["seed"] if uploaded_seed: - a = BytesIO() a.seek(0, 0) uploaded_seed.seek(0) @@ -504,23 +574,31 @@ def models(): h_latest_model_id = self.statestore.get_latest_model() model_info = self.control.get_model_info() - return render_template('models.html', box_plot=box_plot, metrics=valid_metrics, h_latest_model_id=h_latest_model_id, seed=True, - model_info=model_info, configured=True) + return render_template( + "models.html", + box_plot=box_plot, + metrics=valid_metrics, + h_latest_model_id=h_latest_model_id, + seed=True, + model_info=model_info, + configured=True, + ) seed = True - return redirect(url_for('models', seed=seed)) + return redirect(url_for("models", seed=seed)) - @app.route('/delete_model_trail', methods=['GET', 'POST']) + @app.route("/delete_model_trail", methods=["GET", "POST"]) def delete_model_trail(): """ :return: """ - if request.method == 'POST': - + if request.method == "POST": statestore_config = self.control.statestore.get_config() self.tracer = MongoTracer( - statestore_config['mongo_config'], statestore_config['network_id']) + statestore_config["mongo_config"], + statestore_config["network_id"], + ) try: self.control.drop_models() except Exception: @@ -528,28 +606,28 @@ def delete_model_trail(): # drop objects in minio self.control.delete_bucket_objects() - return redirect(url_for('models')) + return redirect(url_for("models")) seed = True - return redirect(url_for('models', seed=seed)) + return redirect(url_for("models", seed=seed)) - @app.route('/drop_control', methods=['GET', 'POST']) + @app.route("/drop_control", methods=["GET", "POST"]) def drop_control(): """ :return: """ - if request.method == 'POST': + if request.method == "POST": self.control.statestore.drop_control() - return redirect(url_for('control')) - return redirect(url_for('control')) + return redirect(url_for("control")) + return redirect(url_for("control")) # http://localhost:8090/control?rounds=4&model_id=879fa112-c861-4cb1-a25d-775153e5b548 - @app.route('/control', methods=['GET', 'POST']) + @app.route("/control", methods=["GET", "POST"]) def control(): - """ Main page for round control. Configure, start and stop training sessions. """ + """Main page for round control. Configure, start and stop training sessions.""" # Token auth if self.token_auth_enabled: - self.authorize(request, app.config.get('SECRET_KEY')) + self.authorize(request, app.config.get("SECRET_KEY")) not_configured = self.check_configured() if not_configured: @@ -560,60 +638,88 @@ def control(): if self.remote_compute_package: try: - self.current_compute_context = self.control.get_compute_package_name() + self.current_compute_context = ( + self.control.get_compute_package_name() + ) except Exception: self.current_compute_context = None else: self.current_compute_context = "None:Local" if self.control.state() == ReducerState.monitoring: return redirect( - url_for('index', state=state, refresh=refresh, message="Reducer is in monitoring state")) - - if request.method == 'POST': + url_for( + "index", + state=state, + refresh=refresh, + message="Reducer is in monitoring state", + ) + ) + + if request.method == "POST": # Get session configuration - round_timeout = float(request.form.get('timeout', 180)) - buffer_size = int(request.form.get('buffer_size', -1)) - rounds = int(request.form.get('rounds', 1)) - delete_models = request.form.get('delete_models', True) - task = (request.form.get('task', '')) - clients_required = request.form.get('clients_required', 1) - clients_requested = request.form.get('clients_requested', 8) + round_timeout = float(request.form.get("timeout", 180)) + buffer_size = int(request.form.get("buffer_size", -1)) + rounds = int(request.form.get("rounds", 1)) + delete_models = request.form.get("delete_models", True) + task = request.form.get("task", "") + clients_required = request.form.get("clients_required", 1) + clients_requested = request.form.get("clients_requested", 8) # checking if there are enough clients connected to start! clients_available = 0 for combiner in self.control.network.get_combiners(): try: combiner_state = combiner.report() - nac = combiner_state['nr_active_clients'] + nac = combiner_state["nr_active_clients"] clients_available = clients_available + int(nac) except Exception: pass if clients_available < clients_required: - return redirect(url_for('index', state=state, - message="Not enough clients available to start rounds! " - "check combiner client capacity", - message_type='warning')) + return redirect( + url_for( + "index", + state=state, + message="Not enough clients available to start rounds! " + "check combiner client capacity", + message_type="warning", + ) + ) - validate = request.form.get('validate', False) - if validate == 'False': + validate = request.form.get("validate", False) + if validate == "False": validate = False - helper_type = request.form.get('helper', 'keras') + helper_type = request.form.get("helper", "keras") # self.control.statestore.set_framework(helper_type) latest_model_id = self.statestore.get_latest_model() - config = {'round_timeout': round_timeout, 'buffer_size': buffer_size, - 'model_id': latest_model_id, 'rounds': rounds, 'delete_models_storage': delete_models, - 'clients_required': clients_required, - 'clients_requested': clients_requested, 'task': task, - 'validate': validate, 'helper_type': helper_type} - - threading.Thread(target=self.control.session, - args=(config,)).start() + config = { + "round_timeout": round_timeout, + "buffer_size": buffer_size, + "model_id": latest_model_id, + "rounds": rounds, + "delete_models_storage": delete_models, + "clients_required": clients_required, + "clients_requested": clients_requested, + "task": task, + "validate": validate, + "helper_type": helper_type, + } + + threading.Thread( + target=self.control.session, args=(config,) + ).start() - return redirect(url_for('index', state=state, refresh=refresh, message="Sent execution plan.", - message_type='SUCCESS')) + return redirect( + url_for( + "index", + state=state, + refresh=refresh, + message="Sent execution plan.", + message_type="SUCCESS", + ) + ) else: seed_model_id = None @@ -624,42 +730,53 @@ def control(): except Exception: pass - return render_template('index.html', latest_model_id=latest_model_id, - compute_package=self.current_compute_context, - seed_model_id=seed_model_id, - helper=self.control.statestore.get_helper(), validate=True, configured=True) - - @app.route('/assign') + return render_template( + "index.html", + latest_model_id=latest_model_id, + compute_package=self.current_compute_context, + seed_model_id=seed_model_id, + helper=self.control.statestore.get_helper(), + validate=True, + configured=True, + ) + + @app.route("/assign") def assign(): - """Handle client assignment requests. """ + """Handle client assignment requests.""" if self.token_auth_enabled: - self.authorize(request, app.config.get('SECRET_KEY')) + self.authorize(request, app.config.get("SECRET_KEY")) response = self.check_configured_response() if response: return response - name = request.args.get('name', None) - combiner_preferred = request.args.get('combiner', None) + name = request.args.get("name", None) + combiner_preferred = request.args.get("combiner", None) if combiner_preferred: - combiner = self.control.network.get_combiner(combiner_preferred) + combiner = self.control.network.get_combiner( + combiner_preferred + ) else: combiner = self.control.network.find_available_combiner() if combiner is None: - return jsonify({'status': 'retry', - 'package': self.package, - 'msg': "Failed to assign to a combiner, try again later."}) + return jsonify( + { + "status": "retry", + "package": self.package, + "msg": "Failed to assign to a combiner, try again later.", + } + ) client = { - 'name': name, - 'combiner_preferred': combiner_preferred, - 'combiner': combiner.name, - 'ip': request.remote_addr, - 'status': 'available' + "name": name, + "combiner_preferred": combiner_preferred, + "combiner": combiner.name, + "ip": request.remote_addr, + "status": "available", } # Add client to database @@ -668,25 +785,25 @@ def assign(): # Return connection information to client if combiner.certificate: cert_b64 = base64.b64encode(combiner.certificate) - cert = str(cert_b64).split('\'')[1] + cert = str(cert_b64).split("'")[1] else: cert = None response = { - 'status': 'assigned', - 'host': combiner.address, - 'fqdn': combiner.fqdn, - 'package': self.package, - 'ip': combiner.ip, - 'port': combiner.port, - 'certificate': cert, - 'model_type': self.control.statestore.get_helper() + "status": "assigned", + "host": combiner.address, + "fqdn": combiner.fqdn, + "package": self.package, + "ip": combiner.ip, + "port": combiner.port, + "certificate": cert, + "model_type": self.control.statestore.get_helper(), } return jsonify(response) def combiner_status(): - """ Get current status reports from all combiners registered in the network. + """Get current status reports from all combiners registered in the network. :return: """ @@ -711,67 +828,90 @@ def client_status(): all_active_validators = [] for client in combiner_info: - active_trainers_str = client['active_trainers'] - active_validators_str = client['active_validators'] + active_trainers_str = client["active_trainers"] + active_validators_str = client["active_validators"] active_trainers_str = re.sub( - '[^a-zA-Z0-9-:\n\.]', '', active_trainers_str).replace('name:', ' ') # noqa: W605 + "[^a-zA-Z0-9-:\n\.]", "", active_trainers_str + ).replace( + "name:", " " + ) # noqa: W605 active_validators_str = re.sub( - '[^a-zA-Z0-9-:\n\.]', '', active_validators_str).replace('name:', ' ') # noqa: W605 + "[^a-zA-Z0-9-:\n\.]", "", active_validators_str + ).replace( + "name:", " " + ) # noqa: W605 all_active_trainers.extend( - ' '.join(active_trainers_str.split(" ")).split()) + " ".join(active_trainers_str.split(" ")).split() + ) all_active_validators.extend( - ' '.join(active_validators_str.split(" ")).split()) + " ".join(active_validators_str.split(" ")).split() + ) active_trainers_list = [ - client for client in client_info if client['name'] in all_active_trainers] + client + for client in client_info + if client["name"] in all_active_trainers + ] active_validators_list = [ - cl for cl in client_info if cl['name'] in all_active_validators] + cl + for cl in client_info + if cl["name"] in all_active_validators + ] all_clients = [cl for cl in client_info] for client in all_clients: - status = 'offline' - role = 'None' + status = "offline" + role = "None" self.control.network.update_client_data( - client, status, role) + client, status, role + ) - all_active_clients = active_validators_list + active_trainers_list + all_active_clients = ( + active_validators_list + active_trainers_list + ) for client in all_active_clients: - status = 'active' - if client in active_trainers_list and client in active_validators_list: - role = 'trainer-validator' + status = "active" + if ( + client in active_trainers_list + and client in active_validators_list + ): + role = "trainer-validator" elif client in active_trainers_list: - role = 'trainer' + role = "trainer" elif client in active_validators_list: - role = 'validator' + role = "validator" else: - role = 'unknown' + role = "unknown" self.control.network.update_client_data( - client, status, role) - - return {'active_clients': all_clients, - 'active_trainers': active_trainers_list, - 'active_validators': active_validators_list - } + client, status, role + ) + + return { + "active_clients": all_clients, + "active_trainers": active_trainers_list, + "active_validators": active_validators_list, + } except Exception: pass - return {'active_clients': [], - 'active_trainers': [], - 'active_validators': [] - } + return { + "active_clients": [], + "active_trainers": [], + "active_validators": [], + } - @app.route('/metric_type', methods=['GET', 'POST']) + @app.route("/metric_type", methods=["GET", "POST"]) def change_features(): """ :return: """ - feature = request.args['selected'] + feature = request.args["selected"] plot = Plot(self.control.statestore) graphJSON = plot.create_box_plot(feature) return graphJSON - @app.route('/dashboard') + @app.route("/dashboard") def dashboard(): """ @@ -779,7 +919,7 @@ def dashboard(): """ # Token auth if self.token_auth_enabled: - self.authorize(request, app.config.get('SECRET_KEY')) + self.authorize(request, app.config.get("SECRET_KEY")) not_configured = self.check_configured() if not_configured: @@ -793,16 +933,18 @@ def dashboard(): clients_plot = plot.create_client_plot() client_histogram_plot = plot.create_client_histogram_plot() - return render_template('dashboard.html', show_plot=True, - table_plot=table_plot, - timeline_plot=timeline_plot, - clients_plot=clients_plot, - client_histogram_plot=client_histogram_plot, - combiners_plot=combiners_plot, - configured=True - ) - - @app.route('/network') + return render_template( + "dashboard.html", + show_plot=True, + table_plot=table_plot, + timeline_plot=timeline_plot, + clients_plot=clients_plot, + client_histogram_plot=client_histogram_plot, + combiners_plot=combiners_plot, + configured=True, + ) + + @app.route("/network") def network(): """ @@ -810,7 +952,7 @@ def network(): """ # Token auth if self.token_auth_enabled: - self.authorize(request, app.config.get('SECRET_KEY')) + self.authorize(request, app.config.get("SECRET_KEY")) not_configured = self.check_configured() if not_configured: @@ -821,17 +963,19 @@ def network(): combiner_info = combiner_status() active_clients = client_status() # print(combiner_info, flush=True) - return render_template('network.html', network_plot=True, - round_time_plot=round_time_plot, - mem_cpu_plot=mem_cpu_plot, - combiner_info=combiner_info, - active_clients=active_clients['active_clients'], - active_trainers=active_clients['active_trainers'], - active_validators=active_clients['active_validators'], - configured=True - ) - - @app.route('/config/download', methods=['GET']) + return render_template( + "network.html", + network_plot=True, + round_time_plot=round_time_plot, + mem_cpu_plot=mem_cpu_plot, + combiner_info=combiner_info, + active_clients=active_clients["active_clients"], + active_trainers=active_clients["active_trainers"], + active_validators=active_clients["active_validators"], + configured=True, + ) + + @app.route("/config/download", methods=["GET"]) def config_download(): """ @@ -839,8 +983,8 @@ def config_download(): """ chk_string = "" name = self.control.get_compute_package_name() - if name is None or name == '': - chk_string = '' + if name is None or name == "": + chk_string = "" else: file_path = os.path.join(UPLOAD_FOLDER, name) print("trying to get {}".format(file_path)) @@ -848,7 +992,7 @@ def config_download(): try: sum = str(sha(file_path)) except FileNotFoundError: - sum = '' + sum = "" chk_string = "checksum: {}".format(sum) network_id = self.network_id @@ -857,20 +1001,24 @@ def config_download(): ctx = """network_id: {network_id} discover_host: {discover_host} discover_port: {discover_port} -{chk_string}""".format(network_id=network_id, - discover_host=discover_host, - discover_port=discover_port, - chk_string=chk_string) +{chk_string}""".format( + network_id=network_id, + discover_host=discover_host, + discover_port=discover_port, + chk_string=chk_string, + ) obj = BytesIO() - obj.write(ctx.encode('UTF-8')) + obj.write(ctx.encode("UTF-8")) obj.seek(0) - return send_file(obj, - as_attachment=True, - download_name='client.yaml', - mimetype='application/x-yaml') - - @app.route('/context', methods=['GET', 'POST']) + return send_file( + obj, + as_attachment=True, + download_name="client.yaml", + mimetype="application/x-yaml", + ) + + @app.route("/context", methods=["GET", "POST"]) def context(): """ @@ -878,78 +1026,85 @@ def context(): """ # Token auth if self.token_auth_enabled: - self.authorize(request, app.config.get('SECRET_KEY')) + self.authorize(request, app.config.get("SECRET_KEY")) # if reset is not empty then allow context re-set - reset = request.args.get('reset', None) + reset = request.args.get("reset", None) if reset: - return render_template('context.html') + return render_template("context.html") - if request.method == 'POST': + if request.method == "POST": + if "file" not in request.files: + flash("No file part") + return redirect(url_for("context")) - if 'file' not in request.files: - flash('No file part') - return redirect(url_for('context')) - - file = request.files['file'] - helper_type = request.form.get('helper', 'kerashelper') + file = request.files["file"] + helper_type = request.form.get("helper", "kerashelper") # if user does not select file, browser also # submit an empty part without filename - if file.filename == '': - flash('No selected file') - return redirect(url_for('context')) + if file.filename == "": + flash("No selected file") + return redirect(url_for("context")) if file and allowed_file(file.filename): filename = secure_filename(file.filename) file_path = os.path.join( - app.config['UPLOAD_FOLDER'], filename) + app.config["UPLOAD_FOLDER"], filename + ) file.save(file_path) - if self.control.state() == ReducerState.instructing or self.control.state() == ReducerState.monitoring: + if ( + self.control.state() == ReducerState.instructing + or self.control.state() == ReducerState.monitoring + ): return "Not allowed to change context while execution is ongoing." self.control.set_compute_package(filename, file_path) self.control.statestore.set_helper(helper_type) - return redirect(url_for('control')) + return redirect(url_for("control")) - name = request.args.get('name', '') + name = request.args.get("name", "") - if name == '': + if name == "": name = self.control.get_compute_package_name() - if name is None or name == '': - return render_template('context.html') + if name is None or name == "": + return render_template("context.html") # There is a potential race condition here, if one client requests a package and at # the same time another one triggers a fetch from Minio and writes to disk. try: mutex = Lock() mutex.acquire() - return send_from_directory(app.config['UPLOAD_FOLDER'], name, as_attachment=True) + return send_from_directory( + app.config["UPLOAD_FOLDER"], name, as_attachment=True + ) except Exception: try: data = self.control.get_compute_package(name) - file_path = os.path.join(app.config['UPLOAD_FOLDER'], name) - with open(file_path, 'wb') as fh: + file_path = os.path.join(app.config["UPLOAD_FOLDER"], name) + with open(file_path, "wb") as fh: fh.write(data) - return send_from_directory(app.config['UPLOAD_FOLDER'], name, as_attachment=True) + return send_from_directory( + app.config["UPLOAD_FOLDER"], name, as_attachment=True + ) except Exception: raise finally: mutex.release() - return render_template('context.html') + return render_template("context.html") - @app.route('/checksum', methods=['GET', 'POST']) + @app.route("/checksum", methods=["GET", "POST"]) def checksum(): """ :return: """ # sum = '' - name = request.args.get('name', None) - if name == '' or name is None: + name = request.args.get("name", None) + if name == "" or name is None: name = self.control.get_compute_package_name() - if name is None or name == '': + if name is None or name == "": return jsonify({}) file_path = os.path.join(UPLOAD_FOLDER, name) @@ -958,13 +1113,13 @@ def checksum(): try: sum = str(sha(file_path)) except FileNotFoundError: - sum = '' + sum = "" - data = {'checksum': sum} + data = {"checksum": sum} return jsonify(data) - @app.route('/infer', methods=['POST']) + @app.route("/infer", methods=["POST"]) def infer(): """ @@ -972,7 +1127,7 @@ def infer(): """ # Token auth if self.token_auth_enabled: - self.authorize(request, app.config.get('SECRET_KEY')) + self.authorize(request, app.config.get("SECRET_KEY")) # Check configured not_configured = self.check_configured() @@ -982,7 +1137,9 @@ def infer(): # Check compute context if self.remote_compute_context: try: - self.current_compute_context = self.control.get_compute_package() + self.current_compute_context = ( + self.control.get_compute_package() + ) except Exception as e: print(e, flush=True) self.current_compute_context = None @@ -992,27 +1149,43 @@ def infer(): # Redirect if in monitoring state if self.control.state() == ReducerState.monitoring: return redirect( - url_for('index', state=ReducerStateToString(self.control.state()), refresh=True, message="Reducer is in monitoring state")) + url_for( + "index", + state=ReducerStateToString(self.control.state()), + refresh=True, + message="Reducer is in monitoring state", + ) + ) # POST params - timeout = int(request.form.get('timeout', 180)) - helper_type = request.form.get('helper', 'keras') - clients_required = request.form.get('clients_required', 1) - clients_requested = request.form.get('clients_requested', 8) + timeout = int(request.form.get("timeout", 180)) + helper_type = request.form.get("helper", "keras") + clients_required = request.form.get("clients_required", 1) + clients_requested = request.form.get("clients_requested", 8) # Start inference request - config = {'round_timeout': timeout, - 'model_id': self.statestore.get_latest_model(), - 'clients_required': clients_required, - 'clients_requested': clients_requested, - 'task': 'inference', - 'helper_type': helper_type} - threading.Thread(target=self.control.infer_instruct, - args=(config,)).start() + config = { + "round_timeout": timeout, + "model_id": self.statestore.get_latest_model(), + "clients_required": clients_required, + "clients_requested": clients_requested, + "task": "inference", + "helper_type": helper_type, + } + threading.Thread( + target=self.control.infer_instruct, args=(config,) + ).start() # Redirect - return redirect(url_for('index', state=ReducerStateToString(self.control.state()), refresh=True, message="Sent execution plan (inference).", - message_type='SUCCESS')) + return redirect( + url_for( + "index", + state=ReducerStateToString(self.control.state()), + refresh=True, + message="Sent execution plan (inference).", + message_type="SUCCESS", + ) + ) if not self.host: bind = "0.0.0.0" diff --git a/fedn/fedn/network/dashboard/templates/events.html b/fedn/fedn/network/dashboard/templates/events.html index d3c34beb5..1fb5fac74 100644 --- a/fedn/fedn/network/dashboard/templates/events.html +++ b/fedn/fedn/network/dashboard/templates/events.html @@ -3,41 +3,44 @@ {% block content %} -
-
-
Events
-
-
- - - - + + + -
- -
+ }); + +
+
+ -{% endblock %} +{% endblock %} \ No newline at end of file diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index 980801569..ae30a64c3 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -416,8 +416,16 @@ def get_events(self, **kwargs): :rtype: ObjectId """ # check if kwargs is empty + + result = None + count = None + projection = {"_id": False} + if not kwargs: - result = self.control.status.find() + result = self.control.status.find({}, projection).sort( + "timestamp", pymongo.DESCENDING + ) + count = self.control.status.count_documents({}) else: limit = kwargs.pop("limit", None) skip = kwargs.pop("skip", None) @@ -426,12 +434,22 @@ def get_events(self, **kwargs): limit = int(limit) skip = int(skip) result = ( - self.control.status.find(kwargs).limit(limit).skip(skip) + self.control.status.find(kwargs, projection) + .sort("timestamp", pymongo.DESCENDING) + .limit(limit) + .skip(skip) ) else: - result = self.control.status.find(kwargs) + result = self.control.status.find(kwargs, projection).sort( + "timestamp", pymongo.DESCENDING + ) - return result + count = self.control.status.count_documents(kwargs) + + return { + "result": result, + "count": count, + } def get_storage_backend(self): """Get the storage backend. From ae3d6913d5e88cff8bf29cbe292a6ba832390bfc Mon Sep 17 00:00:00 2001 From: Niklas Date: Mon, 23 Oct 2023 15:55:20 +0200 Subject: [PATCH 09/23] get plot data --- fedn/fedn/network/api/interface.py | 26 ++++++++++++++++++++++++++ fedn/fedn/network/api/server.py | 14 ++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index e45d15919..effcaa05d 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -7,6 +7,7 @@ from flask import jsonify, send_from_directory from werkzeug.utils import secure_filename +from fedn.network.dashboard.plots import Plot from fedn.common.config import get_controller_config, get_network_config from fedn.network.combiner.interfaces import ( @@ -733,6 +734,31 @@ def get_client_config(self, checksum=True): payload["checksum"] = checksum_str return jsonify(payload) + def get_plot_data(self, feature=None): + """Get plot data. + + :return: The plot data as json response. + :rtype: :py:class:`flask.Response` + """ + + plot = Plot(self.control.statestore) + + try: + valid_metrics = plot.fetch_valid_metrics() + feature = feature or valid_metrics[0] + box_plot = plot.create_box_plot(feature) + except Exception as e: + valid_metrics = None + box_plot = None + print(e, flush=True) + + result = { + "valid_metrics": valid_metrics, + "box_plot": box_plot, + } + + return jsonify(result) + def start_session( self, session_id, diff --git a/fedn/fedn/network/api/server.py b/fedn/fedn/network/api/server.py index 43cf34024..25826bbdd 100644 --- a/fedn/fedn/network/api/server.py +++ b/fedn/fedn/network/api/server.py @@ -343,6 +343,20 @@ def add_client(): return response +@app.route("/get_plot_data", methods=["GET"]) +def get_plot_data(): + """Get plot data from the statestore. + rtype: json + """ + + try: + feature = request.args.get("feature", None) + response = api.get_plot_data(feature=feature) + except TypeError as e: + return jsonify({"success": False, "message": str(e)}), 400 + return response + + if __name__ == "__main__": config = get_controller_config() port = config["port"] From 8212430aaa52ec884c370df8a1b8b05082bb133d Mon Sep 17 00:00:00 2001 From: Niklas Date: Thu, 26 Oct 2023 17:21:28 +0200 Subject: [PATCH 10/23] pagination for list_clients --- fedn/fedn/common/tracer/mongotracer.py | 13 ++++++ fedn/fedn/network/api/interface.py | 40 ++++++++++--------- fedn/fedn/network/api/server.py | 7 +++- fedn/fedn/network/combiner/server.py | 2 + .../network/statestore/mongostatestore.py | 24 ++++++++--- 5 files changed, 60 insertions(+), 26 deletions(-) diff --git a/fedn/fedn/common/tracer/mongotracer.py b/fedn/fedn/common/tracer/mongotracer.py index 92af569ea..99725c2d9 100644 --- a/fedn/fedn/common/tracer/mongotracer.py +++ b/fedn/fedn/common/tracer/mongotracer.py @@ -1,4 +1,5 @@ import uuid +from datetime import datetime from google.protobuf.json_format import MessageToDict @@ -18,6 +19,7 @@ def __init__(self, mongo_config, network_id): self.rounds = self.mdb['control.rounds'] self.sessions = self.mdb['control.sessions'] self.validations = self.mdb['control.validations'] + self.clients = self.mdb['network.clients'] except Exception as e: print("FAILED TO CONNECT TO MONGO, {}".format(e), flush=True) self.status = None @@ -82,3 +84,14 @@ def set_round_data(self, round_data): """ self.rounds.update_one({'round_id': str(round_data['round_id'])}, { '$push': {'reducer': round_data}}, True) + + + def update_client_timestamp(self, client_name): + datetime_now = datetime.now() + + filter_query = {"name": client_name} # Replace with the desired name + + # Define the update operation + update_query = {"$set": {"last_seen": datetime_now}} # Replace with the property and value to update + + self.clients.update_one(filter_query, update_query) \ No newline at end of file diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index effcaa05d..b4bdabc70 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -1,19 +1,16 @@ import base64 import copy - import os import threading from io import BytesIO from flask import jsonify, send_from_directory from werkzeug.utils import secure_filename -from fedn.network.dashboard.plots import Plot from fedn.common.config import get_controller_config, get_network_config -from fedn.network.combiner.interfaces import ( - CombinerInterface, - CombinerUnavailableError, -) +from fedn.network.combiner.interfaces import (CombinerInterface, + CombinerUnavailableError) +from fedn.network.dashboard.plots import Plot from fedn.network.state import ReducerState, ReducerStateToString from fedn.utils.checksum import sha @@ -66,27 +63,32 @@ def _allowed_file_extension( and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS ) - def get_all_clients(self): + def get_clients(self, limit=None, skip=None, active_only=False): """Get all clients from the statestore. :return: All clients as a json response. :rtype: :class:`flask.Response` """ # Will return list of ObjectId - clients_objects = self.statestore.list_clients() - payload = {} - for object in clients_objects: - id = object["name"] - info = { - "combiner": object["combiner"], - "combiner_preferred": object["combiner_preferred"], - "ip": object["ip"], - "updated_at": object["updated_at"], - "status": object["status"], + response = self.statestore.list_clients(limit, skip, active_only) + + arr = [] + + for element in response["result"]: + obj = { + "id": element["name"], + "combiner": element["combiner"], + "combiner_preferred": element["combiner_preferred"], + "ip": element["ip"], + "updated_at": element["updated_at"], + "last_seen": element["last_seen"], } - payload[id] = info - return jsonify(payload) + arr.append(obj) + + result = {"result": , "count": response["count"]} + + return jsonify(result) def get_active_clients(self, combiner_id): """Get all active clients, i.e that are assigned to a combiner. diff --git a/fedn/fedn/network/api/server.py b/fedn/fedn/network/api/server.py index 25826bbdd..cea8bedec 100644 --- a/fedn/fedn/network/api/server.py +++ b/fedn/fedn/network/api/server.py @@ -69,7 +69,12 @@ def list_clients(): return: All clients as a json object. rtype: json """ - return api.get_all_clients() + + limit = request.args.get("limit", None) + skip = request.args.get("skip", None) + active_only = request.args.get("active_only", None) + + return api.get_clients(limit, skip, active_only) @app.route("/get_active_clients", methods=["GET"]) diff --git a/fedn/fedn/network/combiner/server.py b/fedn/fedn/network/combiner/server.py index 11d874ea6..71c17b901 100644 --- a/fedn/fedn/network/combiner/server.py +++ b/fedn/fedn/network/combiner/server.py @@ -712,6 +712,8 @@ def ModelUpdateRequestStream(self, response, context): self._send_status(status) + self.tracer.update_client_timestamp(client.name) + while context.is_active(): try: yield q.get(timeout=1.0) diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index ae30a64c3..3bf23d8c3 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -584,17 +584,29 @@ def get_client(self, name): except Exception: return None - def list_clients(self): + def list_clients(self, limit=None, skip=None, active_only=False): """List all clients registered on the network. :return: list of clients. :rtype: list(ObjectId) """ - try: - ret = self.clients.find() - return list(ret) - except Exception: - return None + + result = None + count = None + + if limit is not None and skip is not None: + limit = int(limit) + skip = int(skip) + result = self.clients.find().limit(limit).skip(skip) + else: + result = self.clients.find() + + count = self.clients.count_documents({}) + + return { + "result": result, + "count": count, + } def update_client_status(self, client_data, status, role): """Set or update client status. From b4c21df83486f342fd6503c3cd17c6d8f81e2ee3 Mon Sep 17 00:00:00 2001 From: Niklas Date: Fri, 27 Oct 2023 15:02:27 +0200 Subject: [PATCH 11/23] clients: log when clients i online and not + api list clients sorted and by status --- fedn/fedn/common/tracer/mongotracer.py | 7 +++---- fedn/fedn/network/api/interface.py | 8 ++++---- fedn/fedn/network/api/server.py | 12 ++++-------- fedn/fedn/network/combiner/server.py | 7 ++++++- fedn/fedn/network/statestore/mongostatestore.py | 11 +++++++---- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/fedn/fedn/common/tracer/mongotracer.py b/fedn/fedn/common/tracer/mongotracer.py index 99725c2d9..fedeab3ca 100644 --- a/fedn/fedn/common/tracer/mongotracer.py +++ b/fedn/fedn/common/tracer/mongotracer.py @@ -86,12 +86,11 @@ def set_round_data(self, round_data): '$push': {'reducer': round_data}}, True) - def update_client_timestamp(self, client_name): + def update_client_status(self, client_name, status): datetime_now = datetime.now() - filter_query = {"name": client_name} # Replace with the desired name - + # Define the update operation - update_query = {"$set": {"last_seen": datetime_now}} # Replace with the property and value to update + update_query = {"$set": {"last_seen": datetime_now, "status": status}} # Replace with the property and value to update self.clients.update_one(filter_query, update_query) \ No newline at end of file diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index b4bdabc70..4da5bfcb3 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -63,14 +63,14 @@ def _allowed_file_extension( and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS ) - def get_clients(self, limit=None, skip=None, active_only=False): + def get_clients(self, limit=None, skip=None, status=False): """Get all clients from the statestore. :return: All clients as a json response. :rtype: :class:`flask.Response` """ # Will return list of ObjectId - response = self.statestore.list_clients(limit, skip, active_only) + response = self.statestore.list_clients(limit, skip, status) arr = [] @@ -80,13 +80,13 @@ def get_clients(self, limit=None, skip=None, active_only=False): "combiner": element["combiner"], "combiner_preferred": element["combiner_preferred"], "ip": element["ip"], - "updated_at": element["updated_at"], + "status": element["status"], "last_seen": element["last_seen"], } arr.append(obj) - result = {"result": , "count": response["count"]} + result = {"result": arr, "count": response["count"]} return jsonify(result) diff --git a/fedn/fedn/network/api/server.py b/fedn/fedn/network/api/server.py index cea8bedec..3edbaeee9 100644 --- a/fedn/fedn/network/api/server.py +++ b/fedn/fedn/network/api/server.py @@ -1,11 +1,7 @@ from flask import Flask, jsonify, request -from fedn.common.config import ( - get_controller_config, - get_modelstorage_config, - get_network_config, - get_statestore_config, -) +from fedn.common.config import (get_controller_config, get_modelstorage_config, + get_network_config, get_statestore_config) from fedn.network.api.interface import API from fedn.network.controller.control import Control from fedn.network.statestore.mongostatestore import MongoStateStore @@ -72,9 +68,9 @@ def list_clients(): limit = request.args.get("limit", None) skip = request.args.get("skip", None) - active_only = request.args.get("active_only", None) + status = request.args.get("status", None) - return api.get_clients(limit, skip, active_only) + return api.get_clients(limit, skip, status) @app.route("/get_active_clients", methods=["GET"]) diff --git a/fedn/fedn/network/combiner/server.py b/fedn/fedn/network/combiner/server.py index 71c17b901..cf116c053 100644 --- a/fedn/fedn/network/combiner/server.py +++ b/fedn/fedn/network/combiner/server.py @@ -98,9 +98,11 @@ def __init__(self, config): break if status == Status.UnAuthorized: print(response, flush=True) + print("Status.UnAuthorized", flush=True) sys.exit("Exiting: Unauthorized") if status == Status.UnMatchedConfig: print(response, flush=True) + print("Status.UnMatchedConfig", flush=True) sys.exit("Exiting: Missing config") cert = announce_config['certificate'] @@ -712,7 +714,7 @@ def ModelUpdateRequestStream(self, response, context): self._send_status(status) - self.tracer.update_client_timestamp(client.name) + self.tracer.update_client_status(client.name, "online") while context.is_active(): try: @@ -720,6 +722,9 @@ def ModelUpdateRequestStream(self, response, context): except queue.Empty: pass + self.tracer.update_client_status(client.name, "offline") + + def ModelValidationStream(self, update, context): """ Model validation stream RPC endpoint. Update status for client is connecting to stream. diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index 3bf23d8c3..38a582ccd 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -584,7 +584,7 @@ def get_client(self, name): except Exception: return None - def list_clients(self, limit=None, skip=None, active_only=False): + def list_clients(self, limit=None, skip=None, status=False, sort_key="last_seen", sort_order=pymongo.DESCENDING): """List all clients registered on the network. :return: list of clients. @@ -594,14 +594,17 @@ def list_clients(self, limit=None, skip=None, active_only=False): result = None count = None + find = {} if status is None else {"status": status} + projection = {"_id": False, "updated_at": False} + if limit is not None and skip is not None: limit = int(limit) skip = int(skip) - result = self.clients.find().limit(limit).skip(skip) + result = self.clients.find(find, projection).limit(limit).skip(skip).sort(sort_key, sort_order) else: - result = self.clients.find() + result = self.clients.find(find, projection).sort(sort_key, sort_order) - count = self.clients.count_documents({}) + count = self.clients.count_documents(find) return { "result": result, From 6f20a66e32ac4c3079388e01a6535481a81b88e9 Mon Sep 17 00:00:00 2001 From: Niklas Date: Fri, 27 Oct 2023 15:41:50 +0200 Subject: [PATCH 12/23] sessions sorting --- fedn/fedn/network/api/interface.py | 11 +++++------ fedn/fedn/network/statestore/mongostatestore.py | 10 +++++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index 4da5bfcb3..2ef24b178 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -173,13 +173,12 @@ def get_all_sessions(self, limit=None, skip=None): jsonify({"success": False, "message": "No sessions found."}), 404, ) - payload = {} - for object in sessions_object["result"]: - id = object["session_id"] - info = object["session_config"][0] - payload[id] = info + arr = [] + for element in sessions_object["result"]: + obj = element["session_config"][0] + arr.append(obj) - result = {"result": payload, "count": sessions_object["count"]} + result = {"result": arr, "count": sessions_object["count"]} return jsonify(result) diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index 38a582ccd..cfe3b75e1 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -111,7 +111,7 @@ def transition(self, state): ) ) - def get_sessions(self, limit=None, skip=None): + def get_sessions(self, limit=None, skip=None, sort_key="_id", sort_order=pymongo.DESCENDING): """Get all sessions. :return: All sessions. @@ -124,9 +124,13 @@ def get_sessions(self, limit=None, skip=None): limit = int(limit) skip = int(skip) - result = self.sessions.find().limit(limit).skip(skip) + result = self.sessions.find().limit(limit).skip(skip).sort( + sort_key, sort_order + ) else: - result = self.sessions.find() + result = self.sessions.find().sort( + sort_key, sort_order + ) count = self.sessions.count_documents({}) From 384a4fd738bf737019e31f86cdb02945f11f5d29 Mon Sep 17 00:00:00 2001 From: Niklas Date: Fri, 27 Oct 2023 17:39:22 +0200 Subject: [PATCH 13/23] group clients by combiner name and count them --- fedn/fedn/network/api/interface.py | 25 +++++++++++++ fedn/fedn/network/api/server.py | 16 +++++++++ .../network/statestore/mongostatestore.py | 36 +++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index 2ef24b178..6c999c82d 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -759,6 +759,31 @@ def get_plot_data(self, feature=None): } return jsonify(result) + + def list_combiners_data(self, limit=None, skip=None): + """Get combiners data. + """ + + response = self.statestore.list_combiners_data(limit, skip) + + arr = [] + + ## order list by combiner name + + for element in response["result"]: + + obj = { + "combiner": element["_id"], + "count": element["count"], + } + + arr.append(obj) + + arr.sort(key=lambda x: x["count"], reverse=True) + + result = {"result": arr, "count": response["count"]} + + return jsonify(result) def start_session( self, diff --git a/fedn/fedn/network/api/server.py b/fedn/fedn/network/api/server.py index 3edbaeee9..300ee1141 100644 --- a/fedn/fedn/network/api/server.py +++ b/fedn/fedn/network/api/server.py @@ -344,6 +344,22 @@ def add_client(): return response +@app.route("/list_combiners_data", methods=["GET"]) +def list_combiners_data(): + """Add a client to the network. + return: The response from control. + rtype: json + """ + limit = request.args.get("limit", None) + skip = request.args.get("skip", None) + + try: + response = api.list_combiners_data(limit=limit, skip=skip) + except TypeError as e: + return jsonify({"success": False, "message": str(e)}), 400 + return response + + @app.route("/get_plot_data", methods=["GET"]) def get_plot_data(): """Get plot data from the statestore. diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index cfe3b75e1..7c7b9eb7c 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -615,6 +615,42 @@ def list_clients(self, limit=None, skip=None, status=False, sort_key="last_seen" "count": count, } + def list_combiners_data(self, limit=None, skip=None): + """List all combiner data. + + :return: list of combiner data. + :rtype: list(ObjectId) + """ + + result = None + count = None + + pipeline = [ + {"$group": {"_id": "$combiner", "count": {"$sum": 1}}} + ] + + if limit is not None and skip is not None: + limit = int(limit) + skip = int(skip) + pipeline.append({"$limit": limit}) + pipeline.append({"$skip": skip}) + + result = self.clients.aggregate(pipeline) + + pipeline_count = [ + {"$group": {"_id": "$combiner"}}, + {"$group": {"_id": None, "count": {"$sum": 1}}} + ] + + result_count = list(self.clients.aggregate(pipeline_count)) + + count = result_count[0]['count'] + + return { + "result": result, + "count": count, + } + def update_client_status(self, client_data, status, role): """Set or update client status. From 371bbfbf5c3e2b1d86791b1b46d0388dfa8f414d Mon Sep 17 00:00:00 2001 From: Niklas Date: Mon, 30 Oct 2023 11:21:12 +0100 Subject: [PATCH 14/23] combiner (grouped clients) pagination --- fedn/fedn/common/tracer/mongotracer.py | 5 ++--- fedn/fedn/network/api/interface.py | 13 +++++-------- fedn/fedn/network/statestore/mongostatestore.py | 14 ++++++++------ 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/fedn/fedn/common/tracer/mongotracer.py b/fedn/fedn/common/tracer/mongotracer.py index fedeab3ca..76402fd29 100644 --- a/fedn/fedn/common/tracer/mongotracer.py +++ b/fedn/fedn/common/tracer/mongotracer.py @@ -85,12 +85,11 @@ def set_round_data(self, round_data): self.rounds.update_one({'round_id': str(round_data['round_id'])}, { '$push': {'reducer': round_data}}, True) - def update_client_status(self, client_name, status): datetime_now = datetime.now() filter_query = {"name": client_name} # Replace with the desired name - + # Define the update operation update_query = {"$set": {"last_seen": datetime_now, "status": status}} # Replace with the property and value to update - self.clients.update_one(filter_query, update_query) \ No newline at end of file + self.clients.update_one(filter_query, update_query) diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index 6c999c82d..8ba5e5e86 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -71,9 +71,9 @@ def get_clients(self, limit=None, skip=None, status=False): """ # Will return list of ObjectId response = self.statestore.list_clients(limit, skip, status) - + arr = [] - + for element in response["result"]: obj = { "id": element["name"], @@ -87,7 +87,7 @@ def get_clients(self, limit=None, skip=None, status=False): arr.append(obj) result = {"result": arr, "count": response["count"]} - + return jsonify(result) def get_active_clients(self, combiner_id): @@ -759,7 +759,7 @@ def get_plot_data(self, feature=None): } return jsonify(result) - + def list_combiners_data(self, limit=None, skip=None): """Get combiners data. """ @@ -768,8 +768,7 @@ def list_combiners_data(self, limit=None, skip=None): arr = [] - ## order list by combiner name - + # order list by combiner name for element in response["result"]: obj = { @@ -779,8 +778,6 @@ def list_combiners_data(self, limit=None, skip=None): arr.append(obj) - arr.sort(key=lambda x: x["count"], reverse=True) - result = {"result": arr, "count": response["count"]} return jsonify(result) diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index 7c7b9eb7c..bbafe36a3 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -607,7 +607,7 @@ def list_clients(self, limit=None, skip=None, status=False, sort_key="last_seen" result = self.clients.find(find, projection).limit(limit).skip(skip).sort(sort_key, sort_order) else: result = self.clients.find(find, projection).sort(sort_key, sort_order) - + count = self.clients.count_documents(find) return { @@ -615,7 +615,7 @@ def list_clients(self, limit=None, skip=None, status=False, sort_key="last_seen" "count": count, } - def list_combiners_data(self, limit=None, skip=None): + def list_combiners_data(self, limit=None, skip=None, sort_key="count", sort_order=pymongo.DESCENDING): """List all combiner data. :return: list of combiner data. @@ -624,19 +624,21 @@ def list_combiners_data(self, limit=None, skip=None): result = None count = None - + pipeline = [ - {"$group": {"_id": "$combiner", "count": {"$sum": 1}}} + {"$group": {"_id": "$combiner", "count": {"$sum": 1}}}, + {"$sort": {sort_key: sort_order, "_id": pymongo.ASCENDING}} ] if limit is not None and skip is not None: limit = int(limit) skip = int(skip) - pipeline.append({"$limit": limit}) + pipeline.append({"$skip": skip}) + pipeline.append({"$limit": limit}) result = self.clients.aggregate(pipeline) - + pipeline_count = [ {"$group": {"_id": "$combiner"}}, {"$group": {"_id": None, "count": {"$sum": 1}}} From 22b430818de3b367ecd3599f7aba21a68b06d189 Mon Sep 17 00:00:00 2001 From: Niklas Date: Mon, 30 Oct 2023 14:21:41 +0100 Subject: [PATCH 15/23] list combiners pagination fix + get combiners data now post, accepts list of combiner names --- fedn/fedn/network/api/interface.py | 34 ++++---- fedn/fedn/network/api/server.py | 19 +++-- .../network/statestore/mongostatestore.py | 85 +++++++++++-------- 3 files changed, 77 insertions(+), 61 deletions(-) diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index 8ba5e5e86..a6a8a5da8 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -114,28 +114,26 @@ def get_active_clients(self, combiner_id): response = combiner.list_active_clients() return response - def get_all_combiners(self): + def get_all_combiners(self, limit=None, skip=None): """Get all combiners from the statestore. :return: All combiners as a json response. :rtype: :class:`flask.Response` """ # Will return list of ObjectId - combiner_objects = self.statestore.get_combiners() - payload = {} - for object in combiner_objects: - id = object["name"] - info = { - "address": object["address"], - "fqdn": object["fqdn"], - "parent_reducer": object["parent"]["name"], - "port": object["port"], - "report": object["report"], - "updated_at": object["updated_at"], + response = self.statestore.get_combiners(limit, skip) + arr = [] + for element in response["result"]: + obj = { + "name": element["name"], + "updated_at": element["updated_at"], } - payload[id] = info - return jsonify(payload) + arr.append(obj) + + result = {"result": arr, "count": response["count"]} + + return jsonify(result) def get_combiner(self, combiner_id): """Get a combiner from the statestore. @@ -760,16 +758,16 @@ def get_plot_data(self, feature=None): return jsonify(result) - def list_combiners_data(self, limit=None, skip=None): + def list_combiners_data(self, combiners): """Get combiners data. """ - response = self.statestore.list_combiners_data(limit, skip) + response = self.statestore.list_combiners_data(combiners) arr = [] # order list by combiner name - for element in response["result"]: + for element in response: obj = { "combiner": element["_id"], @@ -778,7 +776,7 @@ def list_combiners_data(self, limit=None, skip=None): arr.append(obj) - result = {"result": arr, "count": response["count"]} + result = {"result": arr} return jsonify(result) diff --git a/fedn/fedn/network/api/server.py b/fedn/fedn/network/api/server.py index 300ee1141..cba87b059 100644 --- a/fedn/fedn/network/api/server.py +++ b/fedn/fedn/network/api/server.py @@ -69,7 +69,7 @@ def list_clients(): limit = request.args.get("limit", None) skip = request.args.get("skip", None) status = request.args.get("status", None) - + return api.get_clients(limit, skip, status) @@ -96,7 +96,11 @@ def list_combiners(): return: All combiners as a json object. rtype: json """ - return api.get_all_combiners() + + limit = request.args.get("limit", None) + skip = request.args.get("skip", None) + + return api.get_all_combiners(limit, skip) @app.route("/get_combiner", methods=["GET"]) @@ -344,17 +348,20 @@ def add_client(): return response -@app.route("/list_combiners_data", methods=["GET"]) +@app.route("/list_combiners_data", methods=["POST"]) def list_combiners_data(): """Add a client to the network. return: The response from control. rtype: json """ - limit = request.args.get("limit", None) - skip = request.args.get("skip", None) + + json_data = request.get_json() + + # expects a list of combiner names (strings) in an array + combiners = json_data.get("combiners", None) try: - response = api.list_combiners_data(limit=limit, skip=skip) + response = api.list_combiners_data(combiners) except TypeError as e: return jsonify({"success": False, "message": str(e)}), 400 return response diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index bbafe36a3..1ad56c60e 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -519,18 +519,36 @@ def get_combiner(self, name): except Exception: return None - def get_combiners(self): + def get_combiners(self, limit=None, skip=None, sort_key="updated_at", sort_order=pymongo.DESCENDING): """Get all combiners. :return: list of combiners. :rtype: list """ + + result = None + count = None + + projection = {"name": True, "updated_at": True} + try: - ret = self.combiners.find() - return list(ret) + if limit is not None and skip is not None: + limit = int(limit) + skip = int(skip) + result = self.combiners.find({}, projection).limit(limit).skip(skip).sort(sort_key, sort_order) + else: + result = self.combiners.find({}, projection).sort(sort_key, sort_order) + + count = self.combiners.count_documents({}) + except Exception: return None + return { + "result": result, + "count": count, + } + def set_combiner(self, combiner_data): """Set combiner in statestore. @@ -598,24 +616,28 @@ def list_clients(self, limit=None, skip=None, status=False, sort_key="last_seen" result = None count = None - find = {} if status is None else {"status": status} - projection = {"_id": False, "updated_at": False} + try: + find = {} if status is None else {"status": status} + projection = {"_id": False, "updated_at": False} - if limit is not None and skip is not None: - limit = int(limit) - skip = int(skip) - result = self.clients.find(find, projection).limit(limit).skip(skip).sort(sort_key, sort_order) - else: - result = self.clients.find(find, projection).sort(sort_key, sort_order) + if limit is not None and skip is not None: + limit = int(limit) + skip = int(skip) + result = self.clients.find(find, projection).limit(limit).skip(skip).sort(sort_key, sort_order) + else: + result = self.clients.find(find, projection).sort(sort_key, sort_order) + + count = self.clients.count_documents(find) - count = self.clients.count_documents(find) + except Exception as e: + print("ERROR: {}".format(e), flush=True) return { "result": result, "count": count, } - def list_combiners_data(self, limit=None, skip=None, sort_key="count", sort_order=pymongo.DESCENDING): + def list_combiners_data(self, combiners, sort_key="count", sort_order=pymongo.DESCENDING): """List all combiner data. :return: list of combiner data. @@ -623,35 +645,24 @@ def list_combiners_data(self, limit=None, skip=None, sort_key="count", sort_orde """ result = None - count = None - - pipeline = [ - {"$group": {"_id": "$combiner", "count": {"$sum": 1}}}, - {"$sort": {sort_key: sort_order, "_id": pymongo.ASCENDING}} - ] - - if limit is not None and skip is not None: - limit = int(limit) - skip = int(skip) - - pipeline.append({"$skip": skip}) - pipeline.append({"$limit": limit}) - result = self.clients.aggregate(pipeline) + try: - pipeline_count = [ - {"$group": {"_id": "$combiner"}}, - {"$group": {"_id": None, "count": {"$sum": 1}}} - ] + pipeline = [ + {"$match": {"combiner": {"$in": combiners}}}, + {"$group": {"_id": "$combiner", "count": {"$sum": 1}}}, + {"$sort": {sort_key: sort_order, "_id": pymongo.ASCENDING}} + ] if combiners is not None else [ + {"$group": {"_id": "$combiner", "count": {"$sum": 1}}}, + {"$sort": {sort_key: sort_order, "_id": pymongo.ASCENDING}} + ] - result_count = list(self.clients.aggregate(pipeline_count)) + result = self.clients.aggregate(pipeline) - count = result_count[0]['count'] + except Exception as e: + print("ERROR: {}".format(e), flush=True) - return { - "result": result, - "count": count, - } + return result def update_client_status(self, client_data, status, role): """Set or update client status. From 7b64d3e03da05d435dfae70b77558e6efa621e38 Mon Sep 17 00:00:00 2001 From: Fredrik Wrede Date: Mon, 30 Oct 2023 17:04:05 +0000 Subject: [PATCH 16/23] fix get_combiners --- fedn/fedn/network/api/interface.py | 8 +++++- fedn/fedn/network/api/network.py | 2 +- fedn/fedn/network/api/server.py | 2 +- .../network/statestore/mongostatestore.py | 26 +++++++++++++++---- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/fedn/fedn/network/api/interface.py b/fedn/fedn/network/api/interface.py index a6a8a5da8..3a20e3502 100644 --- a/fedn/fedn/network/api/interface.py +++ b/fedn/fedn/network/api/interface.py @@ -121,7 +121,8 @@ def get_all_combiners(self, limit=None, skip=None): :rtype: :class:`flask.Response` """ # Will return list of ObjectId - response = self.statestore.get_combiners(limit, skip) + projection = {"name": True, "updated_at": True} + response = self.statestore.get_combiners(limit, skip, projection=projection) arr = [] for element in response["result"]: obj = { @@ -760,6 +761,11 @@ def get_plot_data(self, feature=None): def list_combiners_data(self, combiners): """Get combiners data. + + :param combiners: The combiners to get data for. + :type combiners: list + :return: The combiners data as json response. + :rtype: :py:class:`flask.Response` """ response = self.statestore.list_combiners_data(combiners) diff --git a/fedn/fedn/network/api/network.py b/fedn/fedn/network/api/network.py index 26b366a76..6fcaad053 100644 --- a/fedn/fedn/network/api/network.py +++ b/fedn/fedn/network/api/network.py @@ -46,7 +46,7 @@ def get_combiners(self): """ data = self.statestore.get_combiners() combiners = [] - for c in data: + for c in data["result"]: if c['certificate']: cert = base64.b64decode(c['certificate']) key = base64.b64decode(c['key']) diff --git a/fedn/fedn/network/api/server.py b/fedn/fedn/network/api/server.py index cba87b059..cfb91bece 100644 --- a/fedn/fedn/network/api/server.py +++ b/fedn/fedn/network/api/server.py @@ -350,7 +350,7 @@ def add_client(): @app.route("/list_combiners_data", methods=["POST"]) def list_combiners_data(): - """Add a client to the network. + """List data from combiners. return: The response from control. rtype: json """ diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index 1ad56c60e..545ca9749 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -510,6 +510,8 @@ def get_reducer(self): def get_combiner(self, name): """Get combiner by name. + :param name: name of combiner to get. + :type name: str :return: The combiner. :rtype: ObjectId """ @@ -519,18 +521,26 @@ def get_combiner(self, name): except Exception: return None - def get_combiners(self, limit=None, skip=None, sort_key="updated_at", sort_order=pymongo.DESCENDING): + def get_combiners(self, limit=None, skip=None, sort_key="updated_at", sort_order=pymongo.DESCENDING, projection={}): """Get all combiners. - :return: list of combiners. - :rtype: list + :param limit: The maximum number of combiners to return. + :type limit: int + :param skip: The number of combiners to skip. + :type skip: int + :param sort_key: The key to sort by. + :type sort_key: str + :param sort_order: The sort order. + :type sort_order: pymongo.ASCENDING or pymongo.DESCENDING + :param projection: The projection. + :type projection: dict + :return: Dictionary of combiners in result and count. + :rtype: dict """ result = None count = None - projection = {"name": True, "updated_at": True} - try: if limit is not None and skip is not None: limit = int(limit) @@ -640,6 +650,12 @@ def list_clients(self, limit=None, skip=None, status=False, sort_key="last_seen" def list_combiners_data(self, combiners, sort_key="count", sort_order=pymongo.DESCENDING): """List all combiner data. + :param combiners: list of combiners to get data for. + :type combiners: list + :param sort_key: The key to sort by. + :type sort_key: str + :param sort_order: The sort order. + :type sort_order: pymongo.ASCENDING or pymongo.DESCENDING :return: list of combiner data. :rtype: list(ObjectId) """ From 74030a6d0e75dfa09673cbe9606444f3cc74a73a Mon Sep 17 00:00:00 2001 From: Niklas Date: Tue, 31 Oct 2023 12:07:38 +0100 Subject: [PATCH 17/23] active clients needs to be online --- fedn/fedn/network/statestore/mongostatestore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index 545ca9749..0047615e4 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -665,7 +665,7 @@ def list_combiners_data(self, combiners, sort_key="count", sort_order=pymongo.DE try: pipeline = [ - {"$match": {"combiner": {"$in": combiners}}}, + {"$match": {"combiner": {"$in": combiners}, "status": "online"}}, {"$group": {"_id": "$combiner", "count": {"$sum": 1}}}, {"$sort": {sort_key: sort_order, "_id": pymongo.ASCENDING}} ] if combiners is not None else [ From e3ec001d7b69e137b7506c59fe0f45d55be43bba Mon Sep 17 00:00:00 2001 From: Niklas Date: Tue, 31 Oct 2023 18:20:59 +0100 Subject: [PATCH 18/23] code checks --- fedn/fedn/network/dashboard/restservice.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/fedn/fedn/network/dashboard/restservice.py b/fedn/fedn/network/dashboard/restservice.py index 5dfde03c0..0ba31d3d1 100644 --- a/fedn/fedn/network/dashboard/restservice.py +++ b/fedn/fedn/network/dashboard/restservice.py @@ -12,19 +12,9 @@ import pandas as pd from bokeh.embed import json_item from bson import json_util -from flask import ( - Flask, - abort, - flash, - jsonify, - make_response, - redirect, - render_template, - request, - send_file, - send_from_directory, - url_for, -) +from flask import (Flask, abort, flash, jsonify, make_response, redirect, + render_template, request, send_file, send_from_directory, + url_for) from werkzeug.utils import secure_filename from fedn.common.tracer.mongotracer import MongoTracer From 6a6c89ccebf594a3878bf1f11eb56113b333cb0e Mon Sep 17 00:00:00 2001 From: Niklas Date: Tue, 31 Oct 2023 18:26:48 +0100 Subject: [PATCH 19/23] code checks --- fedn/fedn/network/combiner/server.py | 1 - fedn/fedn/network/dashboard/restservice.py | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/fedn/fedn/network/combiner/server.py b/fedn/fedn/network/combiner/server.py index cf116c053..7a9c87ff9 100644 --- a/fedn/fedn/network/combiner/server.py +++ b/fedn/fedn/network/combiner/server.py @@ -724,7 +724,6 @@ def ModelUpdateRequestStream(self, response, context): self.tracer.update_client_status(client.name, "offline") - def ModelValidationStream(self, update, context): """ Model validation stream RPC endpoint. Update status for client is connecting to stream. diff --git a/fedn/fedn/network/dashboard/restservice.py b/fedn/fedn/network/dashboard/restservice.py index 0ba31d3d1..14e7266bb 100644 --- a/fedn/fedn/network/dashboard/restservice.py +++ b/fedn/fedn/network/dashboard/restservice.py @@ -821,15 +821,15 @@ def client_status(): active_trainers_str = client["active_trainers"] active_validators_str = client["active_validators"] active_trainers_str = re.sub( - "[^a-zA-Z0-9-:\n\.]", "", active_trainers_str + "[^a-zA-Z0-9-:\n\.]", "", active_trainers_str # noqa: W605 ).replace( "name:", " " - ) # noqa: W605 + ) active_validators_str = re.sub( - "[^a-zA-Z0-9-:\n\.]", "", active_validators_str + "[^a-zA-Z0-9-:\n\.]", "", active_validators_str # noqa: W605 ).replace( "name:", " " - ) # noqa: W605 + ) all_active_trainers.extend( " ".join(active_trainers_str.split(" ")).split() ) From df3185917d39f28f0a3230b29a6448650e2ee607 Mon Sep 17 00:00:00 2001 From: Niklas Date: Tue, 31 Oct 2023 19:00:51 +0100 Subject: [PATCH 20/23] integration test fix --- fedn/fedn/common/tracer/mongotracer.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fedn/fedn/common/tracer/mongotracer.py b/fedn/fedn/common/tracer/mongotracer.py index 76402fd29..55711809b 100644 --- a/fedn/fedn/common/tracer/mongotracer.py +++ b/fedn/fedn/common/tracer/mongotracer.py @@ -87,9 +87,7 @@ def set_round_data(self, round_data): def update_client_status(self, client_name, status): datetime_now = datetime.now() - filter_query = {"name": client_name} # Replace with the desired name - - # Define the update operation - update_query = {"$set": {"last_seen": datetime_now, "status": status}} # Replace with the property and value to update + filter_query = {"name": client_name} + update_query = {"$set": {"last_seen": datetime_now, "status": status}} self.clients.update_one(filter_query, update_query) From 09af677f1ff181373d8dd291860ae3f015087789 Mon Sep 17 00:00:00 2001 From: Niklas Date: Tue, 31 Oct 2023 19:04:05 +0100 Subject: [PATCH 21/23] ingration test fix# --- .ci/tests/examples/wait_for.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.ci/tests/examples/wait_for.py b/.ci/tests/examples/wait_for.py index 7fa75506d..12f88331f 100644 --- a/.ci/tests/examples/wait_for.py +++ b/.ci/tests/examples/wait_for.py @@ -46,8 +46,11 @@ def _test_nodes(n_nodes, node_type, reducer_host='localhost', reducer_port='8090 return False if resp.status_code == 200: gr = json.loads(resp.content) + + status_key = "online" if node_type == "client" else "active" + n = sum(values.get('type') == node_type and values.get( - 'status') == 'active' for values in gr['nodes']) + 'status') == status_key for values in gr['nodes']) _eprint(f'Active {node_type}s: {n}.') return n == n_nodes _eprint(f'Reducer returned {resp.status_code}.') From 7195626c393c44bbefc5a8ae49d9d307a433bb7d Mon Sep 17 00:00:00 2001 From: Niklas Date: Wed, 1 Nov 2023 09:50:59 +0100 Subject: [PATCH 22/23] test_nodes update --- .ci/tests/examples/wait_for.py | 35 ++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/.ci/tests/examples/wait_for.py b/.ci/tests/examples/wait_for.py index 12f88331f..dc3345da0 100644 --- a/.ci/tests/examples/wait_for.py +++ b/.ci/tests/examples/wait_for.py @@ -37,24 +37,31 @@ def _test_rounds(n_rounds): return n == n_rounds -def _test_nodes(n_nodes, node_type, reducer_host='localhost', reducer_port='8090'): +def _test_nodes(n_nodes, node_type, reducer_host='localhost', reducer_port='8092'): try: - resp = requests.get( - f'http://{reducer_host}:{reducer_port}/netgraph', verify=False) + + endpoint = "list_clients" if node_type == "client" else "list_combiners" + + response = requests.get( + f'http://{reducer_host}:{reducer_port}/{endpoint}', verify=False) + + if response.status_code == 200: + + data = json.loads(response.content) + + count = 0 + if node_type == "client": + arr = data.get('result') + count = sum(element.get('status') == "online" for element in arr) + else: + count = data.get('count') + + _eprint(f'Active {node_type}s: {count}.') + return count == n_nodes + except Exception as e: _eprint(f'Reques exception econuntered: {e}.') return False - if resp.status_code == 200: - gr = json.loads(resp.content) - - status_key = "online" if node_type == "client" else "active" - - n = sum(values.get('type') == node_type and values.get( - 'status') == status_key for values in gr['nodes']) - _eprint(f'Active {node_type}s: {n}.') - return n == n_nodes - _eprint(f'Reducer returned {resp.status_code}.') - return False def rounds(n_rounds=3): From 52be847fd16a56fc57d088b248e09a7e32b5e35a Mon Sep 17 00:00:00 2001 From: Niklas Date: Wed, 1 Nov 2023 11:30:49 +0100 Subject: [PATCH 23/23] docstrings added --- fedn/fedn/common/tracer/mongotracer.py | 7 ++++++ .../network/statestore/mongostatestore.py | 22 ++++++++++++++----- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/fedn/fedn/common/tracer/mongotracer.py b/fedn/fedn/common/tracer/mongotracer.py index 55711809b..0a3e28cdc 100644 --- a/fedn/fedn/common/tracer/mongotracer.py +++ b/fedn/fedn/common/tracer/mongotracer.py @@ -86,6 +86,13 @@ def set_round_data(self, round_data): '$push': {'reducer': round_data}}, True) def update_client_status(self, client_name, status): + """ Update client status in statestore. + :param client_name: The client name + :type client_name: str + :param status: The client status + :type status: str + :return: None + """ datetime_now = datetime.now() filter_query = {"name": client_name} diff --git a/fedn/fedn/network/statestore/mongostatestore.py b/fedn/fedn/network/statestore/mongostatestore.py index 0047615e4..19d514f59 100644 --- a/fedn/fedn/network/statestore/mongostatestore.py +++ b/fedn/fedn/network/statestore/mongostatestore.py @@ -114,8 +114,15 @@ def transition(self, state): def get_sessions(self, limit=None, skip=None, sort_key="_id", sort_order=pymongo.DESCENDING): """Get all sessions. - :return: All sessions. - :rtype: ObjectID + :param limit: The maximum number of sessions to return. + :type limit: int + :param skip: The number of sessions to skip. + :type skip: int + :param sort_key: The key to sort by. + :type sort_key: str + :param sort_order: The sort order. + :type sort_order: pymongo.ASCENDING or pymongo.DESCENDING + :return: Dictionary of sessions in result (array of session objects) and count. """ result = None @@ -616,11 +623,16 @@ def get_client(self, name): except Exception: return None - def list_clients(self, limit=None, skip=None, status=False, sort_key="last_seen", sort_order=pymongo.DESCENDING): + def list_clients(self, limit=None, skip=None, status=None, sort_key="last_seen", sort_order=pymongo.DESCENDING): """List all clients registered on the network. - :return: list of clients. - :rtype: list(ObjectId) + :param limit: The maximum number of clients to return. + :type limit: int + :param skip: The number of clients to skip. + :type skip: int + :param status: online | offline + :type status: str + :param sort_key: The key to sort by. """ result = None