Lint code with flake8 (#407)

.ci/tests/examples/wait_for.py

@@ -1,5 +1,4 @@
 import json
-import ssl
 import sys
 from time import sleep
 
@@ -16,15 +15,14 @@ def _eprint(*args, **kwargs):
 
 
 def _retry(try_func, **func_args):
-    n = 0
     for _ in range(RETRIES):
         is_success = try_func(**func_args)
         if is_success:
-            _eprint(f'Sucess.')
+            _eprint('Sucess.')
             return True
         _eprint(f'Sleeping for {SLEEP}.')
         sleep(SLEEP)
-    _eprint(f'Giving up.')
+    _eprint('Giving up.')
     return False
 
 

.devcontainer/bin/init_venv.sh

@@ -11,5 +11,6 @@ python -m venv .venv
     sphinx_press_theme==0.8.0 \
     sphinx-autobuild==2021.3.14 \
     autopep8==1.5.7 \
-    isort==5.10.1
+    isort==5.10.1 \
+    flake8==4.0.1
 .venv/bin/pip install -e fedn

.github/workflows/code-checks.yaml

@@ -27,4 +27,18 @@ jobs:
           --exclude .mnist-pytorch
           .
 
+      - name: run Python linter
+        run: >
+          .venv/bin/flake8 . 
+          --exclude ".venv,.mnist-keras,.mnist-pytorch,fedn_pb2.py"
+
+      - name: check for floating imports
+        run: >
+          ! grep -E -R 
+          --exclude-dir='.venv' 
+          --exclude-dir='.mnist-pytorch' 
+          --exclude-dir='.mnist-keras' 
+          --exclude-dir='docs' 
+          '^[ \t]+(import|from) ' -I . 
+
       # TODO: add linting/formatting for all file types

.vscode/settings.json

@@ -3,4 +3,6 @@
     "editor.codeActionsOnSave": {
         "source.organizeImports": true
     },
+    "python.linting.enabled": true,
+    "python.linting.flake8Enabled": true,
 }

docs/source/conf.py

@@ -59,8 +59,8 @@
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
 #
-#html_theme = 'furo'
-#html_theme = 'sphinx_book_theme'
+# html_theme = 'furo'
+# html_theme = 'sphinx_book_theme'
 html_theme = 'press'
 
 

fedn/cli/__init__.py

@@ -1,3 +1,3 @@
-from .control_cmd import control_cmd
-from .main import main
-from .run_cmd import run_cmd
+from .control_cmd import control_cmd  # noqa: F401
+from .main import main  # noqa: F401
+from .run_cmd import run_cmd  # noqa: F401

fedn/cli/control_cmd.py

@@ -1,5 +1,10 @@
+import os
+from datetime import datetime
+
 import click
 
+from fedn.common.control.package import Package, PackageRuntime
+
 from .main import main
 
 
@@ -37,20 +42,17 @@ def package_cmd(ctx, reducer, port, token, name, upload, validate, cwd):
     :param cwd:
     """
     if not cwd:
-        import os
         cwd = os.getcwd()
 
     print("CONTROL: Bundling {} dir for distribution. Please wait for operation to complete..".format(cwd))
 
     if not name:
-        from datetime import datetime
         name = str(os.path.basename(cwd)) + '-' + \
             datetime.today().strftime('%Y-%m-%d-%H%M%S')
 
     config = {'host': reducer, 'port': port, 'token': token, 'name': name,
               'cwd': cwd}
 
-    from fedn.common.control.package import Package
     package = Package(config)
 
     print("CONTROL: Bundling package..")
@@ -85,14 +87,12 @@ def unpack_cmd(ctx, reducer, port, token, name, download, validate, cwd):
     :param validate:
     :param cwd:
     """
-    import os
     if not cwd:
         cwd = os.getcwd()
 
-    config = {'host': reducer, 'port': port, 'token': token, 'name': name,
-              'cwd': cwd}
+    # config = {'host': reducer, 'port': port, 'token': token, 'name': name,
+    #           'cwd': cwd}
 
-    from fedn.common.control.package import PackageRuntime
     package = PackageRuntime(cwd, os.path.join(cwd, 'client'))
     package.download(reducer, port, token)
     package.unpack()

fedn/cli/run_cmd.py

@@ -4,8 +4,13 @@
 import click
 import yaml
 
+from fedn.client import Client
 from fedn.clients.reducer.restservice import (decode_auth_token,
                                               encode_auth_token)
+from fedn.clients.reducer.statestore.mongoreducerstatestore import \
+    MongoReducerStateStore
+from fedn.combiner import Combiner
+from fedn.reducer import Reducer
 
 from .main import main
 
@@ -100,7 +105,7 @@ def client_cmd(ctx, discoverhost, discoverport, token, name, client_id, local_pa
         with open(config['init'], 'r') as file:
             try:
                 settings = dict(yaml.safe_load(file))
-            except Exception as e:
+            except Exception:
                 print('Failed to read config from settings file, exiting.', flush=True)
                 return
                 # raise(e)
@@ -119,11 +124,10 @@ def client_cmd(ctx, discoverhost, discoverport, token, name, client_id, local_pa
             print(
                 "Missing required configuration: discover_host, discover_port", flush=True)
             return
-    except Exception as e:
+    except Exception:
         print("Could not load config appropriately. Check config", flush=True)
         return
 
-    from fedn.client import Client
     client = Client(config)
     client.run()
 
@@ -160,8 +164,8 @@ def reducer_cmd(ctx, discoverhost, discoverport, secret_key, local_package, name
         print(e, flush=True)
         exit(-1)
 
-    if not remote:
-        helper = check_helper_config_file(fedn_config)
+    # if not remote:
+    #     helper = check_helper_config_file(fedn_config)
 
     try:
         network_id = fedn_config['network_id']
@@ -171,8 +175,7 @@ def reducer_cmd(ctx, discoverhost, discoverport, secret_key, local_package, name
 
     statestore_config = fedn_config['statestore']
     if statestore_config['type'] == 'MongoDB':
-        from fedn.clients.reducer.statestore.mongoreducerstatestore import \
-            MongoReducerStateStore
+
         statestore = MongoReducerStateStore(
             network_id, statestore_config['mongo_config'], defaults=config['init'])
     else:
@@ -190,7 +193,7 @@ def reducer_cmd(ctx, discoverhost, discoverport, secret_key, local_package, name
                 if status != 'Success':
                     token = encode_auth_token(config['secret_key'])
                     config['token'] = token
-            except:
+            except Exception:
                 raise
 
         else:
@@ -199,7 +202,7 @@ def reducer_cmd(ctx, discoverhost, discoverport, secret_key, local_package, name
 
     try:
         statestore.set_reducer(config)
-    except:
+    except Exception:
         print("Failed to set reducer config in statestore, exiting.", flush=True)
         exit(-1)
 
@@ -208,19 +211,18 @@ def reducer_cmd(ctx, discoverhost, discoverport, secret_key, local_package, name
     except KeyError:
         print("storage configuration missing in statestore_config.", flush=True)
         exit(-1)
-    except:
+    except Exception:
         print("Failed to set storage config in statestore, exiting.", flush=True)
         exit(-1)
 
     # Control config
     control_config = fedn_config['control']
     try:
         statestore.set_round_config(control_config)
-    except:
+    except Exception:
         print("Failed to set control config, exiting.", flush=True)
         exit(-1)
 
-    from fedn.reducer import Reducer
     reducer = Reducer(statestore)
     reducer.run()
 
@@ -274,6 +276,5 @@ def combiner_cmd(ctx, discoverhost, discoverport, token, name, hostname, port, s
             config['myport'] = combiner_config['port']
             config['max_clients'] = combiner_config['max_clients']
 
-    from fedn.combiner import Combiner
     combiner = Combiner(config)
     combiner.run()

fedn/cli/tests/tests.py

@@ -1,8 +1,5 @@
 import unittest
-from unittest.mock import MagicMock, patch
-from uuid import UUID
 
-import yaml
 from click.testing import CliRunner
 from run_cmd import check_helper_config_file
 
@@ -68,7 +65,7 @@ def test_check_helper_config_file(self):
         del COPY_INIT_FILE["control"]["helper"]
 
         with self.assertRaises(SystemExit):
-            helper = check_helper_config_file(COPY_INIT_FILE)
+            check_helper_config_file(COPY_INIT_FILE)
 
 
 if __name__ == '__main__':

fedn/fedn/aggregators/aggregator.py

@@ -1,6 +1,3 @@
-import collections
-import os
-import tempfile
 from abc import ABC, abstractmethod
 
 

fedn/fedn/aggregators/fedavg.py

@@ -1,25 +1,20 @@
 import json
-import os
 import queue
-import sys
-import tempfile
 import time
-import uuid
 
 import fedn.common.net.grpc.fedn_pb2 as fedn
 from fedn.aggregators.aggregator import AggregatorBase
-from fedn.utils.helpers import get_helper
 
 
 class FedAvgAggregator(AggregatorBase):
-    """ Local SGD / Federated Averaging (FedAvg) aggregator. 
+    """ Local SGD / Federated Averaging (FedAvg) aggregator.
 
-    :param id: A reference to id of :class: `fedn.combiner.Combiner` 
+    :param id: A reference to id of :class: `fedn.combiner.Combiner`
     :type id: str
-    :param storage: Model repository for :class: `fedn.combiner.Combiner` 
+    :param storage: Model repository for :class: `fedn.combiner.Combiner`
     :type storage: class: `fedn.common.storage.s3.s3repo.S3ModelRepository`
     :param server: A handle to the Combiner class :class: `fedn.combiner.Combiner`
-    :type server: class: `fedn.combiner.Combiner` 
+    :type server: class: `fedn.combiner.Combiner`
     :param modelservice: A handle to the model service :class: `fedn.clients.combiner.modelservice.ModelService`
     :type modelservice: class: `fedn.clients.combiner.modelservice.ModelService`
     :param control: A handle to the :class: `fedn.clients.combiner.roundcontrol.RoundControl`
@@ -40,7 +35,7 @@ def __init__(self, id, storage, server, modelservice, control):
     def on_model_update(self, model_id):
         """Callback when a new model update is recieved from a client.
             Performs (optional) pre-processing and the puts the update id
-            on the aggregation queue. 
+            on the aggregation queue.
 
         :param model_id: ID of model update
         :type model_id: str
@@ -57,9 +52,9 @@ def on_model_update(self, model_id):
             pass
 
     def on_model_validation(self, validation):
-        """ Callback when a new model validation is recieved from a client. 
+        """ Callback when a new model validation is recieved from a client.
 
-        :param validation: Dict containing validation data sent by client. 
+        :param validation: Dict containing validation data sent by client.
                            Must be valid JSON.
         :type validation: dict
         """