Add flake8 to ci and fix flake8 findings

.circleci/config.yml

@@ -1,10 +1,23 @@
 version: 2
 jobs:
-  build:
+  lint:
     docker:
-      - image: circleci/python:3.5-stretch
+      - image: circleci/python:3.5
     steps:
       - checkout
+      - run:
+          name: Install test requirements and run lint
+          command: |
+            virtualenv .venv
+            source .venv/bin/activate
+            pip install --require-hashes -r test-requirements.txt
+            make lint
+      - run:
+          name: Check Python dependencies for CVEs
+          command: |
+            set -e
+            source .venv/bin/activate
+            make safety
 
   test:
     docker:
@@ -22,14 +35,10 @@ jobs:
             source .venv/bin/activate
             pip install --require-hashes -r test-requirements.txt
             make test
-      - run:
-          name: Check Python dependencies for CVEs
-          command: |
-            set -e
-            source .venv/bin/activate
-            make safety
 workflows:
   version: 2
   securedrop_export_ci:
     jobs:
+      - lint
       - test
+

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 99

Makefile

@@ -16,6 +16,10 @@ update-pip-requirements: ## Updates all Python requirements files via pip-compil
 test:
 	pytest -v tests/
 
+.PHONY: lint
+lint:
+	flake8 securedrop_export/ tests/
+
 # Explaination of the below shell command should it ever break.
 # 1. Set the field separator to ": ##" and any make targets that might appear between : and ##
 # 2. Use sed-like syntax to remove the make targets

securedrop_export/entrypoint.py

@@ -11,6 +11,7 @@
 CONFIG_PATH = "/etc/sd-export-config.json"
 DEFAULT_HOME = os.path.join(os.path.expanduser("~"), ".securedrop_export")
 
+
 def configure_logging():
     """
     All logging related settings are set up by this function.
@@ -35,12 +36,13 @@ def configure_logging():
     log.setLevel(logging.DEBUG)
     log.addHandler(handler)
 
+
 def start():
     try:
         configure_logging()
     except Exception:
         msg = "ERROR_LOGGING"
-        my_sub.exit_gracefully(msg)
+        export.SDExport.exit_gracefully(msg)
 
     logging.info('Starting SecureDrop Export {}'.format(__version__))
     my_sub = export.SDExport(sys.argv[1], CONFIG_PATH)

securedrop_export/export.py

@@ -22,6 +22,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 class Metadata(object):
     """
     Object to parse, validate and store json metadata from the sd-export archive.
@@ -46,8 +47,14 @@ def __init__(self, archive_path):
                 json_config = json.loads(f.read())
                 self.export_method = json_config.get("device", None)
                 self.encryption_method = json_config.get("encryption_method", None)
-                self.encryption_key = json_config.get("encryption_key", None)
-                logging.info('Exporting to device {} with encryption_method {}'.format(self.export_method, self.encryption_method))
+                self.encryption_key = json_config.get(
+                    "encryption_key", None
+                )
+                logging.info(
+                    'Exporting to device {} with encryption_method {}'.format(
+                        self.export_method, self.encryption_method
+                    )
+                )
 
         except Exception:
             logging.error('Metadata parsing failure')
@@ -56,12 +63,20 @@ def __init__(self, archive_path):
     def is_valid(self):
         logging.info('Validating metadata contents')
         if self.export_method not in self.SUPPORTED_EXPORT_METHODS:
-            logging.error('Archive metadata: Export method {} is not supported'.format(self.export_method))
+            logging.error(
+                'Archive metadata: Export method {} is not supported'.format(
+                    self.export_method
+                )
+            )
             return False
 
         if self.export_method == "disk":
             if self.encryption_method not in self.SUPPORTED_ENCRYPTION_METHODS:
-                logging.error('Archive metadata: Encryption method {} is not supported'.format(self.encryption_method))
+                logging.error(
+                    'Archive metadata: Encryption method {} is not supported'.format(
+                        self.encryption_method
+                    )
+                )
                 return False
         return True
 
@@ -180,7 +195,7 @@ def check_luks_volume(self):
         try:
             # cryptsetup isLuks returns 0 if the device is a luks volume
             # subprocess with throw if the device is not luks (rc !=0)
-            p = subprocess.check_call(["sudo", "cryptsetup", "isLuks", DEVICE])
+            subprocess.check_call(["sudo", "cryptsetup", "isLuks", DEVICE])
             msg = "USB_ENCRYPTED"
             self.exit_gracefully(msg)
         except subprocess.CalledProcessError:
@@ -195,13 +210,13 @@ def unlock_luks_volume(self, encryption_key):
                 ["sudo", "cryptsetup", "luksOpen", self.device, self.encrypted_device],
                 stdin=subprocess.PIPE,
                 stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
+                stderr=subprocess.PIPE
             )
             logging.info('Passing key')
             p.communicate(input=str.encode(encryption_key, "utf-8"))
             rc = p.returncode
             if rc != 0:
-                logging.error('Bad phassphrase for {}',format(self.encrypted_device))
+                logging.error('Bad phassphrase for {}'.format(self.encrypted_device))
                 msg = "USB_BAD_PASSPHRASE"
                 self.exit_gracefully(msg)
 
@@ -388,10 +403,10 @@ def is_open_office_file(self, filename):
 
     def print_file(self, file_to_print):
         try:
-            # if the file to print is an (open)office document, we need to call unoconf to convert
-            # the file to pdf as printer drivers do not immediately support this format out of the box
+            # If the file to print is an (open)office document, we need to call unoconf to
+            # convert the file to pdf as printer drivers do not support this format
             if self.is_open_office_file(file_to_print):
-                logging.info('Converting Office document to pdf for printing'.format(self.printer_name))
+                logging.info('Converting Office document to pdf'.format(self.printer_name))
                 folder = os.path.dirname(file_to_print)
                 converted_filename = file_to_print + ".pdf"
                 converted_path = os.path.join(folder, converted_filename)
@@ -405,7 +420,7 @@ def print_file(self, file_to_print):
             self.exit_gracefully(msg)
 
 
-## class ends here
+# class ends here
 class TimeoutException(Exception):
     pass
 

securedrop_export/main.py

@@ -4,6 +4,7 @@
 
 logger = logging.getLogger(__name__)
 
+
 def __main__(submission):
     submission.extract_tarball()
 

test-requirements.in

@@ -1 +1,2 @@
+flake8
 pytest

test-requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --generate-hashes --output-file test-requirements.txt test-requirements.in
+#    pip-compile --generate-hashes --output-file=test-requirements.txt test-requirements.in
 #
 atomicwrites==1.3.0 \
     --hash=sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4 \
@@ -12,22 +12,21 @@ attrs==19.1.0 \
     --hash=sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79 \
     --hash=sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399 \
     # via pytest
-configparser==3.7.4 \
-    --hash=sha256:8be81d89d6e7b4c0d4e44bcc525845f6da25821de80cb5e06e7e0238a2899e32 \
-    --hash=sha256:da60d0014fd8c55eb48c1c5354352e363e2d30bbf7057e5e171a468390184c75 \
-    # via importlib-metadata
-contextlib2==0.5.5 \
-    --hash=sha256:509f9419ee91cdd00ba34443217d5ca51f5a364a404e1dce9e8979cea969ca48 \
-    --hash=sha256:f5260a6e679d2ff42ec91ec5252f4eeffdcf21053db9113bd0a8e4d953769c00 \
-    # via importlib-metadata
-funcsigs==1.0.2 \
-    --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
-    --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50 \
-    # via pytest
+entrypoints==0.3 \
+    --hash=sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19 \
+    --hash=sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451 \
+    # via flake8
+flake8==3.7.8 \
+    --hash=sha256:19241c1cbc971b9962473e4438a2ca19749a7dd002dd1a946eaba171b4114548 \
+    --hash=sha256:8e9dfa3cecb2400b3738a42c54c3043e821682b9c840b0448c0503f781130696
 importlib-metadata==0.18 \
     --hash=sha256:6dfd58dfe281e8d240937776065dd3624ad5469c835248219bd16cf2e12dbeb7 \
     --hash=sha256:cb6ee23b46173539939964df59d3d72c3e0c1b5d54b84f1d8a7e912fe43612db \
     # via pluggy, pytest
+mccabe==0.6.1 \
+    --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
+    --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f \
+    # via flake8
 more-itertools==5.0.0 \
     --hash=sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4 \
     --hash=sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc \
@@ -40,7 +39,7 @@ packaging==19.0 \
 pathlib2==2.3.4 \
     --hash=sha256:2156525d6576d21c4dcaddfa427fae887ef89a7a9de5cbfe0728b3aafa78427e \
     --hash=sha256:446014523bb9be5c28128c4d2a10ad6bb60769e78bd85658fe44a450674e0ef8 \
-    # via importlib-metadata, pytest
+    # via pytest
 pluggy==0.12.0 \
     --hash=sha256:0825a152ac059776623854c1543d65a4ad408eb3d33ee114dff91e57ec6ae6fc \
     --hash=sha256:b9817417e95936bf75d85d3f8767f7df6cdde751fc40aed3bb3074cbcb77757c \
@@ -49,25 +48,21 @@ py==1.8.0 \
     --hash=sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa \
     --hash=sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53 \
     # via pytest
+pycodestyle==2.5.0 \
+    --hash=sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56 \
+    --hash=sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c \
+    # via flake8
+pyflakes==2.1.1 \
+    --hash=sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0 \
+    --hash=sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2 \
+    # via flake8
 pyparsing==2.4.1.1 \
     --hash=sha256:43c5486cefefa536c9aab528881c992328f020eefe4f6d06332449c365218580 \
-    --hash=sha256:d6c5ffe9d0305b9b977f7a642d36b9370954d1da7ada4c62393382cbadad4265
+    --hash=sha256:d6c5ffe9d0305b9b977f7a642d36b9370954d1da7ada4c62393382cbadad4265 \
+    # via packaging
 pytest==4.6.4 \
     --hash=sha256:6aa9bc2f6f6504d7949e9df2a756739ca06e58ffda19b5e53c725f7b03fb4aae \
     --hash=sha256:b77ae6f2d1a760760902a7676887b665c086f71e3461c64ed2a312afcedc00d6
-scandir==1.10.0 \
-    --hash=sha256:2586c94e907d99617887daed6c1d102b5ca28f1085f90446554abf1faf73123e \
-    --hash=sha256:2ae41f43797ca0c11591c0c35f2f5875fa99f8797cb1a1fd440497ec0ae4b022 \
-    --hash=sha256:2b8e3888b11abb2217a32af0766bc06b65cc4a928d8727828ee68af5a967fa6f \
-    --hash=sha256:2c712840c2e2ee8dfaf36034080108d30060d759c7b73a01a52251cc8989f11f \
-    --hash=sha256:4d4631f6062e658e9007ab3149a9b914f3548cb38bfb021c64f39a025ce578ae \
-    --hash=sha256:67f15b6f83e6507fdc6fca22fedf6ef8b334b399ca27c6b568cbfaa82a364173 \
-    --hash=sha256:7d2d7a06a252764061a020407b997dd036f7bd6a175a5ba2b345f0a357f0b3f4 \
-    --hash=sha256:8c5922863e44ffc00c5c693190648daa6d15e7c1207ed02d6f46a8dcc2869d32 \
-    --hash=sha256:92c85ac42f41ffdc35b6da57ed991575bdbe69db895507af88b9f499b701c188 \
-    --hash=sha256:b24086f2375c4a094a6b51e78b4cf7ca16c721dcee2eddd7aa6494b42d6d519d \
-    --hash=sha256:cb925555f43060a1745d0a321cca94bcea927c50114b623d73179189a4e100ac \
-    # via pathlib2
 six==1.12.0 \
     --hash=sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c \
     --hash=sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73 \