Merge pull request #54 from freedomofpress/add-qc-tools

Add quality control tools

.circleci/config.yml

@@ -7,15 +7,7 @@ common-steps:
         virtualenv .venv
         source .venv/bin/activate
         pip install --require-hashes -r dev-requirements.txt
-        make test
-
-  - &check_python_dependencies_for_vulns
-    run:
-      name: Check Python dependencies for CVEs
-      command: |
-        set -e
-        source .venv/bin/activate
-        make safety
+        make check
 
   - &install_packaging_dependencies
     run:
@@ -81,15 +73,13 @@ jobs:
     steps:
       - checkout
       - *run_tests
-      - *check_python_dependencies_for_vulns
 
   test-buster:
     docker:
       - image: circleci/python:3.7-buster
     steps:
       - checkout
       - *run_tests
-      - *check_python_dependencies_for_vulns
 
 workflows:
   version: 2

Makefile

@@ -1,10 +1,13 @@
 # Bandit is a static code analysis tool to detect security vulnerabilities in Python applications
 # https://wiki.openstack.org/wiki/Security/Projects/Bandit
+.PHONY: all
+all: help
+
 .PHONY: bandit
 bandit: ## Run bandit with medium level excluding test-related folders
 	pip install --upgrade pip && \
-        pip install --upgrade bandit!=1.6.0 && \
-	bandit -ll --recursive . --exclude tests,.venv
+		pip install --upgrade bandit!=1.6.0 && \
+	bandit -ll --recursive securedrop_proxy
 
 .PHONY: safety
 safety: ## Runs `safety check` to check python dependencies for vulnerabilities
@@ -16,16 +19,43 @@ safety: ## Runs `safety check` to check python dependencies for vulnerabilities
 			|| exit 1; \
 		done
 
+.PHONY: lint
+lint: ## Run flake8
+	@flake8 securedrop_proxy tests
+
+.PHONY: mypy
+mypy: ## Run mypy static type checker
+	@mypy --ignore-missing-imports securedrop_proxy
+
+
 .PHONY: update-pip-requirements
 update-pip-requirements: ## Updates all Python requirements files via pip-compile.
 	pip-compile --generate-hashes --output-file dev-requirements.txt dev-requirements.in requirements.in
 	pip-compile --generate-hashes --output-file requirements.txt requirements.in
 
 .PHONY: test
-test:
-	python -m unittest -v
+test: clean .coverage ## Runs tests with coverage
+
+.coverage:
+	@coverage run --source securedrop_proxy -m unittest
+
+.PHONY: browse-coverage
+browse-coverage: .coverage ## Generates and opens HTML coverage report
+	@coverage html
+	@xdg-open htmlcov/index.html 2>/dev/null || open htmlcov/index.html 2>/dev/null
+
+.PHONY: check
+check: clean lint test mypy safety bandit  ## Runs all tests and code checkers
+
+.PHONY: clean
+clean:  ## Clean the workspace of generated resources
+	@rm -rf .mypy_cache build dist *.egg-info .coverage .eggs docs/_build .pytest_cache lib htmlcov .cache && \
+		find . \( -name '*.py[co]' -o -name dropin.cache \) -delete && \
+		find . \( -name '*.bak' -o -name dropin.cache \) -delete && \
+		find . \( -name '*.tgz' -o -name dropin.cache \) -delete && \
+		find . -name __pycache__ -print0 | xargs -0 rm -rf
 
-# Explaination of the below shell command should it ever break.
+# Explanation of the below shell command should it ever break.
 # 1. Set the field separator to ": ##" and any make targets that might appear between : and ##
 # 2. Use sed-like syntax to remove the make targets
 # 3. Format the split fields into $$1) the target name (in blue) and $$2) the target descrption

dev-requirements.in

@@ -1,8 +1,12 @@
-flake8==3.5.0
+coverage==5.0
+flake8==3.6.0
+mccabe==0.6.1
 multidict==4.4.2
+mypy==0.701
+mypy-extensions==0.4.1
 pip-tools==3.1.0
-pycodestyle==2.3.1
-pyflakes==1.6.0
+pycodestyle==2.4.0
+pyflakes==2.0.0
 six==1.11.0
 vcrpy==2.0.1
 wrapt==1.10.11

dev-requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --generate-hashes --output-file dev-requirements.txt dev-requirements.in requirements.in
+#    pip-compile --generate-hashes --output-file=dev-requirements.txt dev-requirements.in requirements.in
 #
 certifi==2018.10.15 \
     --hash=sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c \
@@ -14,9 +14,41 @@ click==7.0 \
     --hash=sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13 \
     --hash=sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7 \
     # via pip-tools
-flake8==3.5.0 \
-    --hash=sha256:7253265f7abd8b313e3892944044a365e3f4ac3fcdcfb4298f55ee9ddf188ba0 \
-    --hash=sha256:c7841163e2b576d435799169b78703ad6ac1bbb0f199994fc05f700b2a90ea37
+coverage==5.0 \
+    --hash=sha256:0cd13a6e98c37b510a2d34c8281d5e1a226aaf9b65b7d770ef03c63169965351 \
+    --hash=sha256:1a4b6b6a2a3a6612e6361130c2cc3dc4378d8c221752b96167ccbad94b47f3cd \
+    --hash=sha256:2ee55e6dba516ddf6f484aa83ccabbb0adf45a18892204c23486938d12258cde \
+    --hash=sha256:3be5338a2eb4ef03c57f20917e1d12a1fd10e3853fed060b6d6b677cb3745898 \
+    --hash=sha256:44b783b02db03c4777d8cf71bae19eadc171a6f2a96777d916b2c30a1eb3d070 \
+    --hash=sha256:475bf7c4252af0a56e1abba9606f1e54127cdf122063095c75ab04f6f99cf45e \
+    --hash=sha256:47c81ee687eafc2f1db7f03fbe99aab81330565ebc62fb3b61edfc2216a550c8 \
+    --hash=sha256:4a7f8e72b18f2aca288ff02255ce32cc830bc04d993efbc87abf6beddc9e56c0 \
+    --hash=sha256:50197163a22fd17f79086e087a787883b3ec9280a509807daf158dfc2a7ded02 \
+    --hash=sha256:56b13000acf891f700f5067512b804d1ec8c301d627486c678b903859d07f798 \
+    --hash=sha256:79388ae29c896299b3567965dbcd93255f175c17c6c7bca38614d12718c47466 \
+    --hash=sha256:79fd5d3d62238c4f583b75d48d53cdae759fe04d4fb18fe8b371d88ad2b6f8be \
+    --hash=sha256:7fe3e2fde2bf1d7ce25ebcd2d3de3650b8d60d9a73ce6dcef36e20191291613d \
+    --hash=sha256:81042a24f67b96e4287774014fa27220d8a4d91af1043389e4d73892efc89ac6 \
+    --hash=sha256:81326f1095c53111f8afc95da281e1414185f4a538609a77ca50bdfa39a6c207 \
+    --hash=sha256:8873dc0d8f42142ea9f20c27bbdc485190fff93823c6795be661703369e5877d \
+    --hash=sha256:88d2cbcb0a112f47eef71eb95460b6995da18e6f8ca50c264585abc2c473154b \
+    --hash=sha256:91f2491aeab9599956c45a77c5666d323efdec790bfe23fcceafcd91105d585a \
+    --hash=sha256:979daa8655ae5a51e8e7a24e7d34e250ae8309fd9719490df92cbb2fe2b0422b \
+    --hash=sha256:9c871b006c878a890c6e44a5b2f3c6291335324b298c904dc0402ee92ee1f0be \
+    --hash=sha256:a6d092545e5af53e960465f652e00efbf5357adad177b2630d63978d85e46a72 \
+    --hash=sha256:b5ed7837b923d1d71c4f587ae1539ccd96bfd6be9788f507dbe94dab5febbb5d \
+    --hash=sha256:ba259f68250f16d2444cbbfaddaa0bb20e1560a4fdaad50bece25c199e6af864 \
+    --hash=sha256:be1d89614c6b6c36d7578496dc8625123bda2ff44f224cf8b1c45b810ee7383f \
+    --hash=sha256:c1b030a79749aa8d1f1486885040114ee56933b15ccfc90049ba266e4aa2139f \
+    --hash=sha256:c95bb147fab76f2ecde332d972d8f4138b8f2daee6c466af4ff3b4f29bd4c19e \
+    --hash=sha256:d52c1c2d7e856cecc05aa0526453cb14574f821b7f413cc279b9514750d795c1 \
+    --hash=sha256:d609a6d564ad3d327e9509846c2c47f170456344521462b469e5cb39e48ba31c \
+    --hash=sha256:e1bad043c12fb58e8c7d92b3d7f2f49977dcb80a08a6d1e7a5114a11bf819fca \
+    --hash=sha256:e5a675f6829c53c87d79117a8eb656cc4a5f8918185a32fc93ba09778e90f6db \
+    --hash=sha256:fec32646b98baf4a22fdceb08703965bd16dea09051fbeb31a04b5b6e72b846c
+flake8==3.6.0 \
+    --hash=sha256:6a35f5b8761f45c5513e3405f110a86bea57982c3b75b766ce7b65217abe1670 \
+    --hash=sha256:c01f8a3963b3571a8e6bd7a4063359aff90749e160778e03817cd9b71c9e07d2
 furl==2.0.0 \
     --hash=sha256:f7e90e9f85ef3f2e64485f04c2a80b50af6133942812fd87a44d45305b079018 \
     --hash=sha256:fdcaedc1fb19a63d7d875b0105b0a5b496dd0989330d454a42bcb401fa5454ec
@@ -25,8 +57,7 @@ idna==2.7 \
     --hash=sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16
 mccabe==0.6.1 \
     --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
-    --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f \
-    # via flake8
+    --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f
 multidict==4.4.2 \
     --hash=sha256:05eeab69bf2b0664644c62bd92fabb045163e5b8d4376a31dfb52ce0210ced7b \
     --hash=sha256:0c85880efa7cadb18e3b5eef0aa075dc9c0a3064cbbaef2e20be264b9cf47a64 \
@@ -57,18 +88,33 @@ multidict==4.4.2 \
     --hash=sha256:e8848ae3cd6a784c29fae5055028bee9bffcc704d8bcad09bd46b42b44a833e2 \
     --hash=sha256:e8a048bfd7d5a280f27527d11449a509ddedf08b58a09a24314828631c099306 \
     --hash=sha256:f6dd28a0ac60e2426a6918f36f1b4e2620fc785a0de7654cd206ba842eee57fd
+mypy-extensions==0.4.1 \
+    --hash=sha256:37e0e956f41369209a3d5f34580150bcacfabaa57b33a15c0b25f4b5725e0812 \
+    --hash=sha256:b16cabe759f55e3409a7d231ebd2841378fb0c27a5d1994719e340e4f429ac3e
+mypy==0.701 \
+    --hash=sha256:2afe51527b1f6cdc4a5f34fc90473109b22bf7f21086ba3e9451857cf11489e6 \
+    --hash=sha256:56a16df3e0abb145d8accd5dbb70eba6c4bd26e2f89042b491faa78c9635d1e2 \
+    --hash=sha256:5764f10d27b2e93c84f70af5778941b8f4aa1379b2430f85c827e0f5464e8714 \
+    --hash=sha256:5bbc86374f04a3aa817622f98e40375ccb28c4836f36b66706cf3c6ccce86eda \
+    --hash=sha256:6a9343089f6377e71e20ca734cd8e7ac25d36478a9df580efabfe9059819bf82 \
+    --hash=sha256:6c9851bc4a23dc1d854d3f5dfd5f20a016f8da86bcdbb42687879bb5f86434b0 \
+    --hash=sha256:b8e85956af3fcf043d6f87c91cbe8705073fc67029ba6e22d3468bfee42c4823 \
+    --hash=sha256:b9a0af8fae490306bc112229000aa0c2ccc837b49d29a5c42e088c132a2334dd \
+    --hash=sha256:bbf643528e2a55df2c1587008d6e3bda5c0445f1240dfa85129af22ae16d7a9a \
+    --hash=sha256:c46ab3438bd21511db0f2c612d89d8344154c0c9494afc7fbc932de514cf8d15 \
+    --hash=sha256:f7a83d6bd805855ef83ec605eb01ab4fa42bcef254b13631e451cbb44914a9b0
 orderedmultidict==1.0 \
     --hash=sha256:24e3b730cf84e4a6a68be5cc760864905cf66abc89851e724bd5b4e849eaa96b \
     --hash=sha256:b89895ba6438038d0bdf88020ceff876cf3eae0d5c66a69b526fab31125db2c5
 pip-tools==3.1.0 \
     --hash=sha256:31b43e5f8d605fc84f7506199025460abcb98a29d12cc99db268f73e39cf55e5 \
     --hash=sha256:b1ceca03b4a48346b2f6870565abb09d8d257d5b1524b4c6b222185bf26c3870
-pycodestyle==2.3.1 \
-    --hash=sha256:682256a5b318149ca0d2a9185d365d8864a768a28db66a84a2ea946bcc426766 \
-    --hash=sha256:6c4245ade1edfad79c3446fadfc96b0de2759662dc29d07d80a6f27ad1ca6ba9
-pyflakes==1.6.0 \
-    --hash=sha256:08bd6a50edf8cffa9fa09a463063c425ecaaf10d1eb0335a7e8b1401aef89e6f \
-    --hash=sha256:8d616a382f243dbf19b54743f280b80198be0bca3a5396f1d2e1fca6223e8805
+pycodestyle==2.4.0 \
+    --hash=sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83 \
+    --hash=sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a
+pyflakes==2.0.0 \
+    --hash=sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49 \
+    --hash=sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae
 pyyaml==5.1 \
     --hash=sha256:1adecc22f88d38052fb787d959f003811ca858b799590a5eaa70e63dca50308c \
     --hash=sha256:436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95 \
@@ -87,6 +133,27 @@ requests==2.20.0 \
 six==1.11.0 \
     --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \
     --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb
+typed-ast==1.3.5 \
+    --hash=sha256:132eae51d6ef3ff4a8c47c393a4ef5ebf0d1aecc96880eb5d6c8ceab7017cc9b \
+    --hash=sha256:18141c1484ab8784006c839be8b985cfc82a2e9725837b0ecfa0203f71c4e39d \
+    --hash=sha256:2baf617f5bbbfe73fd8846463f5aeafc912b5ee247f410700245d68525ec584a \
+    --hash=sha256:3d90063f2cbbe39177e9b4d888e45777012652d6110156845b828908c51ae462 \
+    --hash=sha256:4304b2218b842d610aa1a1d87e1dc9559597969acc62ce717ee4dfeaa44d7eee \
+    --hash=sha256:4983ede548ffc3541bae49a82675996497348e55bafd1554dc4e4a5d6eda541a \
+    --hash=sha256:5315f4509c1476718a4825f45a203b82d7fdf2a6f5f0c8f166435975b1c9f7d4 \
+    --hash=sha256:6cdfb1b49d5345f7c2b90d638822d16ba62dc82f7616e9b4caa10b72f3f16649 \
+    --hash=sha256:7b325f12635598c604690efd7a0197d0b94b7d7778498e76e0710cd582fd1c7a \
+    --hash=sha256:8d3b0e3b8626615826f9a626548057c5275a9733512b137984a68ba1598d3d2f \
+    --hash=sha256:8f8631160c79f53081bd23446525db0bc4c5616f78d04021e6e434b286493fd7 \
+    --hash=sha256:912de10965f3dc89da23936f1cc4ed60764f712e5fa603a09dd904f88c996760 \
+    --hash=sha256:b010c07b975fe853c65d7bbe9d4ac62f1c69086750a574f6292597763781ba18 \
+    --hash=sha256:c908c10505904c48081a5415a1e295d8403e353e0c14c42b6d67f8f97fae6616 \
+    --hash=sha256:c94dd3807c0c0610f7c76f078119f4ea48235a953512752b9175f9f98f5ae2bd \
+    --hash=sha256:ce65dee7594a84c466e79d7fb7d3303e7295d16a83c22c7c4037071b059e2c21 \
+    --hash=sha256:eaa9cfcb221a8a4c2889be6f93da141ac777eb8819f077e1d09fb12d00a09a93 \
+    --hash=sha256:f3376bc31bad66d46d44b4e6522c5c21976bf9bca4ef5987bb2bf727f4506cbb \
+    --hash=sha256:f9202fa138544e13a4ec1a6792c35834250a85958fde1251b6a22e07d1260ae7 \
+    # via mypy
 urllib3==1.24.3 \
     --hash=sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4 \
     --hash=sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb
@@ -108,3 +175,7 @@ yarl==1.2.6 \
     --hash=sha256:db6f70a4b09cde813a4807843abaaa60f3b15fb4a2a06f9ae9c311472662daa1 \
     --hash=sha256:f17495e6fe3d377e3faac68121caef6f974fcb9e046bc075bcff40d8e5cc69a4 \
     --hash=sha256:f85900b9cca0c67767bb61b2b9bd53208aaa7373dae633dbe25d179b4bf38aa7
+
+# WARNING: The following packages were not pinned, but pip requires them to be
+# pinned when the requirements file includes hashes. Consider using the --allow-unsafe flag.
+# setuptools

requirements.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --generate-hashes --output-file requirements.txt requirements.in
+#    pip-compile --generate-hashes --output-file=requirements.txt requirements.in
 #
 certifi==2018.10.15 \
     --hash=sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c \

securedrop_proxy/callbacks.py

@@ -1,13 +1,16 @@
+import os
 import subprocess
 import sys
 import json
+import tempfile
 import uuid
 
 
 def err_on_done(res):
     print(json.dumps(res.__dict__))
     sys.exit(1)
 
+
 # callback for handling non-JSON content. in production-like
 # environments, we want to call `qvm-move-to-vm` (and expressly not
 # `qvm-move`, since we want to include the destination VM name) to
@@ -20,9 +23,11 @@ def on_save(fh, res, conf):
     fn = str(uuid.uuid4())
 
     try:
-        subprocess.run(["cp", fh.name, "/tmp/{}".format(fn)])
-        if conf.dev is not True:
-            subprocess.run(['qvm-move-to-vm', conf.target_vm, "/tmp/{}".format(fn)])
+        with tempfile.TemporaryDirectory() as tmpdir:
+            tmpfile = os.path.join(os.path.abspath(tmpdir), fn)
+            subprocess.run(["cp", fh.name, tmpfile])
+            if conf.dev is not True:
+                subprocess.run(["qvm-move-to-vm", conf.target_vm, tmpfile])
     except Exception:
         res.status = 500
         res.headers['Content-Type'] = 'application/json'
@@ -32,7 +37,7 @@ def on_save(fh, res, conf):
 
     res.headers['Content-Type'] = 'application/json'
     res.headers['X-Origin-Content-Type'] = res.headers['Content-Type']
-    res.body = json.dumps({'filename': fn })
+    res.body = json.dumps({'filename': fn})
 
 
 def on_done(res):

securedrop_proxy/config.py

@@ -1,12 +1,14 @@
 import os
 import yaml
 
+
 class Conf:
     scheme = ''
     host = ''
     port = 0
     dev = False
 
+
 def read_conf(conf_path, p):
 
     if not os.path.isfile(conf_path):
@@ -17,13 +19,17 @@ def read_conf(conf_path, p):
         fh = open(conf_path, 'r')
         conf_in = yaml.safe_load(fh)
     except yaml.YAMLError:
-        p.simple_error(500, 'YAML syntax error while reading configuration file {}'.format(conf_path))
+        p.simple_error(
+            500, "YAML syntax error while reading configuration file {}".format(conf_path)
+        )
         p.on_done(p.res)
     except Exception:
-        p.simple_error(500, 'Error while opening or reading configuration file {}'.format(conf_path))
+        p.simple_error(
+            500, "Error while opening or reading configuration file {}".format(conf_path)
+        )
         p.on_done(p.res)
 
-    req_conf_keys = set(('host','scheme','port'))
+    req_conf_keys = set(('host', 'scheme', 'port'))
     missing_keys = req_conf_keys - set(conf_in.keys())
     if len(missing_keys) > 0:
         p.simple_error(500, 'Configuration file missing required keys: {}'.format(missing_keys))
@@ -37,8 +43,14 @@ def read_conf(conf_path, p):
     if 'dev' in conf_in and conf_in['dev'] is True:
         c.dev = True
     else:
-        if 'target_vm' not in conf_in:
-            p.simple_error(500, 'Configuration file missing `target_vm` key, which is required when not in development mode')
+        if "target_vm" not in conf_in:
+            p.simple_error(
+                500,
+                (
+                    "Configuration file missing `target_vm` key, which is required "
+                    "when not in development mode"
+                ),
+            )
             p.on_done(p.res)
 
         c.target_vm = conf_in['target_vm']

securedrop_proxy/entrypoint.py

@@ -6,12 +6,9 @@
 # called with exactly one argument: the path to its config file. See
 # the README for configuration options.
 
-import json
 import logging
 import os
-import subprocess
 import sys
-import uuid
 
 from logging.handlers import TimedRotatingFileHandler
 

securedrop_proxy/version.py

@@ -1,3 +1,8 @@
 import pkgutil
 
-version = pkgutil.get_data("securedrop_proxy", "VERSION").decode("utf-8")
+version = None
+version_content = pkgutil.get_data("securedrop_proxy", "VERSION")
+if isinstance(version_content, bytes):
+    version = version_content.decode("utf-8")
+else:
+    raise ValueError("Could not read VERSION file")

setup.cfg

@@ -0,0 +1,9 @@
+[flake8]
+exclude =
+    .git,
+    __pycache__,
+
+max-line-length = 100
+
+builtins =
+    _,