Merge pull request #33 from freedomofpress/bye-pipenv

remove Pipenv in favor of pip-tools, security updates

.circleci/config.yml

@@ -9,10 +9,14 @@ jobs:
       - run:
           name: Install requirements and run tests
           command: |
-            pipenv install --dev
-            pipenv run python -m unittest -v
+            virtualenv .venv
+            source .venv/bin/activate
+            pip install --require-hashes -r dev-requirements.txt
+            make test
 
       - run:
           name: Check Python dependencies for CVEs
           command: |
-            pipenv check
+            set -e
+            source .venv/bin/activate
+            make safety

MANIFEST.in

@@ -1,6 +1,4 @@
 include LICENSE
-include Pipfile
-include Pipfile.lock
 include README.md
 include VERSION
 include config-example.yaml

Makefile

@@ -1,6 +1,41 @@
-install:
-	pipenv install
-	sudo cp qubes/securedrop.Proxy /etc/qubes-rpc/securedrop.Proxy
+# Bandit is a static code analysis tool to detect security vulnerabilities in Python applications
+# https://wiki.openstack.org/wiki/Security/Projects/Bandit
+.PHONY: bandit
+bandit: ## Run bandit with medium level excluding test-related folders
+	pip install --upgrade pip && \
+        pip install --upgrade bandit!=1.6.0 && \
+	bandit -ll --recursive . --exclude tests,.venv
 
+.PHONY: safety
+safety: ## Runs `safety check` to check python dependencies for vulnerabilities
+	pip install --upgrade safety && \
+		for req_file in `find . -type f -name '*requirements.txt'`; do \
+			echo "Checking file $$req_file" \
+			&& safety check --full-report -r $$req_file \
+			&& echo -e '\n' \
+			|| exit 1; \
+		done
+
+.PHONY: update-pip-requirements
+update-pip-requirements: ## Updates all Python requirements files via pip-compile.
+	pip-compile --generate-hashes --output-file dev-requirements.txt dev-requirements.in requirements.in
+	pip-compile --generate-hashes --output-file requirements.txt requirements.in
+
+.PHONY: test
 test:
-	pipenv run python -m unittest -v
+	python -m unittest -v
+
+# Explaination of the below shell command should it ever break.
+# 1. Set the field separator to ": ##" and any make targets that might appear between : and ##
+# 2. Use sed-like syntax to remove the make targets
+# 3. Format the split fields into $$1) the target name (in blue) and $$2) the target descrption
+# 4. Pass this file as an arg to awk
+# 5. Sort it alphabetically
+# 6. Format columns with colon as delimiter.
+.PHONY: help
+help: ## Print this message and exit.
+	@printf "Makefile for developing and testing the SecureDrop proxy.\n"
+	@printf "Subcommands:\n\n"
+	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \
+		| sort \
+		| column -s ':' -t

README.md

@@ -22,10 +22,20 @@ be poked at and demonstrated.
 
 #### install requirements
 
-To try the proxy script, first use `pipenv` to create an environment
+To try the proxy script, first use `venv` to create an environment
 and install requirements. In the root of the project directory, run
 
-    pipenv install
+```
+virtualenv .venv
+source .venv/bin/activate
+pip install --require-hashes -r dev-requirements.txt
+```
+
+#### Update dependencies
+
+To add or update a dependency, modify either `dev-requirements.in` and `requirements.in` and then run `make update-pip-dependencies`. This will generate `dev-requirements.txt` and `requirements.txt`.
+
+**IMPORTANT:** Do not modify `build-requirements.txt` during normal development. We use a pip mirror for our build process and the hashes in that file point to wheels on our mirror.
 
 #### configuration
 
@@ -90,7 +100,7 @@ this documentation, we assume the client is running in
 
 Edit `qubes/securedrop.Proxy` to reflect the path to `entrypoint.sh`
 in this repo. Also edit the directory to this repo code in `entrypoint.sh`.
-Next, run `make install`, which will move `securedrop.Proxy`
+Next, run `sudo cp qubes/securedrop.Proxy /etc/qubes-rpc/securedrop.Proxy`, which will move `securedrop.Proxy`
 (the qubes-rpc "server path definition" file) into place in
 `/etc/qubes-rpc/`.
 

build-requirements.txt

@@ -0,0 +1,10 @@
+certifi==2018.10.15 --hash=sha256:a5471c55b011bd45d6155f5c3629310c1d2f1e1a5a899b7e438a223343de583d
+chardet==3.0.4 --hash=sha256:9f178988ca4c86e8a319b51aac1185b6fe5192328eb5a163c286f4bf50b7b3d8
+furl==2.0.0 --hash=sha256:cc0eb8998dcc7c5b58bc8625891a9ff563e2765e112024fa3d1e3521481de8b6
+idna==2.7 --hash=sha256:954e65e127d0433a352981f43f291a438423d5b385ebf643c70fd740e0634111
+orderedmultidict==1.0 --hash=sha256:25489716d76d2cc8aa656bfb00cd40b6ca29d5e11ccde0db60c2b46ad52bb40a
+pyyaml==5.1 --hash=sha256:b8d80623e9d4e348c59ea726ce3032a2eb15abca6a48d3828362d11c6014a0a7
+requests==2.20.0 --hash=sha256:2a539dd6af40a611f3b8eb3f99d3567781352ece1698b2fab42bf4c2218705b5
+six==1.11.0 --hash=sha256:4663c7a1dbed033cfb294f2d534bd6151c0698dc12ecabb4eaa3cb041d758528
+urllib3==1.24.1 --hash=sha256:ac4755b0ae019d670d5fb420c39cb531389ab6ca8b652e12f56259f5cbc0ce21
+werkzeug==0.14.1 --hash=sha256:177ea4248bf0475cbc060edb35a0bdcf6e6daeac9e1296de5ddb3493e5ec15b9

dev-requirements.in

@@ -0,0 +1,9 @@
+flake8==3.5.0
+multidict==4.4.2
+pip-tools==3.1.0
+pycodestyle==2.3.1
+pyflakes==1.6.0
+six==1.11.0
+vcrpy==2.0.1
+wrapt==1.10.11
+yarl==1.2.6