rename dev-requirements-debian to dev-sdw-requirements

rename dev-requirements to dev-buster-requirements remove redundant pip install update dev-requirements for bullseye instead of buster add ci jobs for bullseye fix circleci bullseye permission issue Signed-off-by: Allie Crevier <[email protected]> reformat using black remove unnecessary dev env clean up requirements files for different dev envs Signed-off-by: Allie Crevier <[email protected]> remove duplicate makefile target for updating a prod dependency update pip-tools to address jazzband#1617 update all dev dependencies and fix typo Signed-off-by: Allie Crevier <[email protected]>
freedomofpress · Jul 6, 2022 · d7cd4bb · d7cd4bb
1 parent b1525ff
commit d7cd4bb
Show file tree

Hide file tree

Showing 14 changed files with 1,758 additions and 806 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,7 +2,15 @@
 common-steps:
   - &install_deps
     run:
-      name: Install base dependencies for Debian python
+      name: Install base dependencies for Bullseye python
+      command: |
+        set -e
+        pip uninstall virtualenv -y || true
+        sudo apt update && sudo apt install -y make git gnupg
+
+  - &install_deps_on_buster
+    run:
+      name: Install base dependencies for Buster python
       command: |
         set -e
         pip uninstall virtualenv -y || true
@@ -15,7 +23,16 @@ common-steps:
         set -e
         make venv
         source .venv/bin/activate
-        pip install --require-hashes -r requirements/dev-requirements.txt
+        export PYTHONPATH=$PYTHONPATH:.  # so alembic can get to Base metadata
+        make check --keep-going
+
+  - &run_tests_on_buster
+    run:
+      name: Install requirements and run tests
+      command: |
+        set -e
+        make venv-buster
+        source .venv/bin/activate
         export PYTHONPATH=$PYTHONPATH:.  # so alembic can get to Base metadata
         make check --keep-going
 
@@ -36,6 +53,20 @@ common-steps:
         make safety
 
   - &install_packaging_dependencies
+    run:
+      name: Install Debian packaging dependencies and download wheels
+      command: |
+        set -x
+        mkdir ~/packaging && cd ~/packaging
+        # local builds may not have an ssh url, so || true
+        git config --global --unset url.ssh://[email protected] || true
+        git clone https://github.com/freedomofpress/securedrop-debian-packaging.git
+        cd securedrop-debian-packaging
+        sudo apt update && sudo apt install -y make
+        make install-deps
+        PKG_DIR=~/project make requirements
+
+  - &install_packaging_dependencies_buster
     run:
       name: Install Debian packaging dependencies and download wheels
       command: |
@@ -71,24 +102,47 @@ common-steps:
 
 version: 2
 jobs:
-  build-buster:
+  build-bullseye:
     docker:
-      - image: debian:buster
+      - image: circleci/python:3.9-bullseye
     steps:
       - *install_deps
       - checkout
       - *install_packaging_dependencies
       - *verify_requirements
       - *build_debian_package
 
+  test-bullseye:
+    docker:
+      - image: circleci/python:3.9-bullseye
+    steps:
+      - *install_deps
+      - checkout
+      - run: sudo apt update && sudo apt install -y sqlite3 libqt5x11extras5 xvfb python3-tk python3-dev
+      - *run_tests
+      - store_test_results:
+          path: test-results
+      - *run_lint
+      - *check_python_dependencies_for_vulns
+
+  build-buster:
+    docker:
+      - image: debian:buster
+    steps:
+      - *install_deps_on_buster
+      - checkout
+      - *install_packaging_dependencies_buster
+      - *verify_requirements
+      - *build_debian_package
+
   test-buster:
     docker:
       - image: debian:buster
     steps:
-      - *install_deps
+      - *install_deps_on_buster
       - checkout
       - run: apt-get update && apt-get install -y sqlite3 libqt5x11extras5 xvfb python3-tk python3-dev
-      - *run_tests
+      - *run_tests_on_buster
       - store_test_results:
           path: test-results
       - *run_lint
@@ -98,6 +152,8 @@ workflows:
   version: 2
   securedrop_client_ci:
     jobs:
+      - test-bullseye
+      - build-bullseye
       - test-buster
       - build-buster
 
@@ -110,5 +166,7 @@ workflows:
               only:
                 - main
     jobs:
+      - test-bullseye
+      - build-bullseye
       - test-buster
       - build-buster
diff --git a/Makefile b/Makefile
@@ -4,21 +4,29 @@ all: help
 # Default to plain "python3"
 PYTHON ?= python3
 
-.PHONY: venv-debian
-venv-debian: hooks ## Provision a Python 3 virtualenv for development on a prod-like system that has installed dependencies specified in https://github.com/freedomofpress/securedrop-debian-packaging/blob/main/securedrop-client/debian/control
-	$(PYTHON) -m venv .venv-debian --system-site-packages
-	.venv-debian/bin/pip install --upgrade pip wheel
-	.venv-debian/bin/pip install --require-hashes -r "requirements/dev-requirements-debian.txt"
+.PHONY: venv
+venv: hooks ## Provision a Python 3 virtualenv for development on Linux
+	$(PYTHON) -m venv .venv
+	.venv/bin/pip install --upgrade pip wheel
+	.venv/bin/pip install --require-hashes -r "requirements/dev-requirements.txt"
+	@echo "#################"
+	@echo "Make sure to run: source .venv/bin/activate"
+
+.PHONY: venv-sdw
+venv-sdw: hooks ## Provision a Python 3 virtualenv for development on a prod-like system that has installed dependencies specified in https://github.com/freedomofpress/securedrop-debian-packaging/blob/main/securedrop-client/debian/control
+	$(PYTHON) -m venv .venv --system-site-packages
+	.venv/bin/pip install --upgrade pip wheel
+	.venv/bin/pip install --require-hashes -r "requirements/dev-sdw-requirements.txt"
 	@echo "#################"
 	@echo "Virtualenv with Debian system-packages is complete."
 	@echo "Make sure to install the apt packages for system Qt."
-	@echo "Then run: source .venv-debian/bin/activate"
+	@echo "Then run: source .venv/bin/activate"
 
-.PHONY: venv
-venv: hooks ## Provision a Python 3 virtualenv for development on Linux
+.PHONY: venv-buster
+venv-buster: hooks ## Provision a Python 3 virtualenv for development on Linux
 	$(PYTHON) -m venv .venv
 	.venv/bin/pip install --upgrade pip wheel
-	.venv/bin/pip install --require-hashes -r "requirements/dev-requirements.txt"
+	.venv/bin/pip install --require-hashes -r "requirements/dev-buster-requirements.txt"
 	@echo "#################"
 	@echo "Make sure to run: source .venv/bin/activate"
 
@@ -61,11 +69,11 @@ check-black: ## Check Python source code formatting with black
 
 .PHONY: isort
 isort: ## Run isort to organize Python imports
-	@isort --skip-glob .venv-debian ./
+	@isort --skip-glob .venv ./
 
 .PHONY: check-isort
 check-isort: ## Check Python import organization with isort
-	@isort --skip-glob .venv-debian --check-only --diff ./
+	@isort --skip-glob .venv --check-only --diff ./
 
 .PHONY: mypy
 mypy: ## Run static type checker
@@ -136,39 +144,26 @@ safety: ## Runs `safety check` to check python dependencies for vulnerabilities
 bandit: ## Run bandit with medium level excluding test-related folders
 	pip install --upgrade pip && \
 	pip install --upgrade bandit && \
-	bandit -ll --recursive . --exclude ./tests,./.venv,./.venv-debian
+	bandit -ll --recursive . --exclude ./tests,./.venv
 
 .PHONY: check
 check: clean check-black check-isort semgrep bandit lint mypy test-random test-integration test-functional ## Run the full CI test suite
 
-.PHONY: sync-requirements
-sync-requirements:  ## Update dev-requirements.txt to pin to the same versions of prod dependencies
+.PHONY: dev-requirements
+dev-requirements:  ## Update dev-*requirements.txt files if pinned versions do not comply with the dependency specifications in dev-*requirements.in
+	pip-compile --allow-unsafe --generate-hashes --output-file requirements/dev-requirements.txt requirements/dev-requirements.in
+	pip-compile --allow-unsafe --generate-hashes --output-file requirements/dev-sdw-requirements.txt requirements/dev-sdw-requirements.in
+
+.PHONY: update-dev-dependencies
+update-dev-dependencies:  ## Update dev requirements in case there are newer versions of packages or updates to prod dependencies
 	if test -f "requirements/dev-requirements.txt"; then rm -r requirements/dev-requirements.txt; fi
-	if test -f "requirements/dev-requirements-debian.txt"; then rm -r requirements/dev-requirements-debian.txt; fi
-	cp requirements/requirements.txt requirements/dev-requirements.txt
-	cp requirements/requirements.txt requirements/dev-requirements-debian.txt
-	pip-compile --allow-unsafe --generate-hashes --output-file requirements/dev-requirements.txt requirements/requirements.in requirements/dev-requirements.in
-	pip-compile --allow-unsafe --generate-hashes --output-file requirements/dev-requirements-debian.txt requirements/requirements.in requirements/dev-requirements-debian.in
+	if test -f "requirements/dev-sdw-requirements.txt"; then rm -r requirements/dev-sdw-requirements.txt; fi
+	$(MAKE) dev-requirements
 
 .PHONY: requirements
 requirements:  ## Update *requirements.txt files if pinned versions do not comply with the dependency specifications in *requirements.in
 	pip-compile --generate-hashes --output-file requirements/requirements.txt requirements/requirements.in
-	$(MAKE) sync-requirements
-
-.PHONY: update-dependency
-update-dependency:  ## Add or upgrade a package to the latest version that complies with the dependency specifications in requirements.in
-	pip-compile --generate-hashes --upgrade-package $(PACKAGE) --output-file requirements/requirements.txt requirements/requirements.in
-	$(MAKE) sync-requirements
-
-.PHONY: update-dev-only-dependencies
-update-dev-only-dependencies:  ## Update dev-requirements.txt to pin to the latest versions of dev-only dependencies that comply with the dependency specifications in dev-requirements.in
-	$(MAKE) sync-requirements
-	@while read line; do \
-		pip-compile --allow-unsafe --generate-hashes --upgrade-package $file --output-file requirements/dev-requirements.txt requirements/requirements.in requirements/dev-requirements.in; \
-	done < 'requirements/dev-requirements.in'
-	@while read line; do \
-		pip-compile --allow-unsafe --generate-hashes --upgrade-package $file --output-file requirements/dev-requirements-debian.txt requirements/requirements.in requirements/dev-requirements-debian.in; \
-	done < 'requirements/dev-requirements-debian.in'
+	$(MAKE) dev-requirements
 
 # Explaination of the below shell command should it ever break.
 # 1. Set the field separator to ": ##" and any make targets that might appear between : and ##

diff --git a/requirements/dev-buster-requirements.in b/requirements/dev-buster-requirements.in
@@ -0,0 +1,3 @@
+-r dev-sdw-requirements.in
+PyQt5==5.11.3  # Match version of system package on Buster
+sip==4.19.8  # Match version of system package on Buster