Merge pull request #106 from freedomofpress/matrix-ci

Update CI configuration, add bookworm jobs
freedomofpress · Feb 20, 2023 · 3316fa0 · 3316fa0
2 parents 0e303f8 + 95a5c4b
commit 3316fa0
Show file tree

Hide file tree

Showing 4 changed files with 129 additions and 76 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,22 +1,67 @@
 ---
 common-steps:
-  - &run_tests
+  - &install_testing_dependencies
     run:
-      name: Install requirements and run tests
+      name: Install testing dependencies
       command: |
-        apt-get update && apt-get install -y make python3-venv
+        set -e
+        apt update && apt install -y git gnupg make python3-dev python3-venv
+
+  - &install_build_dependencies
+    run:
+      name: Install build dependencies
+      command: |
+        set -e
+        apt update && apt install -y git make sudo
+
+  - &run_unit_tests
+    run:
+      name: Install requirements and run unit tests
+      command: |
+        set -e
+        make venv
+        source .venv/bin/activate
+        export PYTHONPATH=$PYTHONPATH:.  # so alembic can get to Base metadata
+        make test
+
+  - &run_lint
+    run:
+      name: Run lint, type checking, code formatting
+      command: |
+        set -e
+        make venv
+        source .venv/bin/activate
+        make lint
+
+  - &check_security
+    run:
+      name: Run static analysis on source code to find security issues
+      command: |
+        set -e
+        make venv
+        source .venv/bin/activate
+        make bandit
+
+  - &check_python_dependencies_for_vulnerabilities
+    run:
+      name: Check Python dependencies for known vulnerabilities
+      command: |
+        set -e
         make venv
         source .venv/bin/activate
-        make check
+        make safety
 
   - &install_packaging_dependencies
     run:
-      name: Install Debian packaging dependencies and download wheels
+      name: Install Debian packaging dependencies and download Python wheels
       command: |
-        apt-get update && apt-get install -y git git-lfs make sudo
+        set -x
         mkdir ~/packaging && cd ~/packaging
+        # local builds may not have an ssh url, so || true
+        git config --global --unset url.ssh://[email protected] || true
         git clone https://github.com/freedomofpress/securedrop-debian-packaging.git
         cd securedrop-debian-packaging
+        apt-get update && apt-get install -y sudo make
         make install-deps
         PKG_DIR=~/project make requirements
 
@@ -29,48 +74,83 @@ common-steps:
         # previous run step), else return 0.
         git diff --quiet
 
-  - &make_source_tarball
-    run:
-      name: Tag and make source tarball
-      command: |
-        cd ~/project
-        ./update_version.sh 1000.0  # Dummy version number, doesn't matter what we put here
-        python3 setup.py sdist
-
   - &build_debian_package
     run:
       name: Build debian package
       command: |
         cd ~/packaging/securedrop-debian-packaging
         export PKG_VERSION=1000.0
-        export PKG_PATH=~/project
+        export PKG_PATH=~/project/
         make securedrop-proxy
 
-version: 2
+version: 2.1
+
 jobs:
-  build-bullseye:
-    docker:
-      - image: debian:bullseye
+  build:
+    parameters: &parameters
+      image:
+        type: string
+    docker: &docker
+      - image: debian:<< parameters.image >>
     steps:
+      - *install_build_dependencies
       - checkout
       - *install_packaging_dependencies
       - *verify_requirements
-      - *make_source_tarball
       - *build_debian_package
 
-  test-bullseye:
-    docker:
-      - image: debian:bullseye
+  unit-test:
+    parameters: *parameters
+    docker: *docker
+    steps:
+      - *install_testing_dependencies
+      - checkout
+      - *run_unit_tests
+      - store_test_results:
+          path: test-results
+
+  lint:
+    parameters: *parameters
+    docker: *docker
+    steps:
+      - *install_testing_dependencies
+      - checkout
+      - *run_lint
+
+  check-security:
+    parameters: *parameters
+    docker: *docker
     steps:
+      - *install_testing_dependencies
       - checkout
-      - *run_tests
+      - *check_security
+
+  check-python-security:
+    parameters: *parameters
+    docker: *docker
+    steps:
+      - *install_testing_dependencies
+      - checkout
+      - *check_python_dependencies_for_vulnerabilities
+
 
 workflows:
-  version: 2
   securedrop_proxy_ci:
-    jobs:
-      - test-bullseye
-      - build-bullseye
+    jobs: &jobs
+      - unit-test:
+          matrix: &matrix
+            parameters:
+              image:
+                - bullseye
+                - bookworm
+      - lint:
+          matrix: *matrix
+      - check-security:
+          matrix: *matrix
+      - check-python-security:
+          matrix: *matrix
+      - build:
+          matrix: *matrix
 
   nightly:
     triggers:
@@ -80,6 +160,4 @@ workflows:
             branches:
               only:
                 - main
-    jobs:
-      - test-bullseye
-      - build-bullseye
+    jobs: *jobs
diff --git a/Makefile b/Makefile
@@ -26,7 +26,7 @@ safety: ## Runs `safety check` to check python dependencies for vulnerabilities
 		done
 
 .PHONY: lint
-lint: isort-check black-check ## Run isort, black and flake8
+lint: check-isort check-black mypy ## Run isort, black and flake8 and mypy
 	@flake8 securedrop_proxy tests
 
 .PHONY: mypy
@@ -37,16 +37,16 @@ mypy: ## Run mypy static type checker
 black: ## Run black for file formatting
 	@black securedrop_proxy tests
 
-.PHONY: black-check
-black-check: ## Check Python source code formatting with black
+.PHONY: check-black
+check-black: ## Check Python source code formatting with black
 	@black --check --diff securedrop_proxy tests
 
 .PHONY: isort
 isort: ## Run isort for file formatting
 	@isort securedrop_proxy/*.py tests/*.py
 
-.PHONY: isort-check
-isort-check: ## Check isort for file formatting
+.PHONY: check-isort
+check-isort: ## Check isort for file formatting
 	@isort --check-only --diff securedrop_proxy/*.py tests/*.py
 
 .PHONY: sync-requirements

diff --git a/requirements/dev-requirements.in b/requirements/dev-requirements.in
@@ -2,16 +2,8 @@ black
 coverage
 flake8
 isort
-mccabe
-multidict
 mypy
-mypy-extensions
 pip-tools
-pycodestyle
-pyflakes
 types-PyYAML
 types-requests
-six
 vcrpy
-wrapt
-yarl
diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt
@@ -110,9 +110,7 @@ isort==5.11.2 \
 mccabe==0.7.0 \
     --hash=sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325 \
     --hash=sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e
-    # via
-    #   -r requirements/dev-requirements.in
-    #   flake8
+    # via flake8
 multidict==6.0.3 \
     --hash=sha256:018c8e3be7f161a12b3e41741b6721f9baeb2210f4ab25a6359b7d76c1017dce \
     --hash=sha256:01b456046a05ff7cceefb0e1d2a9d32f05efcb1c7e0d152446304e11557639ce \
@@ -188,9 +186,7 @@ multidict==6.0.3 \
     --hash=sha256:f76109387e1ec8d8e2137c94c437b89fe002f29e0881aae8ae45529bdff92000 \
     --hash=sha256:f8a728511c977df6f3d8af388fcb157e49f11db4a6637dd60131b8b6e40b0253 \
     --hash=sha256:fb6c3dc3d65014d2c782f5acf0b3ba14e639c6c33d3ed8932ead76b9080b3544
-    # via
-    #   -r requirements/dev-requirements.in
-    #   yarl
+    # via yarl
 mypy==0.991 \
     --hash=sha256:0714258640194d75677e86c786e80ccf294972cc76885d3ebbb560f11db0003d \
     --hash=sha256:0c8f3be99e8a8bd403caa8c03be619544bc2c77a7093685dcf308c6b109426c6 \
@@ -227,7 +223,6 @@ mypy-extensions==0.4.3 \
     --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \
     --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8
     # via
-    #   -r requirements/dev-requirements.in
     #   black
     #   mypy
 packaging==22.0 \
@@ -242,6 +237,10 @@ pep517==0.13.0 \
     --hash=sha256:4ba4446d80aed5b5eac6509ade100bff3e7943a8489de249654a5ae9b33ee35b \
     --hash=sha256:ae69927c5c172be1add9203726d4b84cf3ebad1edcd5f71fcdc746e66e829f59
     # via build
+pip==22.3.1 \
+    --hash=sha256:65fd48317359f3af8e593943e6ae1506b66325085ea64b706a998c6e83eeaf38 \
+    --hash=sha256:908c78e6bc29b676ede1c4d57981d490cb892eb45cd8c214ab6298125119e077
+    # via pip-tools
 pip-tools==6.12.0 \
     --hash=sha256:8e22fbc84ede7ca522ba4b033c4fcf6a6419adabc75d24747be3d8262504489a \
     --hash=sha256:f441603c63b16f4af0dd5026f7522a49eddec2bc8a4a4979af44e1f6b0a1c13e
@@ -253,15 +252,11 @@ platformdirs==2.6.0 \
 pycodestyle==2.10.0 \
     --hash=sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053 \
     --hash=sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610
-    # via
-    #   -r requirements/dev-requirements.in
-    #   flake8
+    # via flake8
 pyflakes==3.0.1 \
     --hash=sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf \
     --hash=sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd
-    # via
-    #   -r requirements/dev-requirements.in
-    #   flake8
+    # via flake8
 pyyaml==5.4.1 \
     --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \
     --hash=sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696 \
@@ -299,12 +294,14 @@ requests==2.26.0 \
     --hash=sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24 \
     --hash=sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7
     # via -r requirements/requirements.in
+setuptools==65.6.3 \
+    --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \
+    --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75
+    # via pip-tools
 six==1.11.0 \
     --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \
     --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb
-    # via
-    #   -r requirements/dev-requirements.in
-    #   vcrpy
+    # via vcrpy
 tomli==2.0.1 \
     --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
     --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
@@ -409,9 +406,7 @@ wrapt==1.14.1 \
     --hash=sha256:ee2b1b1769f6707a8a445162ea16dddf74285c3964f605877a20e38545c3c462 \
     --hash=sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015 \
     --hash=sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af
-    # via
-    #   -r requirements/dev-requirements.in
-    #   vcrpy
+    # via vcrpy
 yarl==1.8.2 \
     --hash=sha256:009a028127e0a1755c38b03244c0bea9d5565630db9c4cf9572496e947137a87 \
     --hash=sha256:0414fd91ce0b763d4eadb4456795b307a71524dbacd015c657bb2a39db2eab89 \
@@ -487,16 +482,4 @@ yarl==1.8.2 \
     --hash=sha256:fb742dcdd5eec9f26b61224c23baea46c9055cf16f62475e11b9b15dfd5c117b \
     --hash=sha256:fc77086ce244453e074e445104f0ecb27530d6fd3a46698e33f6c38951d5a0f1 \
     --hash=sha256:ff205b58dc2929191f68162633d5e10e8044398d7a45265f90a0f1d51f85f72c
-    # via
-    #   -r requirements/dev-requirements.in
-    #   vcrpy
-
-# The following packages are considered to be unsafe in a requirements file:
-pip==22.3.1 \
-    --hash=sha256:65fd48317359f3af8e593943e6ae1506b66325085ea64b706a998c6e83eeaf38 \
-    --hash=sha256:908c78e6bc29b676ede1c4d57981d490cb892eb45cd8c214ab6298125119e077
-    # via pip-tools
-setuptools==65.6.3 \
-    --hash=sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54 \
-    --hash=sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75
-    # via pip-tools
+    # via vcrpy