Merge pull request #372 from freedomofpress/plain-debian-images

Use plain Debian images in CI
freedomofpress · Aug 31, 2022 · d1adbde · d1adbde
2 parents 4fdd949 + 29ccbe0
commit d1adbde
Show file tree

Hide file tree

Showing 11 changed files with 49 additions and 184 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -6,15 +6,19 @@ common-steps:
       paths:
         - "*"
 
-  - &removevirtualenv
-    run:
-      name: Removes the upstream virtualenv from the original container image
-      command: sudo pip uninstall virtualenv -y
-
   - &installdeps
     run:
       name: Install Debian packaging dependencies
-      command: make install-deps
+      command: |
+        apt-get update && apt-get install -y make sudo
+        make install-deps
+
+  - &installtestdeps
+    run:
+      name: Install test dependencies
+      command: |
+        apt-get install reprotest faketime -y --no-install-recommends
+        .venv/bin/pip install -r test-requirements.txt
 
   - &getnightlyversion
     run:
@@ -123,51 +127,51 @@ version: 2.1
 jobs:
   lint-and-test:
     docker:
-      - image: circleci/python:3.7-buster
+      - image: debian:bullseye
     steps:
       - checkout
+      - *installdeps
+      - *installtestdeps
       - run:
           name: install test requirements, run linters, and run tests
           command: |
-            make install-deps
-            sudo apt-get install reprotest -y
             make lint-desktop-files
-            virtualenv -p /usr/bin/python3 .venv
             source .venv/bin/activate
-            pip install -r test-requirements.txt
-            sudo sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py
+            sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py
             make test
 
   reprotest-wheels:
     docker:
-      - image: quay.io/freedomofpress/packaging-debian-bullseye@sha256:b23206cff095aa5f0764d03c18ff1212a386386b9026441cc36ea836b19b0919
+      - image: debian:bullseye
+    # Our "ci" user will have problems if the repository is in /root
+    working_directory: "/srv"
     steps:
       - checkout
+      - *installdeps
+      - *installtestdeps
       - run:
           name: install test requirements and run tests
           command: |
-            make install-deps
-            sudo apt-get install reprotest -y
-            source .venv/bin/activate
-            pip install -r test-requirements.txt
-            sudo sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py
-            pytest -vvs tests/test_reproducible_wheels.py
+            adduser --system ci --ingroup root
+            sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py
+            # Have our "ci" user take over the git repo
+            chown ci:root -R .
+            sudo -u ci bash -c "source .venv/bin/activate && pytest -vvs tests/test_reproducible_wheels.py"
 
   reprotest-debs:
     docker:
-      - image: quay.io/freedomofpress/packaging-debian-buster@sha256:16d2df1935807c6a751d0536e3cb36970c4c22d7324915d25ee84c90b032c307
+      - image: debian:bullseye
     steps:
       - checkout
+      - *installdeps
+      - *installtestdeps
       - run:
           name: install test requirements and run tests
           command: |
-            make install-deps
-            sudo apt-get install reprotest -y
             source .venv/bin/activate
-            pip install -r test-requirements.txt
             # Patch reprotest in-place to skip 'setarch' prefix, which fails under containers.
             # We cannot use Ubuntu 20.04 python3.8 to build Debian 10 python3.7 packages.
-            sudo sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py
+            sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py
             pytest -vvs tests/test_reproducible_debian_packages.py
 
   reprepro-update-tor:
@@ -242,14 +246,13 @@ jobs:
         type: string
         default: ""
     docker:
-      - image: circleci/python:<< parameters.image >>
+      - image: debian:<< parameters.image >>
     environment:
       PKG_NAME: << parameters.package >>
       SCHEDULE_NAME: << pipeline.schedule.name >>
       IS_NIGHTLY: << parameters.nightly >>
     steps:
       - checkout
-      - *removevirtualenv
       - *installdeps
       - *clonefromenv
       - *getnightlyversion
@@ -270,10 +273,9 @@ jobs:
       PKG_NAME: << parameters.package >>
       IS_NIGHTLY: << parameters.nightly >>
     docker:
-      - image: circleci/python:<< parameters.image >>
+      - image: debian:<< parameters.image >>
     steps:
       - checkout
-      - *removevirtualenv
       - *installdeps
       - *setmetapackageversion
       - *updatedebianchangelog
@@ -282,7 +284,7 @@ jobs:
 
   build-buster-securedrop-workstation-grsec:
     docker:
-      - image: circleci/python:3.7-buster
+      - image: debian:buster
     environment:
       PKG_NAME: securedrop-workstation-grsec
     steps:
@@ -293,7 +295,7 @@ jobs:
 
   build-bullseye-securedrop-workstation-grsec:
     docker:
-      - image: circleci/python:3.9-bullseye
+      - image: debian:bullseye
     environment:
       PKG_NAME: securedrop-workstation-grsec
     steps:
@@ -328,8 +330,8 @@ workflows:
                 - securedrop-log
                 - securedrop-proxy
               image:
-                - "3.7-buster"
-                - "3.9-bullseye"
+                - "buster"
+                - "bullseye"
       - build-metapackage:
           matrix:
             parameters:
@@ -338,8 +340,8 @@ workflows:
                 - securedrop-workstation-config
                 - securedrop-workstation-viewer
               image:
-                - "3.7-buster"
-                - "3.9-bullseye"
+                - "buster"
+                - "bullseye"
       - build-buster-securedrop-workstation-grsec
       - build-bullseye-securedrop-workstation-grsec
 
@@ -361,8 +363,8 @@ workflows:
                 - securedrop-log
                 - securedrop-proxy
               image:
-                - "3.7-buster"
-                - "3.9-bullseye"
+                - "buster"
+                - "bullseye"
               nightly: ["nightly"]
       - build-metapackage:
           matrix:
@@ -372,8 +374,8 @@ workflows:
                 - securedrop-workstation-config
                 - securedrop-workstation-viewer
               image:
-                - "3.7-buster"
-                - "3.9-bullseye"
+                - "buster"
+                - "bullseye"
               nightly: ["nightly"]
       - push-packages:
           requires:

diff --git a/dockerfiles/bullseye/Dockerfile b/dockerfiles/bullseye/Dockerfile
diff --git a/dockerfiles/bullseye/Makefile b/dockerfiles/bullseye/Makefile
diff --git a/dockerfiles/bullseye/image_hash b/dockerfiles/bullseye/image_hash
diff --git a/dockerfiles/bullseye/push.sh b/dockerfiles/bullseye/push.sh
diff --git a/dockerfiles/buster/Dockerfile b/dockerfiles/buster/Dockerfile
diff --git a/dockerfiles/buster/Makefile b/dockerfiles/buster/Makefile
diff --git a/dockerfiles/buster/image_hash b/dockerfiles/buster/image_hash
diff --git a/dockerfiles/buster/push.sh b/dockerfiles/buster/push.sh
diff --git a/scripts/build-sync-wheels b/scripts/build-sync-wheels
@@ -11,6 +11,12 @@ import argparse
 from pprint import pprint
 
 
+if os.geteuid() == 0:
+    # tar has issues when resetting permissions and ends up
+    # causing very subtle reproducibility issues.
+    print("This script cannot be run as root.", file=sys.stderr)
+    sys.exit(1)
+
 # Set SOURCE_DATE_EPOCH to a predictable value. Using the first
 # commit to the SecureDrop project:
 #

diff --git a/scripts/install-deps b/scripts/install-deps
@@ -1,7 +1,10 @@
 #!/bin/bash
+set -euxo pipefail
 # Installs required dependencies for building SecureDrop Worsktation packages.
 # Assumes a Debian 10 machine, ideally a Qubes AppVM.
 
+VIRTUAL_ENV="${VIRTUAL_ENV:-}"
+
 sudo apt-get update
 sudo apt-get install  \
     build-essential \