Merge pull request #373 from freedomofpress/bookworm-or-bust

Remove buster, add failing bookworm jobs

.circleci/config.yml

@@ -56,7 +56,7 @@ common-steps:
     run:
       name: Build debian package
       command: |
-        source /etc/os-release
+        export VERSION_CODENAME=$(~/project/scripts/codename)
         export PKG_PATH=~/packaging/$PKG_NAME/
         export PKG_VERSION=$VERSION_TO_BUILD
         make $PKG_NAME
@@ -91,7 +91,7 @@ common-steps:
     run:
       name: Get metapackage version via changelog for the current platform
       command: |
-        source /etc/os-release
+        export VERSION_CODENAME=$(~/project/scripts/codename)
         CURRENT_VERSION=$(grep -oP "\d+\.\d+\.\d+" ${PKG_NAME}/debian/changelog-${VERSION_CODENAME} | head -n1)
         export VERSION_TO_BUILD="$CURRENT_VERSION"
         # Enable access to this env var in subsequent run steps
@@ -104,22 +104,25 @@ common-steps:
       name: Commit workstation debs for deployment to apt-test.freedom.press
       command: |
         apt-get update
-        apt-get install -y ca-certificates git git-lfs openssh-client
+        apt-get install -y ca-certificates git git-lfs openssh-client python3
 
         git clone [email protected]:freedomofpress/securedrop-dev-packages-lfs.git
         cd securedrop-dev-packages-lfs
 
         git config user.email "[email protected]"
         git config user.name "sdcibot"
 
-        for codename in buster bullseye
-        do
-            # Copy built debian packages to the relevant workstation repo and git push.
-            mkdir -p ./workstation/${codename}-nightlies/
-            cp /tmp/workspace/${codename}/*.deb ./workstation/${codename}-nightlies/
-            git add workstation/${codename}-nightlies/*.deb
-            git commit -m "Automated SecureDrop workstation build (${codename})"
-        done
+        # Copy built debian packages to the relevant workstation repo and git push.
+        mkdir -p ./workstation/${CODENAME}-nightlies/
+        cp /tmp/workspace/${CODENAME}/*.deb ./workstation/${CODENAME}-nightlies/ ||:
+        git add workstation/${CODENAME}-nightlies/*.deb ||:
+
+        # Clean up old nightlies too
+        ~/project/scripts/clean-old-nightlies.py workstation/${CODENAME}-nightlies
+
+        # If there are changes, diff-index will fail, so we commit and push
+        git diff-index --quiet HEAD || git commit -m "Automated SecureDrop workstation build (${CODENAME})"
+
         git push origin main
 
 version: 2.1
@@ -205,38 +208,8 @@ jobs:
             # If there are changes, diff-index will fail, so we commit and push
             git diff-index --quiet HEAD || git commit -m "Automatically updating Tor packages" && git push origin main
 
-  clean-old-nightlies:
-    docker:
-      - image: debian:bullseye
-    steps:
-      - checkout
-      - *addsshkeys
-      - run:
-          name: clone and delete old nightlies
-          command: |
-            apt-get update
-            apt-get install -y python3 ca-certificates git git-lfs openssh-client
-
-            # Clone the dev repo and configure it
-            ssh-keyscan github.com >> ~/.ssh/known_hosts
-            git clone [email protected]:freedomofpress/securedrop-dev-packages-lfs.git
-            cd securedrop-dev-packages-lfs
-            git lfs install
-            git config user.email "[email protected]"
-            git config user.name "sdcibot"
-
-            # Run the clean script and `git add` any deletions
-            cd ..
-            ./scripts/clean-old-nightlies.py securedrop-dev-packages-lfs/workstation/buster-nightlies
-            ./scripts/clean-old-nightlies.py securedrop-dev-packages-lfs/workstation/bullseye-nightlies
-            cd securedrop-dev-packages-lfs
-            git add .
-
-            # If there are changes, diff-index will fail, so we commit and push
-            git diff-index --quiet HEAD || git commit -m "Deleting old nightlies" && git push origin main
-
 
-  build:
+  build: &build
     parameters:
       package:
         type: string
@@ -260,7 +233,9 @@ jobs:
       - *builddebianpackage
       - *persist
 
-  build-metapackage:
+  build2: *build
+
+  build-metapackage: &build-metapackage
     parameters:
       package:
         type: string
@@ -282,9 +257,11 @@ jobs:
       - *builddebianpackage
       - *persist
 
-  build-buster-securedrop-workstation-grsec:
+  build2-metapackage: *build-metapackage
+
+  build-bookworm-securedrop-workstation-grsec:
     docker:
-      - image: debian:buster
+      - image: debian:bookworm
     environment:
       PKG_NAME: securedrop-workstation-grsec
     steps:
@@ -304,16 +281,22 @@ jobs:
       - *setmetapackageversionplatform
       - *builddebianpackage
 
-  push-packages:
+  push-bullseye: &push
     docker:
       - image: debian:bullseye
+    environment:
+      CODENAME: bullseye
     steps:
       - checkout
       - attach_workspace:
           at: /tmp/workspace
       - *addsshkeys
       - *commitworkstationdebs
 
+  push-bookworm:
+    <<: *push
+    environment:
+      CODENAME: bookworm
 
 workflows:
   build-packages:
@@ -324,26 +307,26 @@ workflows:
       - build:
           matrix:
             parameters:
-              package:
+              package: &packages
                 - securedrop-client
                 - securedrop-export
                 - securedrop-log
                 - securedrop-proxy
               image:
-                - "buster"
-                - "bullseye"
+                - bullseye
+                - bookworm
       - build-metapackage:
           matrix:
             parameters:
-              package:
+              package: &metapackages
                 - securedrop-keyring
                 - securedrop-workstation-config
                 - securedrop-workstation-viewer
               image:
-                - "buster"
-                - "bullseye"
-      - build-buster-securedrop-workstation-grsec
+                - bullseye
+                - bookworm
       - build-bullseye-securedrop-workstation-grsec
+      - build-bookworm-securedrop-workstation-grsec
 
   nightly:
     triggers:
@@ -354,38 +337,47 @@ workflows:
               only:
                 - main
     jobs:
+      # We need bullseye packages to be pushed if bookworm jobs fail.
+      # So bookworm jobs run in a separate track ("build2") that is identical,
+      # except the push-bullseye job only depends on the bullseye jobs and the
+      # final step is push-bookworm, which might fail.
+      - reprepro-update-tor
       - build:
           matrix:
             parameters:
-              package:
-                - securedrop-client
-                - securedrop-export
-                - securedrop-log
-                - securedrop-proxy
+              package: *packages
+              image:
+                - bullseye
+              nightly: ["nightly"]
+      - build2:
+          matrix:
+            parameters:
+              package: *packages
               image:
-                - "buster"
-                - "bullseye"
+                - bookworm
               nightly: ["nightly"]
       - build-metapackage:
           matrix:
             parameters:
-              package:
-                - securedrop-keyring
-                - securedrop-workstation-config
-                - securedrop-workstation-viewer
+              package: *metapackages
               image:
-                - "buster"
-                - "bullseye"
+                - bullseye
               nightly: ["nightly"]
-      - push-packages:
+      - build2-metapackage:
+          matrix:
+            parameters:
+              package: *metapackages
+              image:
+                - bookworm
+              nightly: ["nightly"]
+      - push-bullseye:
           requires:
+            - reprepro-update-tor
             - build
             - build-metapackage
-      - reprepro-update-tor:
+      # This should be last as we do expect it to fail occasionally.
+      - push-bookworm:
           requires:
-            # Wait for push to finish
-            - push-packages
-      - clean-old-nightlies:
-          requires:
-            # Wait for tor update to finish
-            - reprepro-update-tor
+            - push-bullseye
+            - build2
+            - build2-metapackage

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,5 @@
+## Checklist
+
+* [ ] `bullseye` builds and jobs are passing
+* [ ] This PR does not introduce any new `bookworm` test failures
+* [ ] If there are `bookworm` test failures, an issue has been filed for them

bootstrap-sha256sums.txt

@@ -1,3 +1,4 @@
+2ac1fac97d092631d116bcdc5709dc100d31c969567e792841f6116539ce8fde  Cython-0.29.22-cp310-cp310-linux_x86_64.whl
 8c9eec7e9de2a30861ca347d0a149cc1482de12fc765fa06c414930e8ce20d0a  Cython-0.29.22-cp37-cp37m-linux_x86_64.whl
 bc8c021d836ddada4986836263f75c8215f7f65aff062c449e981e9d8ac352fc  Cython-0.29.22-cp39-cp39-linux_x86_64.whl
 df6b83c7a6d1d967ea89a2903e4a931377634a297459652e4551734c48195406  Cython-0.29.22.tar.gz

bootstrap-sha256sums.txt.asc

@@ -1,16 +1,16 @@
 -----BEGIN PGP SIGNATURE-----
 
-iQIzBAABCAAdFiEE2MtZ8F27ngU4xIGd8QX4EBsFJpsFAmJ77VAACgkQ8QX4EBsF
-JpuoKxAAp0/JAPi/I2EvJr/BSPGPxU1oGZxT6N0RppTY3iyMsVNA1vezy7TAoyDc
-9tm6GeHTHl4uAf1uD0IlZQpFZV8HTquqezp0vKsoBnf5pR42XeNiv6FcUmk2NH15
-r+DtExOEqtjWcpgBzzFnHW1zQ1E4GCkPe37hX0ZPiF26XRkuKiIqXijsKgvrZF31
-+c8LNxUcEA4mODD83JX77Y3WBLGLItxxahaMixmWJzXswkzb5X2fWA7w6omJ0C07
-O6vFbF3gHAHYMhv7JNbmeauc+I48AST8BYeXuCcZsa6HfomNcwK49JBiTmIfbYkb
-F23Cq0YvoWftBCNodwInaJdoQeFrGGkUKOXSOSLmIE/MMZMIQ6duA5C0mgoT2brH
-bD1hFAurW5ibIhjLoE3tKpoQJbQMSXfVovPVgGhOQ71Ck11Da2vK20emQ6OUgTLT
-Xq9kHWvyd7W195lrSHHSn2JdjVbYO74Goiu6kU30SsLAjbxWgw4uedbtOM+rHAdR
-oZ/4h3xZG3HmBYs/BLlJ6Z9ywkxLQN8fjewhAbQrjF9I8inNTHmnMIibC9hot9Ip
-REgkhGqk0J7GbEzyYoxiVsmgs5Dg7fYhtawV1sQJx8kYo+2FU0OjfmGZiuM3C0w4
-JlYePi6LN8LdcEhqwYwhwBExb5h2JWxhhmHo17/qaI9KLxR7ZRs=
-=qxUw
+iQIyBAABCAAdFiEE2MtZ8F27ngU4xIGd8QX4EBsFJpsFAmMPDW0ACgkQ8QX4EBsF
+JpudsA/2MXdnLN2l8dZqVwvM6QhS5mcqOOPXE5KexUfuvX7tc/EydnDEhZNTAl9w
+Imj/cpEcxpXB+etWcLYYaOpHqldAod52fp6oanozLJ22tQg4G8PcroJBmlTggtU/
+kNleluhpLCyOmwQ/XGbsO/WqmHbUs1VtaavmMF0bjRrH0wmfY0OAx9BYkkDDebkq
+3drWksitMYct0k9m5Uq2XEWQIh2w56F7+9Fu5InD9UFtNAGBij4cJ+bEXtQLC97F
+9in/rhtAGgdvv19gV91obOeCjBDEeriUJAuekFs06gQoXN1RLmMoDc+yIoxpJwbH
+QYDzMFTCQpDxDgvnZotYHyMWIea05xKLKIsetq/GLvai4/ez3TKY/OvFEQ02gjEd
+aUksq7qSao4c08aNV5zJJRYtO1ttw0xYqEzXTL/ZzTWQexocXC7dUIDiwmz7/yh4
+ktWyMWWdBq5euPVKMvMVo4Z9GXQv8P9Sw5QYRBFuxt+H1+yGTvSGp5BsXJhSPExq
+mPT82lZnJnb4LkdHgdUA0VzLAj7frKXbfXhn0ErQuBFj64865a0u4gD33qpAM7Pr
+j7RzMwsSexC47SfqCnU5eFegaFYOpHVr9Q99gkwLfqmPWMZ7LX4P7hIfQBSpuJ2g
+5iBJBuKpkYUIMMOo+IyclT/UP/h/bNuWZXklr7GWLZ59pky12w==
+=H9G5
 -----END PGP SIGNATURE-----

build-requirements.txt

@@ -1,6 +1,6 @@
 build==0.3.0 --hash=sha256:3fe8fac398ae76f534dee92c0db01c2960e271094f8cd52aa9d24a633c244e59
 click==7.1.2 --hash=sha256:3b0769bd65ca00ffcb57975e750fac079639a3d3ce393c5a0d4c65123141c87a
-cython==0.29.22 --hash=sha256:8c9eec7e9de2a30861ca347d0a149cc1482de12fc765fa06c414930e8ce20d0a --hash=sha256:bc8c021d836ddada4986836263f75c8215f7f65aff062c449e981e9d8ac352fc
+cython==0.29.22 --hash=sha256:2ac1fac97d092631d116bcdc5709dc100d31c969567e792841f6116539ce8fde --hash=sha256:8c9eec7e9de2a30861ca347d0a149cc1482de12fc765fa06c414930e8ce20d0a --hash=sha256:bc8c021d836ddada4986836263f75c8215f7f65aff062c449e981e9d8ac352fc
 flit-core==2.3.0 --hash=sha256:a14d6cca50a51eff2c418fc3f8e887cd31a3233a6b465451fd57074e232d7c4c
 importlib-metadata==3.7.0 --hash=sha256:35fc3ab05e060b85739da17db4ce33e8532ce1010d892d178f85a211e080e1ff
 packaging==20.9 --hash=sha256:44698376bc57a48290dc82bfdc20d339684856b24d6b8c3760b284de4eebc498

scripts/build-debianpackage

@@ -14,6 +14,7 @@ set -o pipefail
 
 # Store root of repo, since we'll change dirs several times.
 CUR_DIR="$(git rev-parse --show-toplevel)"
+VERSION_CODENAME=$("${CUR_DIR}/scripts/codename")
 
 # Verify sha256sums.txt in the git repo
 "${CUR_DIR}/scripts/verify-sha256sum-signature"
@@ -27,8 +28,6 @@ TOP_BUILDDIR="$PWD/build/debbuild/packaging"
 mkdir -p "$TOP_BUILDDIR"
 rm -rf "${TOP_BUILDDIR:?}/${PKG_NAME}"
 mkdir -p "${TOP_BUILDDIR}/${PKG_NAME}"
-# Platform metadata
-source /etc/os-release
 
 # Validate required args.
 if [[ -z "${PKG_NAME:-}" ]]; then

scripts/codename

@@ -0,0 +1,14 @@
+#!/bin/bash
+# Returns the Debian version's codename (e.g. "bullseye") in a way that should
+# work across both released versions and unreleased ones.
+# See https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1008735
+
+source /etc/os-release
+
+if [[ "$VERSION_CODENAME" != "" ]]; then
+    echo $VERSION_CODENAME
+else
+    # PRETTY_NAME="Debian GNU/Linux bookworm/sid"
+    # Use awk to split on spaces and /
+    echo $PRETTY_NAME | awk '{split($0, a, "[ /]"); print a[4]}'
+fi

scripts/update-changelog

@@ -6,8 +6,8 @@ set -e
 set -u
 set -o pipefail
 
-source /etc/os-release
 TOPLEVEL="$(git rev-parse --show-toplevel)"
+VERSION_CODENAME=$("${TOPLEVEL}/scripts/codename")
 
 # These env vars are only required when updating changelogs,
 # otherwise the developer must edit the fields by hand, lest