From 3c81ff81617ba3729ea1006bcf1143c07d446235 Mon Sep 17 00:00:00 2001 From: Kunal Mehta Date: Tue, 23 Aug 2022 12:17:51 -0400 Subject: [PATCH] Switch to plain Debian images in CI For the most part this is a very straightforward switch, except for the reprotest-wheels job. As noted in the previous commit, we cannot run the build-sync-wheels script as root (which the Debian images default to). We add a "ci" user, chown the repository for them, and then run the test command. One subtle catch is that the repository needs to be checked out in a place other than /root, so we set the working_directory to /srv. I also noticed that we were installing all of the 5GB of diffoscope dependencies without actually ever using diffoscope (and if we were using diffoscope, we don't need extra utilities to diff debs and wheels), so using `--no-install-recommends` shaves off about 3 minutes of runtime. We do need to manually install faketime since it's only a recommends, but that seems like a fair tradeoff. And we can now delete the custom `packaging-debian-{buster,bullseye}` images we were maintaining! --- .circleci/config.yml | 78 +++++++++++++++++---------------- dockerfiles/bullseye/Dockerfile | 37 ---------------- dockerfiles/bullseye/Makefile | 20 --------- dockerfiles/bullseye/image_hash | 2 - dockerfiles/bullseye/push.sh | 12 ----- dockerfiles/buster/Dockerfile | 41 ----------------- dockerfiles/buster/Makefile | 20 --------- dockerfiles/buster/image_hash | 2 - dockerfiles/buster/push.sh | 12 ----- scripts/install-deps | 3 ++ 10 files changed, 43 insertions(+), 184 deletions(-) delete mode 100644 dockerfiles/bullseye/Dockerfile delete mode 100644 dockerfiles/bullseye/Makefile delete mode 100644 dockerfiles/bullseye/image_hash delete mode 100755 dockerfiles/bullseye/push.sh delete mode 100644 dockerfiles/buster/Dockerfile delete mode 100644 dockerfiles/buster/Makefile delete mode 100644 dockerfiles/buster/image_hash delete mode 100755 dockerfiles/buster/push.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 6320ffc8..024af1e7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,15 +6,19 @@ common-steps: paths: - "*" - - &removevirtualenv - run: - name: Removes the upstream virtualenv from the original container image - command: sudo pip uninstall virtualenv -y - - &installdeps run: name: Install Debian packaging dependencies - command: make install-deps + command: | + apt-get update && apt-get install -y make sudo + make install-deps + + - &installtestdeps + run: + name: Install test dependencies + command: | + apt-get install reprotest faketime -y --no-install-recommends + .venv/bin/pip install -r test-requirements.txt - &getnightlyversion run: @@ -123,51 +127,51 @@ version: 2.1 jobs: lint-and-test: docker: - - image: circleci/python:3.7-buster + - image: debian:buster steps: - checkout + - *installdeps + - *installtestdeps - run: name: install test requirements, run linters, and run tests command: | - make install-deps - sudo apt-get install reprotest -y make lint-desktop-files - virtualenv -p /usr/bin/python3 .venv source .venv/bin/activate - pip install -r test-requirements.txt - sudo sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py + sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py make test reprotest-wheels: docker: - - image: quay.io/freedomofpress/packaging-debian-bullseye@sha256:b23206cff095aa5f0764d03c18ff1212a386386b9026441cc36ea836b19b0919 + - image: debian:bullseye + # Our "ci" user will have problems if the repository is in /root + working_directory: "/srv" steps: - checkout + - *installdeps + - *installtestdeps - run: name: install test requirements and run tests command: | - make install-deps - sudo apt-get install reprotest -y - source .venv/bin/activate - pip install -r test-requirements.txt - sudo sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py - pytest -vvs tests/test_reproducible_wheels.py + adduser --system ci --ingroup root + sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py + # Have our "ci" user take over the git repo + chown ci:root -R . + sudo -u ci bash -c "source .venv/bin/activate && pytest -vvs tests/test_reproducible_wheels.py" reprotest-debs: docker: - - image: quay.io/freedomofpress/packaging-debian-buster@sha256:16d2df1935807c6a751d0536e3cb36970c4c22d7324915d25ee84c90b032c307 + - image: debian:buster steps: - checkout + - *installdeps + - *installtestdeps - run: name: install test requirements and run tests command: | - make install-deps - sudo apt-get install reprotest -y source .venv/bin/activate - pip install -r test-requirements.txt # Patch reprotest in-place to skip 'setarch' prefix, which fails under containers. # We cannot use Ubuntu 20.04 python3.8 to build Debian 10 python3.7 packages. - sudo sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py + sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py pytest -vvs tests/test_reproducible_debian_packages.py reprepro-update-tor: @@ -242,14 +246,13 @@ jobs: type: string default: "" docker: - - image: circleci/python:<< parameters.image >> + - image: debian:<< parameters.image >> environment: PKG_NAME: << parameters.package >> SCHEDULE_NAME: << pipeline.schedule.name >> IS_NIGHTLY: << parameters.nightly >> steps: - checkout - - *removevirtualenv - *installdeps - *clonefromenv - *getnightlyversion @@ -270,10 +273,9 @@ jobs: PKG_NAME: << parameters.package >> IS_NIGHTLY: << parameters.nightly >> docker: - - image: circleci/python:<< parameters.image >> + - image: debian:<< parameters.image >> steps: - checkout - - *removevirtualenv - *installdeps - *setmetapackageversion - *updatedebianchangelog @@ -282,7 +284,7 @@ jobs: build-buster-securedrop-workstation-grsec: docker: - - image: circleci/python:3.7-buster + - image: debian:buster environment: PKG_NAME: securedrop-workstation-grsec steps: @@ -293,7 +295,7 @@ jobs: build-bullseye-securedrop-workstation-grsec: docker: - - image: circleci/python:3.9-bullseye + - image: debian:bullseye environment: PKG_NAME: securedrop-workstation-grsec steps: @@ -328,8 +330,8 @@ workflows: - securedrop-log - securedrop-proxy image: - - "3.7-buster" - - "3.9-bullseye" + - "buster" + - "bullseye" - build-metapackage: matrix: parameters: @@ -338,8 +340,8 @@ workflows: - securedrop-workstation-config - securedrop-workstation-viewer image: - - "3.7-buster" - - "3.9-bullseye" + - "buster" + - "bullseye" - build-buster-securedrop-workstation-grsec - build-bullseye-securedrop-workstation-grsec @@ -361,8 +363,8 @@ workflows: - securedrop-log - securedrop-proxy image: - - "3.7-buster" - - "3.9-bullseye" + - "buster" + - "bullseye" nightly: ["nightly"] - build-metapackage: matrix: @@ -372,8 +374,8 @@ workflows: - securedrop-workstation-config - securedrop-workstation-viewer image: - - "3.7-buster" - - "3.9-bullseye" + - "buster" + - "bullseye" nightly: ["nightly"] - push-packages: requires: diff --git a/dockerfiles/bullseye/Dockerfile b/dockerfiles/bullseye/Dockerfile deleted file mode 100644 index bdd1a171..00000000 --- a/dockerfiles/bullseye/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# We want to do things using Debian Bullseye's own Python -FROM debian:bullseye - -# make Apt non-interactive -RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90circleci \ - && echo 'DPkg::Options "--force-confnew";' >> /etc/apt/apt.conf.d/90circleci - -ENV DEBIAN_FRONTEND=noninteractive -# Make sure PATH includes ~/.local/bin -# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=839155 -# This only works for root. The circleci user is done near the end of this Dockerfile -RUN echo 'PATH="$HOME/.local/bin:$PATH"' >> /etc/profile.d/user-local-path.sh - -# man directory is missing in some base images -# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 -RUN apt-get update \ - && mkdir -p /usr/share/man/man1 \ - && apt-get install -y \ - git mercurial xvfb \ - locales sudo openssh-client ca-certificates tar parallel \ - net-tools netcat unzip zip bzip2 gnupg curl wget make python3 python3-venv python3-pip - -# Use unicode -RUN locale-gen C.UTF-8 || true -ENV LANG=C.UTF-8 - -RUN groupadd --gid 3434 ci \ - && useradd --uid 3434 --gid ci --shell /bin/bash --create-home ci \ - && echo 'ci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-ci \ - && echo 'Defaults env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep - - - -USER ci -ENV PATH /home/ci/.local/bin:/home/ci/bin:${PATH} - -CMD ["/bin/sh"] diff --git a/dockerfiles/bullseye/Makefile b/dockerfiles/bullseye/Makefile deleted file mode 100644 index 269b8b1e..00000000 --- a/dockerfiles/bullseye/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -DATE_STR := $(shell date +"%Y_%m_%d") -BUILDER_IMAGE ?= "quay.io/freedomofpress/packaging-debian-bullseye:$(DATE_STR)" - -.PHONY: build-container -build-container: ## Build Docker image for Debian Bullseye wheel and package creation - @echo "███Building Docker image $(BUILDER_IMAGE) for Debian Bullseye wheel and package creation" - @docker build --no-cache -t $(BUILDER_IMAGE) . - -.PHONY: push-container -push-container: ## Push the Docker image for Debian Bullseye wheel and package creation to quay.io - @echo "███Pushing Docker image for Debian package creation to quay.io..." - @./push.sh - -.PHONY: help -help: ## Print this message and exit. - @printf "Build a Docker container for Debian package creation.\n" - @printf "Subcommands:\n\n" - @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \ - | sort \ - | column -s ':' -t diff --git a/dockerfiles/bullseye/image_hash b/dockerfiles/bullseye/image_hash deleted file mode 100644 index ca8e15b0..00000000 --- a/dockerfiles/bullseye/image_hash +++ /dev/null @@ -1,2 +0,0 @@ -# sha256 digest quay.io/freedomofpress/packaging-debian-bullseye:2022_05_10 -197a4b1de4d4b94fc69d242a140684edae70b6760287b066afe3dd6427d4eddb diff --git a/dockerfiles/bullseye/push.sh b/dockerfiles/bullseye/push.sh deleted file mode 100755 index e973638c..00000000 --- a/dockerfiles/bullseye/push.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -DATE_STR=$(date +"%Y_%m_%d") -QUAY_REPO=quay.io/freedomofpress/packaging-debian-bullseye - -set -e -set -x - -docker push "${QUAY_REPO}:${DATE_STR}" - -echo "# sha256 digest ${QUAY_REPO}:${DATE_STR}" > image_hash -docker inspect --format='{{index .RepoDigests 0}}' "${QUAY_REPO}:${DATE_STR}" \ - | sed 's/.*://g' >> image_hash diff --git a/dockerfiles/buster/Dockerfile b/dockerfiles/buster/Dockerfile deleted file mode 100644 index d20c909f..00000000 --- a/dockerfiles/buster/Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# We want to do things using Debian Buster's own Python -FROM debian:buster - -# make Apt non-interactive -RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90circleci \ - && echo 'DPkg::Options "--force-confnew";' >> /etc/apt/apt.conf.d/90circleci - -ENV DEBIAN_FRONTEND=noninteractive -# Make sure PATH includes ~/.local/bin -# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=839155 -# This only works for root. The circleci user is done near the end of this Dockerfile -RUN echo 'PATH="$HOME/.local/bin:$PATH"' >> /etc/profile.d/user-local-path.sh - -# man directory is missing in some base images -# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 -RUN apt-get update \ - && mkdir -p /usr/share/man/man1 \ - && apt-get install -y \ - git mercurial xvfb apt \ - locales sudo openssh-client ca-certificates tar gzip parallel \ - net-tools netcat unzip zip bzip2 gnupg curl wget make python3 python3-venv python3-pip - - -# Set timezone to UTC by default -RUN ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime - -# Use unicode -RUN locale-gen C.UTF-8 || true -ENV LANG=C.UTF-8 - -RUN groupadd --gid 3434 ci \ - && useradd --uid 3434 --gid ci --shell /bin/bash --create-home ci \ - && echo 'ci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-ci \ - && echo 'Defaults env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep - - - -USER ci -ENV PATH /home/ci/.local/bin:/home/ci/bin:${PATH} - -CMD ["/bin/sh"] diff --git a/dockerfiles/buster/Makefile b/dockerfiles/buster/Makefile deleted file mode 100644 index c9593739..00000000 --- a/dockerfiles/buster/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -DATE_STR := $(shell date +"%Y_%m_%d") -BUILDER_IMAGE ?= "quay.io/freedomofpress/packaging-debian-buster:$(DATE_STR)" - -.PHONY: build-container -build-container: ## Build Docker image for Debian Buster wheel and package creation - @echo "███Building Docker image $(BUILDER_IMAGE) for Debian Buster wheel and package creation" - @docker build --no-cache -t $(BUILDER_IMAGE) . - -.PHONY: push-container -push-container: ## Push the Docker image for Debian Buster wheel and package creation to quay.io - @echo "███Pushing Docker image for Debian package creation to quay.io..." - @./push.sh - -.PHONY: help -help: ## Print this message and exit. - @printf "Molecule scenario for building a Docker container for Debian package creation.\n" - @printf "Subcommands:\n\n" - @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \ - | sort \ - | column -s ':' -t diff --git a/dockerfiles/buster/image_hash b/dockerfiles/buster/image_hash deleted file mode 100644 index c549bd91..00000000 --- a/dockerfiles/buster/image_hash +++ /dev/null @@ -1,2 +0,0 @@ -# sha256 digest quay.io/freedomofpress/packaging-debian-buster:2021_07_22 -16d2df1935807c6a751d0536e3cb36970c4c22d7324915d25ee84c90b032c307 diff --git a/dockerfiles/buster/push.sh b/dockerfiles/buster/push.sh deleted file mode 100755 index 4d7a8123..00000000 --- a/dockerfiles/buster/push.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -DATE_STR=$(date +"%Y_%m_%d") -QUAY_REPO=quay.io/freedomofpress/packaging-debian-buster - -set -e -set -x - -docker push "${QUAY_REPO}:${DATE_STR}" - -echo "# sha256 digest ${QUAY_REPO}:${DATE_STR}" > image_hash -docker inspect --format='{{index .RepoDigests 0}}' "${QUAY_REPO}:${DATE_STR}" \ - | sed 's/.*://g' >> image_hash diff --git a/scripts/install-deps b/scripts/install-deps index 39a5c2a2..4b71fcd9 100755 --- a/scripts/install-deps +++ b/scripts/install-deps @@ -1,7 +1,10 @@ #!/bin/bash +set -euxo pipefail # Installs required dependencies for building SecureDrop Worsktation packages. # Assumes a Debian 10 machine, ideally a Qubes AppVM. +VIRTUAL_ENV="${VIRTUAL_ENV:-}" + sudo apt-get update sudo apt-get install \ build-essential \