From f4334b57b4605f5bc92cf19211194508cd6d3352 Mon Sep 17 00:00:00 2001 From: Kushal Das Date: Fri, 9 Jul 2021 17:18:11 +0530 Subject: [PATCH] Fixes #253 adds reproducible testing container This PR adds a new container and also uses it in the CircleCI to test reproducible wheels and Debian packages. The container is based on standard Debian Buster, thus having the same version of Python and environment. --- .circleci/config.yml | 6 ++---- .gitignore | 1 + dockerfiles/Dockerfile | 41 +++++++++++++++++++++++++++++++++++++++++ dockerfiles/Makefile | 20 ++++++++++++++++++++ dockerfiles/image_hash | 2 ++ dockerfiles/push.sh | 12 ++++++++++++ scripts/install-deps | 8 ++------ 7 files changed, 80 insertions(+), 10 deletions(-) create mode 100644 dockerfiles/Dockerfile create mode 100644 dockerfiles/Makefile create mode 100644 dockerfiles/image_hash create mode 100755 dockerfiles/push.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 8330ef5c..76ee9a8a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -234,14 +234,13 @@ jobs: reprotest-wheels: docker: - - image: circleci/python:3.7-buster + - image: quay.io/freedomofpress/packaging-debian-buster@sha256:5e1d3cf73ac6bfe418d499fc419bf789a0d365c51c157b704969d88d2df25e65 steps: - checkout - run: name: install test requirements and run tests command: | make install-deps - virtualenv -p /usr/bin/python3 .venv source .venv/bin/activate pip install -r test-requirements.txt sudo sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py @@ -249,14 +248,13 @@ jobs: reprotest-debs: docker: - - image: circleci/python:3.7-buster + - image: quay.io/freedomofpress/packaging-debian-buster@sha256:5e1d3cf73ac6bfe418d499fc419bf789a0d365c51c157b704969d88d2df25e65 steps: - checkout - run: name: install test requirements and run tests command: | make install-deps - virtualenv -p /usr/bin/python3 .venv source .venv/bin/activate pip install -r test-requirements.txt # Patch reprotest in-place to skip 'setarch' prefix, which fails under containers. diff --git a/.gitignore b/.gitignore index 7783665f..bee56b51 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ tests/__pycache__/ debhelper-build-stamp *.debhelper.log build/ +.venv diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile new file mode 100644 index 00000000..d20c909f --- /dev/null +++ b/dockerfiles/Dockerfile @@ -0,0 +1,41 @@ +# We want to do things using Debian Buster's own Python +FROM debian:buster + +# make Apt non-interactive +RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90circleci \ + && echo 'DPkg::Options "--force-confnew";' >> /etc/apt/apt.conf.d/90circleci + +ENV DEBIAN_FRONTEND=noninteractive +# Make sure PATH includes ~/.local/bin +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=839155 +# This only works for root. The circleci user is done near the end of this Dockerfile +RUN echo 'PATH="$HOME/.local/bin:$PATH"' >> /etc/profile.d/user-local-path.sh + +# man directory is missing in some base images +# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=863199 +RUN apt-get update \ + && mkdir -p /usr/share/man/man1 \ + && apt-get install -y \ + git mercurial xvfb apt \ + locales sudo openssh-client ca-certificates tar gzip parallel \ + net-tools netcat unzip zip bzip2 gnupg curl wget make python3 python3-venv python3-pip + + +# Set timezone to UTC by default +RUN ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime + +# Use unicode +RUN locale-gen C.UTF-8 || true +ENV LANG=C.UTF-8 + +RUN groupadd --gid 3434 ci \ + && useradd --uid 3434 --gid ci --shell /bin/bash --create-home ci \ + && echo 'ci ALL=NOPASSWD: ALL' >> /etc/sudoers.d/50-ci \ + && echo 'Defaults env_keep += "DEBIAN_FRONTEND"' >> /etc/sudoers.d/env_keep + + + +USER ci +ENV PATH /home/ci/.local/bin:/home/ci/bin:${PATH} + +CMD ["/bin/sh"] diff --git a/dockerfiles/Makefile b/dockerfiles/Makefile new file mode 100644 index 00000000..c9593739 --- /dev/null +++ b/dockerfiles/Makefile @@ -0,0 +1,20 @@ +DATE_STR := $(shell date +"%Y_%m_%d") +BUILDER_IMAGE ?= "quay.io/freedomofpress/packaging-debian-buster:$(DATE_STR)" + +.PHONY: build-container +build-container: ## Build Docker image for Debian Buster wheel and package creation + @echo "███Building Docker image $(BUILDER_IMAGE) for Debian Buster wheel and package creation" + @docker build --no-cache -t $(BUILDER_IMAGE) . + +.PHONY: push-container +push-container: ## Push the Docker image for Debian Buster wheel and package creation to quay.io + @echo "███Pushing Docker image for Debian package creation to quay.io..." + @./push.sh + +.PHONY: help +help: ## Print this message and exit. + @printf "Molecule scenario for building a Docker container for Debian package creation.\n" + @printf "Subcommands:\n\n" + @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \ + | sort \ + | column -s ':' -t diff --git a/dockerfiles/image_hash b/dockerfiles/image_hash new file mode 100644 index 00000000..0aee88be --- /dev/null +++ b/dockerfiles/image_hash @@ -0,0 +1,2 @@ +# sha256 digest quay.io/freedomofpress/packaging-debian-buster:2021_07_09 +7ac0e1e1c29d9a60e210e0da246a6d60e49c9eab18cf654bacf95ce5fed1413b diff --git a/dockerfiles/push.sh b/dockerfiles/push.sh new file mode 100755 index 00000000..4d7a8123 --- /dev/null +++ b/dockerfiles/push.sh @@ -0,0 +1,12 @@ +#!/bin/bash +DATE_STR=$(date +"%Y_%m_%d") +QUAY_REPO=quay.io/freedomofpress/packaging-debian-buster + +set -e +set -x + +docker push "${QUAY_REPO}:${DATE_STR}" + +echo "# sha256 digest ${QUAY_REPO}:${DATE_STR}" > image_hash +docker inspect --format='{{index .RepoDigests 0}}' "${QUAY_REPO}:${DATE_STR}" \ + | sed 's/.*://g' >> image_hash diff --git a/scripts/install-deps b/scripts/install-deps index 0dcbf735..f5db84f3 100755 --- a/scripts/install-deps +++ b/scripts/install-deps @@ -2,11 +2,6 @@ # Installs required dependencies for building SecureDrop Worsktation packages. # Assumes a Debian 10 machine, ideally a Qubes AppVM. -# If running in CI, we need to add the Ubuntu Bionic repo to download dh-virtualenv -if [[ -v CIRCLE_BUILD_URL ]]; then - echo "deb http://archive.ubuntu.com/ubuntu/ bionic universe" | sudo tee -a /etc/apt/sources.list -fi - sudo apt-get update sudo apt-get install \ build-essential \ @@ -21,9 +16,10 @@ sudo apt-get install \ libyaml-dev \ python3-all \ python3-pip \ + python3-venv \ python3-setuptools \ reprotest \ - desktop-file-utils + desktop-file-utils -y # Inspect the wheel files present locally. If repo was cloned # without git-lfs, they'll be "text/plain", rather than "application/zip".