From 3bf82735594b5ff841ae5d8dd3c31aa4ca1cffc9 Mon Sep 17 00:00:00 2001 From: Kushal Das Date: Tue, 5 Jan 2021 18:23:22 +0530 Subject: [PATCH 1/5] Uses wheels from local filesystem This patch stops accessing the webbased index for the wheels, instead uses the local filesystem path. --- Makefile | 6 +++--- securedrop-client/debian/rules | 4 +++- securedrop-log/debian/rules | 4 +++- securedrop-proxy/debian/rules | 4 +++- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 4b71524a..dda782cb 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,11 @@ DEFAULT_GOAL: help .PHONY: securedrop-proxy securedrop-proxy: ## Builds Debian package for securedrop-proxy code - PKG_NAME="securedrop-proxy" ./scripts/build-debianpackage + WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-proxy" ./scripts/build-debianpackage .PHONY: securedrop-client securedrop-client: ## Builds Debian package for securedrop-client code - PKG_NAME="securedrop-client" ./scripts/build-debianpackage + WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-client" ./scripts/build-debianpackage .PHONY: securedrop-workstation-config securedrop-workstation-config: ## Builds Debian metapackage for Qubes Workstation base dependencies @@ -30,7 +30,7 @@ securedrop-export: ## Builds Debian package for Qubes Workstation export scripts .PHONY: securedrop-log securedrop-log: ## Builds Debian package for Qubes Workstation securedrop-log scripts - PKG_NAME="securedrop-log" ./scripts/build-debianpackage + WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-log" ./scripts/build-debianpackage .PHONY: securedrop-keyring securedrop-keyring: ## Builds Debian package containing the release key diff --git a/securedrop-client/debian/rules b/securedrop-client/debian/rules index d4943183..06d3e29a 100755 --- a/securedrop-client/debian/rules +++ b/securedrop-client/debian/rules @@ -8,8 +8,10 @@ override_dh_virtualenv: --python /usr/bin/python3 \ --setuptools \ --use-system-packages \ - --index-url https://pypi.securedrop.org/simple \ --extra-pip-arg "--ignore-installed" \ + --extra-pip-arg "--no-index" \ + --extra-pip-arg "--find-links" \ + --extra-pip-arg "$(WHEELS_DIR)" \ --extra-pip-arg "--no-deps" \ --extra-pip-arg "--no-cache-dir" \ --extra-pip-arg "--no-use-pep517" \ diff --git a/securedrop-log/debian/rules b/securedrop-log/debian/rules index 9494ffd6..7df10ba9 100755 --- a/securedrop-log/debian/rules +++ b/securedrop-log/debian/rules @@ -7,8 +7,10 @@ override_dh_virtualenv: dh_virtualenv \ --python /usr/bin/python3 \ --setuptools \ - --index-url https://pypi.securedrop.org/simple \ --extra-pip-arg "--ignore-installed" \ + --extra-pip-arg "--no-index" \ + --extra-pip-arg "--find-links" \ + --extra-pip-arg "$(WHEELS_DIR)" \ --extra-pip-arg "--no-deps" \ --extra-pip-arg "--no-cache-dir" \ --extra-pip-arg "--no-use-pep517" \ diff --git a/securedrop-proxy/debian/rules b/securedrop-proxy/debian/rules index 9494ffd6..7df10ba9 100755 --- a/securedrop-proxy/debian/rules +++ b/securedrop-proxy/debian/rules @@ -7,8 +7,10 @@ override_dh_virtualenv: dh_virtualenv \ --python /usr/bin/python3 \ --setuptools \ - --index-url https://pypi.securedrop.org/simple \ --extra-pip-arg "--ignore-installed" \ + --extra-pip-arg "--no-index" \ + --extra-pip-arg "--find-links" \ + --extra-pip-arg "$(WHEELS_DIR)" \ --extra-pip-arg "--no-deps" \ --extra-pip-arg "--no-cache-dir" \ --extra-pip-arg "--no-use-pep517" \ From db62616b77673c51130c8a49d895080fd48ade75 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Tue, 5 Jan 2021 12:12:25 -0800 Subject: [PATCH 2/5] Minor tweaks to maintainer scripts Adjusting the error-handling based on a recent signing operation. Makes the tooling a bit friendlier for maintainers. --- scripts/sync-sha256sums | 4 ++-- scripts/update-requirements | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/sync-sha256sums b/scripts/sync-sha256sums index f0ff0b3b..ae473d6b 100755 --- a/scripts/sync-sha256sums +++ b/scripts/sync-sha256sums @@ -9,5 +9,5 @@ set -o pipefail cd ./localwheels/ sha256sum * > ../sha256sums.txt -printf "Now you must sign the generated sha256sums.txt file." -gpg --armor --output ../sha256sums.txt.asc --detach-sig ../sha256sums.txt +echo "Now you must sign the generated sha256sums.txt file:" +echo "gpg --armor --output ../sha256sums.txt.asc --detach-sig ../sha256sums.txt" diff --git a/scripts/update-requirements b/scripts/update-requirements index 85dca439..8da43568 100755 --- a/scripts/update-requirements +++ b/scripts/update-requirements @@ -11,7 +11,7 @@ from pprint import pprint def main(): - PKG_DIR = os.environ["PKG_DIR"] + PKG_DIR = os.environ.get("PKG_DIR", "") if not PKG_DIR: print("Set PKG_DIR of the project") From 1a69e39306d6fcf4727545e8d9b22b6b48900590 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Tue, 5 Jan 2021 11:52:05 -0800 Subject: [PATCH 3/5] Rebuilds all wheels, reproducibly In order to regenerate the wheel files, I ran: ./scripts/build-sync-wheels --clobber -p ../securedrop-client ./scripts/build-sync-wheels --clobber -p ../securedrop-log ./scripts/build-sync-wheels --clobber -p ../securedrop-proxy Each repo had the latest "main" branch commit checked out. Then I ran ./scripts/sync-sha256sums, completed the signing ceremony, and committed the results. --- localwheels/Mako-1.0.7-py3-none-any.whl | 2 +- ...rkupSafe-1.1.1-cp37-cp37m-linux_x86_64.whl | 3 ++ .../PyYAML-5.3.1-cp37-cp37m-linux_x86_64.whl | 4 +- ...LAlchemy-1.3.3-cp37-cp37m-linux_x86_64.whl | 4 +- .../Werkzeug-0.16.0-py2.py3-none-any.whl | 4 +- .../alembic-1.0.2-py2.py3-none-any.whl | 4 +- localwheels/arrow-0.12.1-py2.py3-none-any.whl | 4 +- .../certifi-2018.10.15-py2.py3-none-any.whl | 4 +- .../chardet-3.0.4-py2.py3-none-any.whl | 4 +- localwheels/furl-2.0.0-py2.py3-none-any.whl | 4 +- localwheels/idna-2.7-py2.py3-none-any.whl | 4 +- .../orderedmultidict-1.0-py3-none-any.whl | 2 +- .../pathlib2-2.3.2-py2.py3-none-any.whl | 2 +- ...python_dateutil-2.7.5-py2.py3-none-any.whl | 4 +- .../python_editor-1.0.3-py3-none-any.whl | 2 +- localwheels/redis-3.3.11-py2.py3-none-any.whl | 2 +- .../requests-2.22.0-py2.py3-none-any.whl | 4 +- .../securedrop_sdk-0.2.0-py3-none-any.whl | 4 +- localwheels/six-1.11.0-py2.py3-none-any.whl | 2 +- .../urllib3-1.25.10-py2.py3-none-any.whl | 4 +- sha256sums.txt | 39 ++++++++++--------- sha256sums.txt.asc | 26 ++++++------- 22 files changed, 68 insertions(+), 64 deletions(-) create mode 100644 localwheels/MarkupSafe-1.1.1-cp37-cp37m-linux_x86_64.whl diff --git a/localwheels/Mako-1.0.7-py3-none-any.whl b/localwheels/Mako-1.0.7-py3-none-any.whl index ccce96a8..ed7da175 100644 --- a/localwheels/Mako-1.0.7-py3-none-any.whl +++ b/localwheels/Mako-1.0.7-py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:614c22fe1a5b0a3f46f6c5c43ff2e6795e4e784328d559ec9dc49db0f06b3a75 +oid sha256:99d041a616a3655725dffe459916627b75640a7d045ed33f51ce158a168ca3ef size 76583 diff --git a/localwheels/MarkupSafe-1.1.1-cp37-cp37m-linux_x86_64.whl b/localwheels/MarkupSafe-1.1.1-cp37-cp37m-linux_x86_64.whl new file mode 100644 index 00000000..3756c602 --- /dev/null +++ b/localwheels/MarkupSafe-1.1.1-cp37-cp37m-linux_x86_64.whl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab68abcb133e25365454d0572e20c096cfebd3c350b51fe2cf8f54f495e0ca9d +size 33689 diff --git a/localwheels/PyYAML-5.3.1-cp37-cp37m-linux_x86_64.whl b/localwheels/PyYAML-5.3.1-cp37-cp37m-linux_x86_64.whl index 9df74948..df294bdb 100644 --- a/localwheels/PyYAML-5.3.1-cp37-cp37m-linux_x86_64.whl +++ b/localwheels/PyYAML-5.3.1-cp37-cp37m-linux_x86_64.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:cb4442140d3195f5f799096aa35aadce15f493046135a03668023b80824dd44c -size 44622 +oid sha256:ff021f812a5dbaeb10678462952bb447bf6b322ab54923f5591811056a827c15 +size 44620 diff --git a/localwheels/SQLAlchemy-1.3.3-cp37-cp37m-linux_x86_64.whl b/localwheels/SQLAlchemy-1.3.3-cp37-cp37m-linux_x86_64.whl index 5d050626..6196c102 100644 --- a/localwheels/SQLAlchemy-1.3.3-cp37-cp37m-linux_x86_64.whl +++ b/localwheels/SQLAlchemy-1.3.3-cp37-cp37m-linux_x86_64.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:bfb4cd0df5802a01acd738a080a19e60ff4700e030d497de273807f9a8bd4a0a -size 1134959 +oid sha256:a06165a69eab0db62d673c84fd021b209d1036577e7ca7a9e6e16455fc15fc56 +size 1187841 diff --git a/localwheels/Werkzeug-0.16.0-py2.py3-none-any.whl b/localwheels/Werkzeug-0.16.0-py2.py3-none-any.whl index eb366dc4..fee64aae 100644 --- a/localwheels/Werkzeug-0.16.0-py2.py3-none-any.whl +++ b/localwheels/Werkzeug-0.16.0-py2.py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:429de1b931a2a58bf5cfac8447253949f7a930d30a73f2755e0ad0f9824592bf -size 327276 +oid sha256:bd05301a84a9bc3b33f178e53446181879744b74f098ed35850ba21125379be1 +size 327277 diff --git a/localwheels/alembic-1.0.2-py2.py3-none-any.whl b/localwheels/alembic-1.0.2-py2.py3-none-any.whl index da1a4b1e..ba56435a 100644 --- a/localwheels/alembic-1.0.2-py2.py3-none-any.whl +++ b/localwheels/alembic-1.0.2-py2.py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:99cc931e11dbef6e41e9376f18be62fc90fe4be9c541eac1b30a3455b3d655f3 -size 156221 +oid sha256:ad1e93816f37f8445bb12dfe412579d2237818c9bbf34054a06e00d5f430c700 +size 156219 diff --git a/localwheels/arrow-0.12.1-py2.py3-none-any.whl b/localwheels/arrow-0.12.1-py2.py3-none-any.whl index 2f447b02..59ad262a 100644 --- a/localwheels/arrow-0.12.1-py2.py3-none-any.whl +++ b/localwheels/arrow-0.12.1-py2.py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:fc8c8e0b587d00f38986bc161f4496e000acea033fe2ce25f4f5bffa9ae53a7c -size 35209 +oid sha256:e2742eb33011f7aff1d5f27501d802680b6627939f20ff2ef56f050c14af59fa +size 35207 diff --git a/localwheels/certifi-2018.10.15-py2.py3-none-any.whl b/localwheels/certifi-2018.10.15-py2.py3-none-any.whl index 50c1923c..bd7814a6 100644 --- a/localwheels/certifi-2018.10.15-py2.py3-none-any.whl +++ b/localwheels/certifi-2018.10.15-py2.py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:173b19dd31ca7faa50d1fcc0eaf30f5e32e8e99e17d8c7fd4cfc8bc8d94e18a6 -size 144599 +oid sha256:2d5538b9d89b3a9cb423b8f08417c657856cab733d560067e864157f71b1c5aa +size 144596 diff --git a/localwheels/chardet-3.0.4-py2.py3-none-any.whl b/localwheels/chardet-3.0.4-py2.py3-none-any.whl index 2f05dca9..d2470714 100644 --- a/localwheels/chardet-3.0.4-py2.py3-none-any.whl +++ b/localwheels/chardet-3.0.4-py2.py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f5632e583a4f61f1e16d0cc98127d241fb11c3c6ddfddee159307d4215186837 -size 140898 +oid sha256:e5cf39014befb85add77118fdc946f0a3387df7840235ba5d20fd6e3a672410a +size 140896 diff --git a/localwheels/furl-2.0.0-py2.py3-none-any.whl b/localwheels/furl-2.0.0-py2.py3-none-any.whl index 782e2669..cdfff508 100644 --- a/localwheels/furl-2.0.0-py2.py3-none-any.whl +++ b/localwheels/furl-2.0.0-py2.py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:1855003e64dcb934556ad79994ba1a3a852da337e353d84d3b4ef75031913451 -size 19704 +oid sha256:9f50360f6e4a0f1d0a35fb4997878e7186a73331f0fde5f6fc9b1bb9f006e6cc +size 19702 diff --git a/localwheels/idna-2.7-py2.py3-none-any.whl b/localwheels/idna-2.7-py2.py3-none-any.whl index 8a97e46d..e4dd7fed 100644 --- a/localwheels/idna-2.7-py2.py3-none-any.whl +++ b/localwheels/idna-2.7-py2.py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:491f674364ba3232ed1eb4c1eb7407887f62cef6c300aad7df6e01acd88ffb25 -size 58210 +oid sha256:a967a752dba5492be34f30f8633cee512d2ec923fe3a2e201d87353b540d2955 +size 58211 diff --git a/localwheels/orderedmultidict-1.0-py3-none-any.whl b/localwheels/orderedmultidict-1.0-py3-none-any.whl index 3f8ae359..8f0bc7a8 100644 --- a/localwheels/orderedmultidict-1.0-py3-none-any.whl +++ b/localwheels/orderedmultidict-1.0-py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:51efddca0b4ae6d885bbafd8ca44e51758166c144cf006dbead5c9394b2a9eae +oid sha256:f6022beda2b3387c61e6eb7e0e1e3e2832fd9f55f3f64d4b4b226eea7487327f size 10992 diff --git a/localwheels/pathlib2-2.3.2-py2.py3-none-any.whl b/localwheels/pathlib2-2.3.2-py2.py3-none-any.whl index 6e300aa8..6a399b5a 100644 --- a/localwheels/pathlib2-2.3.2-py2.py3-none-any.whl +++ b/localwheels/pathlib2-2.3.2-py2.py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:460e67b14d0574b0529a0017b1eb05d10d9722681e303fec7077c2a628de60c1 +oid sha256:90173e12465846173da76c62892b238c14a2a0e17aae580933041004fc01b713 size 15944 diff --git a/localwheels/python_dateutil-2.7.5-py2.py3-none-any.whl b/localwheels/python_dateutil-2.7.5-py2.py3-none-any.whl index bef51877..78d63c13 100644 --- a/localwheels/python_dateutil-2.7.5-py2.py3-none-any.whl +++ b/localwheels/python_dateutil-2.7.5-py2.py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f6eb9c17acd5a6954e1a5f2f999a41de3e7e25b6bc41baf6344bd053ec25ceeb -size 225680 +oid sha256:2961735b604e5165d923a4e39b9bcbde6a4c363f257f5477b6339459fd7c1679 +size 225692 diff --git a/localwheels/python_editor-1.0.3-py3-none-any.whl b/localwheels/python_editor-1.0.3-py3-none-any.whl index 09d500f6..3f17eb45 100644 --- a/localwheels/python_editor-1.0.3-py3-none-any.whl +++ b/localwheels/python_editor-1.0.3-py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e47dcec4ea883853b8196fbd425b875d7ec791d4ede2e20cfc70b9a25365c65b +oid sha256:42e5b2a7583cd278dff00ae1fc6b8b735da0502ceeb0914991eaaf5f91ac84ac size 6687 diff --git a/localwheels/redis-3.3.11-py2.py3-none-any.whl b/localwheels/redis-3.3.11-py2.py3-none-any.whl index 338e7b25..0ac4b886 100644 --- a/localwheels/redis-3.3.11-py2.py3-none-any.whl +++ b/localwheels/redis-3.3.11-py2.py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:022f124431ae16ee3a3a69c8016e3e2b057b4f4e0bfa7787b6271d893890c3cc +oid sha256:974926675d246ade47a101e305596a0f50282a405a03b7173a70d99c79e6370c size 66739 diff --git a/localwheels/requests-2.22.0-py2.py3-none-any.whl b/localwheels/requests-2.22.0-py2.py3-none-any.whl index 231fdc84..0b9ca993 100644 --- a/localwheels/requests-2.22.0-py2.py3-none-any.whl +++ b/localwheels/requests-2.22.0-py2.py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:e3711bd465fd013abe428a8dade2938b578f05d2b06b0ae83daa98869c4548e8 -size 57951 +oid sha256:83ddbb326afd5524e7fbca582fd9673103652ea3b0c6601ac5ba1a4501f077c9 +size 57952 diff --git a/localwheels/securedrop_sdk-0.2.0-py3-none-any.whl b/localwheels/securedrop_sdk-0.2.0-py3-none-any.whl index c1cc402f..61c3b212 100644 --- a/localwheels/securedrop_sdk-0.2.0-py3-none-any.whl +++ b/localwheels/securedrop_sdk-0.2.0-py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:dd0231e1b873197373a7e84e3610370634eead601223551153eca7cdd166e7d5 -size 23803 +oid sha256:bb949ed7f13d88ae9e269d2c8d037ecf52d36d864fbeea61dd13f4049a978bc4 +size 23804 diff --git a/localwheels/six-1.11.0-py2.py3-none-any.whl b/localwheels/six-1.11.0-py2.py3-none-any.whl index e6b47043..8d2b2c8e 100644 --- a/localwheels/six-1.11.0-py2.py3-none-any.whl +++ b/localwheels/six-1.11.0-py2.py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:aa4ad34049ddff178b533062797fd1db9f0038b7c5c2461a7cde2244300b9f3d +oid sha256:eb52689b06ca7433c1cac3b91f320400bd3b358790b7ff4b6367cb1c81d37561 size 10173 diff --git a/localwheels/urllib3-1.25.10-py2.py3-none-any.whl b/localwheels/urllib3-1.25.10-py2.py3-none-any.whl index 35e766e5..dec8f7ea 100644 --- a/localwheels/urllib3-1.25.10-py2.py3-none-any.whl +++ b/localwheels/urllib3-1.25.10-py2.py3-none-any.whl @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:32bcd1ec52a4dd17ede2725ad166c789f50d402870947d6824598915b89ecf70 -size 127632 +oid sha256:c78fdeffd1a01640ea99d35851539a4bc91e46a672989c4d96663e3808121389 +size 127631 diff --git a/sha256sums.txt b/sha256sums.txt index 554a50e9..b748f524 100644 --- a/sha256sums.txt +++ b/sha256sums.txt @@ -1,38 +1,39 @@ -99cc931e11dbef6e41e9376f18be62fc90fe4be9c541eac1b30a3455b3d655f3 alembic-1.0.2-py2.py3-none-any.whl +ad1e93816f37f8445bb12dfe412579d2237818c9bbf34054a06e00d5f430c700 alembic-1.0.2-py2.py3-none-any.whl 04bcb970ca8659c3607ddd8ffd86cc9d6a99661c9bc590955e8813c66bfa582b alembic-1.0.2.tar.gz -fc8c8e0b587d00f38986bc161f4496e000acea033fe2ce25f4f5bffa9ae53a7c arrow-0.12.1-py2.py3-none-any.whl +e2742eb33011f7aff1d5f27501d802680b6627939f20ff2ef56f050c14af59fa arrow-0.12.1-py2.py3-none-any.whl a558d3b7b6ce7ffc74206a86c147052de23d3d4ef0e17c210dd478c53575c4cd arrow-0.12.1.tar.gz -173b19dd31ca7faa50d1fcc0eaf30f5e32e8e99e17d8c7fd4cfc8bc8d94e18a6 certifi-2018.10.15-py2.py3-none-any.whl +2d5538b9d89b3a9cb423b8f08417c657856cab733d560067e864157f71b1c5aa certifi-2018.10.15-py2.py3-none-any.whl 6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a certifi-2018.10.15.tar.gz -f5632e583a4f61f1e16d0cc98127d241fb11c3c6ddfddee159307d4215186837 chardet-3.0.4-py2.py3-none-any.whl +e5cf39014befb85add77118fdc946f0a3387df7840235ba5d20fd6e3a672410a chardet-3.0.4-py2.py3-none-any.whl 84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae chardet-3.0.4.tar.gz -1855003e64dcb934556ad79994ba1a3a852da337e353d84d3b4ef75031913451 furl-2.0.0-py2.py3-none-any.whl +9f50360f6e4a0f1d0a35fb4997878e7186a73331f0fde5f6fc9b1bb9f006e6cc furl-2.0.0-py2.py3-none-any.whl fdcaedc1fb19a63d7d875b0105b0a5b496dd0989330d454a42bcb401fa5454ec furl-2.0.0.tar.gz -491f674364ba3232ed1eb4c1eb7407887f62cef6c300aad7df6e01acd88ffb25 idna-2.7-py2.py3-none-any.whl +a967a752dba5492be34f30f8633cee512d2ec923fe3a2e201d87353b540d2955 idna-2.7-py2.py3-none-any.whl 684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16 idna-2.7.tar.gz -614c22fe1a5b0a3f46f6c5c43ff2e6795e4e784328d559ec9dc49db0f06b3a75 Mako-1.0.7-py3-none-any.whl +99d041a616a3655725dffe459916627b75640a7d045ed33f51ce158a168ca3ef Mako-1.0.7-py3-none-any.whl 4e02fde57bd4abb5ec400181e4c314f56ac3e49ba4fb8b0d50bba18cb27d25ae Mako-1.0.7.tar.gz c6b726d2e9d6300a044cf6a37627f10994268d6ac39464bc0d725126609311a5 MarkupSafe-1.0-py3-none-any.whl a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665 MarkupSafe-1.0.tar.gz +ab68abcb133e25365454d0572e20c096cfebd3c350b51fe2cf8f54f495e0ca9d MarkupSafe-1.1.1-cp37-cp37m-linux_x86_64.whl 1733a3715d37d075d79b6fe1c2e1055e8cf471fd7da53cdd284de09782bf75d1 MarkupSafe-1.1.1-py3-none-any.whl 29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b MarkupSafe-1.1.1.tar.gz -51efddca0b4ae6d885bbafd8ca44e51758166c144cf006dbead5c9394b2a9eae orderedmultidict-1.0-py3-none-any.whl +f6022beda2b3387c61e6eb7e0e1e3e2832fd9f55f3f64d4b4b226eea7487327f orderedmultidict-1.0-py3-none-any.whl b89895ba6438038d0bdf88020ceff876cf3eae0d5c66a69b526fab31125db2c5 orderedmultidict-1.0.tar.gz -460e67b14d0574b0529a0017b1eb05d10d9722681e303fec7077c2a628de60c1 pathlib2-2.3.2-py2.py3-none-any.whl +90173e12465846173da76c62892b238c14a2a0e17aae580933041004fc01b713 pathlib2-2.3.2-py2.py3-none-any.whl 8eb170f8d0d61825e09a95b38be068299ddeda82f35e96c3301a8a5e7604cb83 pathlib2-2.3.2.tar.gz -f6eb9c17acd5a6954e1a5f2f999a41de3e7e25b6bc41baf6344bd053ec25ceeb python_dateutil-2.7.5-py2.py3-none-any.whl +2961735b604e5165d923a4e39b9bcbde6a4c363f257f5477b6339459fd7c1679 python_dateutil-2.7.5-py2.py3-none-any.whl 88f9287c0174266bb0d8cedd395cfba9c58e87e5ad86b2ce58859bc11be3cf02 python-dateutil-2.7.5.tar.gz -e47dcec4ea883853b8196fbd425b875d7ec791d4ede2e20cfc70b9a25365c65b python_editor-1.0.3-py3-none-any.whl +42e5b2a7583cd278dff00ae1fc6b8b735da0502ceeb0914991eaaf5f91ac84ac python_editor-1.0.3-py3-none-any.whl a3c066acee22a1c94f63938341d4fb374e3fdd69366ed6603d7b24bed1efc565 python-editor-1.0.3.tar.gz 59ff9ed67bd8be210f91fd20f0ef844ee897d11eb4c19c611a77a37896c124b9 PyYAML-5.1-cp37-cp37m-linux_x86_64.whl 436bc774ecf7c103814098159fbb84c2715d25980175292c648f2da143909f95 PyYAML-5.1.tar.gz -cb4442140d3195f5f799096aa35aadce15f493046135a03668023b80824dd44c PyYAML-5.3.1-cp37-cp37m-linux_x86_64.whl +ff021f812a5dbaeb10678462952bb447bf6b322ab54923f5591811056a827c15 PyYAML-5.3.1-cp37-cp37m-linux_x86_64.whl b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d PyYAML-5.3.1.tar.gz -022f124431ae16ee3a3a69c8016e3e2b057b4f4e0bfa7787b6271d893890c3cc redis-3.3.11-py2.py3-none-any.whl +974926675d246ade47a101e305596a0f50282a405a03b7173a70d99c79e6370c redis-3.3.11-py2.py3-none-any.whl 8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2 redis-3.3.11.tar.gz d87b2085783d31d874ac7bc62660e287932aaee7059e80b41b76462eb18d35cc requests-2.20.0-py2.py3-none-any.whl 99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c requests-2.20.0.tar.gz -e3711bd465fd013abe428a8dade2938b578f05d2b06b0ae83daa98869c4548e8 requests-2.22.0-py2.py3-none-any.whl +83ddbb326afd5524e7fbca582fd9673103652ea3b0c6601ac5ba1a4501f077c9 requests-2.22.0-py2.py3-none-any.whl 11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 requests-2.22.0.tar.gz d05bb78652c8771e6aa1aefcd76ade1fef08c563d2641acbc5ac8e1d635e6a53 securedrop_sdk-0.0.12-py3-none-any.whl b5ddca26ce87d4007db5d64fe77d44b4086a902c3f79e69fb9a81343c81ce278 securedrop-sdk-0.0.12.tar.gz @@ -42,15 +43,15 @@ c8d98208fb2074336c06be3fef0994a8a57fde7a765cead12bc36e9128d319e2 securedrop_sdk 970fde25e6238e1808ac120951ee972549f4cd7952966dfe29f731bb308cc0d8 securedrop-sdk-0.1.0.tar.gz a631495acd741ab568410287879c5a3af3ccd38e00a2f3a127cc6b27cba99392 securedrop_sdk-0.1.1-py3-none-any.whl 138ce7a717db519c3c8d19b9475d7660fb7095d7608e8802723e682a7415e677 securedrop-sdk-0.1.1.tar.gz -dd0231e1b873197373a7e84e3610370634eead601223551153eca7cdd166e7d5 securedrop_sdk-0.2.0-py3-none-any.whl +bb949ed7f13d88ae9e269d2c8d037ecf52d36d864fbeea61dd13f4049a978bc4 securedrop_sdk-0.2.0-py3-none-any.whl c4a343077e8c0a38914e17f6369b830f1e361f9d66699b20803c07b39472357f securedrop-sdk-0.2.0.tar.gz -aa4ad34049ddff178b533062797fd1db9f0038b7c5c2461a7cde2244300b9f3d six-1.11.0-py2.py3-none-any.whl +eb52689b06ca7433c1cac3b91f320400bd3b358790b7ff4b6367cb1c81d37561 six-1.11.0-py2.py3-none-any.whl 70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 six-1.11.0.tar.gz -bfb4cd0df5802a01acd738a080a19e60ff4700e030d497de273807f9a8bd4a0a SQLAlchemy-1.3.3-cp37-cp37m-linux_x86_64.whl +a06165a69eab0db62d673c84fd021b209d1036577e7ca7a9e6e16455fc15fc56 SQLAlchemy-1.3.3-cp37-cp37m-linux_x86_64.whl 91c54ca8345008fceaec987e10924bf07dcab36c442925357e5a467b36a38319 SQLAlchemy-1.3.3.tar.gz 3d440cbb168e2c963d5099232bdb3f7390bf031b6270dad1bc79751698a1399a urllib3-1.24.3-py2.py3-none-any.whl 2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4 urllib3-1.24.3.tar.gz -32bcd1ec52a4dd17ede2725ad166c789f50d402870947d6824598915b89ecf70 urllib3-1.25.10-py2.py3-none-any.whl +c78fdeffd1a01640ea99d35851539a4bc91e46a672989c4d96663e3808121389 urllib3-1.25.10-py2.py3-none-any.whl 91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a urllib3-1.25.10.tar.gz -429de1b931a2a58bf5cfac8447253949f7a930d30a73f2755e0ad0f9824592bf Werkzeug-0.16.0-py2.py3-none-any.whl +bd05301a84a9bc3b33f178e53446181879744b74f098ed35850ba21125379be1 Werkzeug-0.16.0-py2.py3-none-any.whl 7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7 Werkzeug-0.16.0.tar.gz diff --git a/sha256sums.txt.asc b/sha256sums.txt.asc index 9d89cd0c..87022f7d 100644 --- a/sha256sums.txt.asc +++ b/sha256sums.txt.asc @@ -1,16 +1,16 @@ -----BEGIN PGP SIGNATURE----- -iQIzBAABCAAdFiEEIiRcgeO660E4s2BhMQ9WEgD0rXcFAl+ZcJAACgkQMQ9WEgD0 -rXfufA//QAaZnb1lGKb+yMl6+Zl463Ivb4vWgBGM2DssvuQ+MSDGE9+2sSID5sTx -WctXnsU2F276p60tOCp+wEIH6/WqZHtNRPrXxlnDF9uSDXFW2SuX2ByVXWCp3Z6A -B5Kpd600JyOg/+EKRI97t5SWnLUZkTnofKepVJUdhuv249Ol7nvKAW/6g2xqnNd8 -0UaLa24Zam+A/944+pp83m5o0fb8W57LjPtuGt8ubPYyvZRHClkaXheRJs3qia9Q -i0zlwp3F47YbVqdBNBFYdpUfp5uSfRYeG8YF2NbINWkOn4hyaYkMpVnUIFlUkGrZ -Fx7B9HgKJSp5UhsaWXap57q+WEzM77ctzOPMz894hC880z39Yu2I1XunVUUw/PDJ -pB4MhIhzv4paQsy9C18BZPD3lOaWit/xzHIvf02OUCJneBfoFDA63CSZvOMm7bkR -2L8TkEaO8B/xzyEviIrdvObAte4eTwMrAE4rE+Dfg/8iMiwlrVd7BCs9JnZCrF45 -aDAirmZkK9zxCPanu38IMslgDsCsqvId7rfqRF3d5d7UhOzzqb4JuR6TddN1M0Uy -4rBixAIssJ0wwcmOZ4uMA35decsJactf11zcuhklZnAFA5PiEYTPUoJHMhRb++N6 -+xFUveAcYqSLPIAza0s0/uekPBsbVN5Fjp3yymnFVqk6RuNI6DA= -=+P35 +iQIzBAABCgAdFiEEIiRcgeO660E4s2BhMQ9WEgD0rXcFAl/0xV0ACgkQMQ9WEgD0 +rXd0cQ/+IPjJjH2ZOS3OE5L78xk5Bi8FXYrbNixCWqigN+yvR11bxlDg6hJlUxUt +XbKtO0iaY4Il4LbuusYkvE3UB5ay89uN3iSMNX2P/II+k6ch/rn/a/94otlOjWtk +lPoJea2MoB2eI34NTaS6O77CxHciLHTFgD6YFAKINbNZIAtgrHMUXiEGkYxu+qpR +73tFHPreTHEo/FTzph3uwySTUbfhBnOy6IOc/WoKu1vgvLr2yiiSNHN9TDZYh/PR +0+DsMHJGj9NcnJ4a7n7tPxQlSbm8eX7QC5lsC4oarO08nGkY/+IujDPpUYhtNczu +RHvNa1YpUTOQ8RxXsFepmX85TnwVcjYNNkHvsK2yR73i+LYu2NUXe5RbHS0ZrvKg +EZaliwR3IKrcIw6saOez1r32DVGt7rtX7FCkLYJOVZ0pXaFqV3FnGcYxiUAmvwns +WvDJnVE7R13QCzPX/uPchhj2Kzxx+HHUyIUsJc3MX7to0jrcl1dCTV3VSJce0uru +FAkdpoQVTRICvOTemENYGTQk0i8UKEBl/vLaviA4pTZ4mtW6/39eO/WCGBM2sMT5 +pkKtWRdCXrhYY/Cy3TuQ3QBREW0SJkr7hfmtHApu1L6WoBlXmv71iYVC0n9v9FER +tYydlhIAGmqNPD4J+oKS3yK0TFqw4B/qtL33DDWJFmAOqCfBXo4= +=PZ1C -----END PGP SIGNATURE----- From 18770bd44ec54da86f436515da4452c05111605e Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Mon, 11 Jan 2021 14:56:57 +0530 Subject: [PATCH 4/5] Adds reprotest for .deb files More pytest-based reprotest invocations, this time focusing on .deb files. Replaces the CircleCI repro tests that manually compared hashes. Also modifies build script to support commit hash The logic assumed we were always building from a prod release tag. As a result, the CI logic was reimplementing the tarball mangling. Let's make the script more flexible, so we can run the script in CI and thereby get a bit more test coverage for it. Modifies CI env for reprotest support When building .deb packages, we need the python version for the packaging environment to match that of the target platform, i.e. python3.7 for buster. In CI, our platform options are: * VM, Ubuntu 20.04 * Container, Debian 10 The container driver in CircleCI does not permit "setarch" calls, erroring out immediately. The setarch calls are not optional in reprotest, unfortunately, so let's hack the file and remove it entirely, only in CI. --- .circleci/config.yml | 106 ++++++++------------- .gitignore | 1 + Makefile | 8 +- build/.gitkeep | 0 scripts/build-debianpackage | 17 ++-- test-requirements.txt | 3 +- tests/test_reproducible_debian_packages.py | 48 ++++++++++ tests/test_reproducible_wheels.py | 5 +- 8 files changed, 108 insertions(+), 80 deletions(-) create mode 100644 build/.gitkeep create mode 100644 tests/test_reproducible_debian_packages.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 69e18859..d492c9f0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -138,30 +138,8 @@ common-steps: export PKG_PATH=~/packaging/$PKG_NAME/dist/$PKG_NAME-$VERSION_TO_BUILD.tar.gz export PKG_VERSION=$VERSION_TO_BUILD make $PKG_NAME - ls ~/debbuild/packaging/*.deb + ls ~/project/build/debbuild/packaging/*.deb - - &builddebianpackagefromexistingtarball - run: - name: Build debian package from committed tarball - command: | - export PKG_PATH=~/project/tarballs/$PKG_NAME-$PKG_VERSION.tar.gz - - # Every tarball should be signed - gpg --import ~/project/pubkeys/release_key.pub - gpg --verify $PKG_PATH.asc - - # Build debian package - make $PKG_NAME - export PKG_HASH_1=$(shasum -a 256 ~/debbuild/packaging/$PKG_NAME*.deb | awk '{print $1}') - echo $PKG_HASH_1 - - # Build debian package again - make $PKG_NAME - export PKG_HASH_2=$(shasum -a 256 ~/debbuild/packaging/$PKG_NAME*.deb | awk '{print $1}') - echo $PKG_HASH_2 - - # Fail build if hashes aren’t equal - python -c "import os, sys; sys.exit(os.environ['PKG_HASH_1'] != os.environ['PKG_HASH_2'])" - &addsshkeys add_ssh_keys: @@ -294,6 +272,38 @@ jobs: pip install -r test-requirements.txt make test + reprotest-wheels: + machine: + image: ubuntu-2004:202010-01 + steps: + - checkout + - run: + name: install test requirements and run tests + command: | + make install-deps + virtualenv -p python3 .venv + source .venv/bin/activate + pip install -r test-requirements.txt + pytest -vvs tests/test_reproducible_wheels.py + + reprotest-debs: + docker: + - image: circleci/python:3.7-buster + steps: + - checkout + - run: + name: install test requirements and run tests + command: | + make install-deps + virtualenv -p python3 .venv + source .venv/bin/activate + pip install -r test-requirements.txt + # Patch reprotest in-place to skip 'setarch' prefix, which fails under containers. + # We cannot use Ubuntu 20.04 python3.8 to build Debian 10 python3.7 packages. + sudo sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py + pytest -vvs tests/test_reproducible_debian_packages.py + + build-buster-securedrop-log: docker: - image: circleci/python:3.7-buster @@ -302,7 +312,7 @@ jobs: - *removevirtualenv - *installdeps - *clonesecuredroplog - - *getlatestreleasedversion + - *getnightlyversion - *makesourcetarball - *builddebianpackage @@ -329,7 +339,7 @@ jobs: - *removevirtualenv - *installdeps - *clonesecuredropclient - - *getlatestreleasedversion + - *getnightlyversion - *makesourcetarball - *builddebianpackage @@ -356,7 +366,7 @@ jobs: - *removevirtualenv - *installdeps - *clonesecuredropproxy - - *getlatestreleasedversion + - *getnightlyversion - *makesourcetarball - *builddebianpackage @@ -481,51 +491,12 @@ jobs: - *setmetapackageversion - *builddebianpackage - reproducibility-checks: - docker: - - image: circleci/python:3.7-buster - steps: - - checkout - - *removevirtualenv - - *installdeps - - run: git lfs pull - - run: - name: Test build process reproducibility on latest securedrop-client tarball - command: | - export TARBALL=$(ls ~/project/tarballs/securedrop-client-*.tar.gz) - echo ${TARBALL%.tar.gz} | awk -F "-" '{ print $3 }' > ~/sd_version - echo 'export PKG_NAME=securedrop-client' >> $BASH_ENV - echo 'export PKG_VERSION=$(cat ~/sd_version)' >> $BASH_ENV - - *builddebianpackagefromexistingtarball - - run: - name: Test build process reproducibility on latest securedrop-proxy tarball - command: | - export TARBALL=$(ls ~/project/tarballs/securedrop-proxy-*.tar.gz) - echo ${TARBALL%.tar.gz} | awk -F "-" '{ print $3 }' > ~/sd_version - echo 'export PKG_NAME=securedrop-proxy' >> $BASH_ENV - echo 'export PKG_VERSION=$(cat ~/sd_version)' >> $BASH_ENV - - *builddebianpackagefromexistingtarball - - run: - name: Test build process reproducibility on latest securedrop-log tarball - command: | - export TARBALL=$(ls ~/project/tarballs/securedrop-log-*.tar.gz) - echo ${TARBALL%.tar.gz} | awk -F "-" '{ print $3 }' > ~/sd_version - echo 'export PKG_NAME=securedrop-log' >> $BASH_ENV - echo 'export PKG_VERSION=$(cat ~/sd_version)' >> $BASH_ENV - - *builddebianpackagefromexistingtarball - - run: - name: Test build process reproducibility on latest securedrop-export tarball - command: | - export TARBALL=$(ls ~/project/tarballs/securedrop-export-*.tar.gz) - echo ${TARBALL%.tar.gz} | awk -F "-" '{ print $3 }' > ~/sd_version - echo 'export PKG_NAME=securedrop-export' >> $BASH_ENV - echo 'export PKG_VERSION=$(cat ~/sd_version)' >> $BASH_ENV - - *builddebianpackagefromexistingtarball - workflows: build-packages: jobs: - tests + - reprotest-wheels + - reprotest-debs - build-buster-securedrop-client - build-buster-securedrop-proxy - build-buster-securedrop-workstation-svs-disp @@ -535,7 +506,6 @@ workflows: - build-buster-securedrop-workstation-config - build-buster-securedrop-keyring - make-dom0-rpm - - reproducibility-checks # Nightly jobs for each package are run in series to ensure there are no # conflicts or race conditions when committing deb packages to git-lfs. diff --git a/.gitignore b/.gitignore index 57679247..7783665f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ tests/__pycache__/ debhelper-build-stamp *.debhelper.log +build/ diff --git a/Makefile b/Makefile index dda782cb..dfc0e202 100644 --- a/Makefile +++ b/Makefile @@ -54,16 +54,16 @@ build-wheels: ## Builds the wheels and adds them to the localwheels directory @printf "to push these changes to the FPF PyPI index\n" .PHONY: test -test: ## Run test suite - pytest -v tests/ +test: ## Run simple test suite (skips reproducibility checks) + pytest -v tests/test_update_requirements.py .PHONY: clean clean: ## Removes all non-version controlled packaging artifacts rm -rf localwheels/* .PHONY: reprotest -reprotest: ## Reproducibility test, currently only for wheels - pytest -vvs tests/test_reproducible_wheels.py +reprotest: ## Runs only reproducibility tests, for .deb and .whl files + pytest -vvs tests/test_reproducible_*.py .PHONY: help help: ## Prints this message and exits diff --git a/build/.gitkeep b/build/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/scripts/build-debianpackage b/scripts/build-debianpackage index befc8498..3b7e521b 100755 --- a/scripts/build-debianpackage +++ b/scripts/build-debianpackage @@ -23,12 +23,12 @@ export DH_PIP_EXTRA_ARGS="--no-cache-dir --require-hashes" # Declare general packaging building workspace; subdirs will # be created within, to build specific packages. -TOP_BUILDDIR="$HOME/debbuild/packaging" +TOP_BUILDDIR="$PWD/build/debbuild/packaging" mkdir -p "$TOP_BUILDDIR" rm -rf "${TOP_BUILDDIR:?}/${PKG_NAME}" mkdir -p "${TOP_BUILDDIR}/${PKG_NAME}" # Move changelog into place (we have separate changelogs for each platform) -PLATFORM="$(lsb_release -sc)" +PLATFORM="${PKG_PLATFORM:-buster}" # Validate required args. if [[ -z "${PKG_NAME:-}" ]]; then @@ -83,11 +83,16 @@ function build_source_tarball() { rm -rf "$build_dir" git clone "$repo_url" "$build_dir" - # Verify tag, using only the prod key - verify_git_tag "$build_dir" "$PKG_VERSION" + if [[ -n "$PKG_GITREF" ]]; then + # Can't expect a prod sig on a gitref, likely a feature branch + git -C "$build_dir" checkout "$PKG_GITREF" + else + # Verify tag, using only the prod key + verify_git_tag "$build_dir" "$PKG_VERSION" + # Tag is verified, proceed with checkout + git -C "$build_dir" checkout "$PKG_VERSION" + fi - # Tag is verified, proceed with checkout - git -C "$build_dir" checkout "$PKG_VERSION" (cd "$build_dir" && LC_ALL="C.UTF-8" python setup.py sdist) # Initial tarball will contain timestamps from NOW, let's repack diff --git a/test-requirements.txt b/test-requirements.txt index 1d6ed5ca..e1ce9504 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,3 @@ pytest -pytest-mock \ No newline at end of file +pytest-mock +virtualenv<16 diff --git a/tests/test_reproducible_debian_packages.py b/tests/test_reproducible_debian_packages.py new file mode 100644 index 00000000..c0b68c3e --- /dev/null +++ b/tests/test_reproducible_debian_packages.py @@ -0,0 +1,48 @@ +import pytest +import subprocess +import os + + +PACKAGE_BUILD_TARGETS = { + "securedrop-client": "main", + "securedrop-log": "main", + "securedrop-proxy": "main", + "securedrop-export": "main", +} + +# These are the package names we want to test reproducibility for +PACKAGE_NAMES = PACKAGE_BUILD_TARGETS.keys() + + +def get_repo_root(): + cmd = "git rev-parse --show-toplevel".split() + top_level = subprocess.check_output(cmd).decode("utf-8").rstrip() + return top_level + +repo_root = get_repo_root() + + +@pytest.mark.parametrize("pkg_name", PACKAGE_NAMES) +def test_deb_builds_are_reproducible(pkg_name): + """ + Uses 'reprotest' to confirm that the Debian package build process + is deterministic, i.e. all .deb files are created with the same checksum + across multiple builds. + + We're not testing many variations, only exec_path, as a simple test + for deterministic builds with most aspects controlled. + """ + + cmd_env = os.environ.copy() + cmd_env["PKG_GITREF"] = os.environ.get("PKG_GITREF", PACKAGE_BUILD_TARGETS[pkg_name]) + cmd_env["TERM"] = "xterm-256color" + cmd = [ + "reprotest", + "-c", + f"make {pkg_name}", + "--variations", + "-all, -kernel, +exec_path", + ".", + f"build/debbuild/packaging/{pkg_name}*.deb", + ] + subprocess.check_call(cmd, env=cmd_env, cwd=repo_root) diff --git a/tests/test_reproducible_wheels.py b/tests/test_reproducible_wheels.py index 2ee3e77f..b21928a3 100644 --- a/tests/test_reproducible_wheels.py +++ b/tests/test_reproducible_wheels.py @@ -1,5 +1,6 @@ import pytest import subprocess +import os # These are the SDW repositories that we build wheels for. @@ -30,6 +31,8 @@ def test_wheel_builds_are_reproducible(repo_name): * kernel: x86_64 is the supported architecure, we don't ship others """ repo_url = f"https://github.com/freedomofpress/{repo_name}" + cmd_env = os.environ.copy() + cmd_env["TERM"] = "xterm-256color" cmd = [ "reprotest", "-c", @@ -40,4 +43,4 @@ def test_wheel_builds_are_reproducible(repo_name): "localwheels/*.whl", ] repo_root = get_repo_root() - subprocess.check_call(cmd, cwd=repo_root) + subprocess.check_call(cmd, env=cmd_env, cwd=repo_root) From da78f9b3e9598cd5bdec298f09f96474d99f6871 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Mon, 25 Jan 2021 16:01:32 -0800 Subject: [PATCH 5/5] Drops custom PyPI for securedrop-export Updated the build logic for securedrop-export to remove use of the https://pypi.securedrop.org URL. At present, the securedrop-export doesn't use any external Python dependencies, it's core-only. So the custom PyPI URL was never necessary, and technically we could excise it altogether right now, but for consistency's sake I'm updating it to match the other Python-based SDW packages. --- Makefile | 2 +- README.md | 4 ++-- securedrop-export/debian/rules | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index dfc0e202..5230947c 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ securedrop-workstation-viewer: ## Builds Debian metapackage for Disposable VM de .PHONY: securedrop-export securedrop-export: ## Builds Debian package for Qubes Workstation export scripts - PKG_NAME="securedrop-export" ./scripts/build-debianpackage + WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-export" ./scripts/build-debianpackage .PHONY: securedrop-log securedrop-log: ## Builds Debian package for Qubes Workstation securedrop-log scripts diff --git a/README.md b/README.md index 4995790a..c0726ae5 100644 --- a/README.md +++ b/README.md @@ -34,8 +34,8 @@ Maintainers of `securedrop-client` and `securedrop-proxy` must ensure that the requirements files which are used for build of these packages (`build-requirements.txt`) using `make requirements` are kept up to date in latest `main` of those repositories. -If new dependencies were added in the `requirements.txt` of that -repo that are not in the FPF PyPI mirror (https://pypi.securedrop.org/), then the maintainer needs +If new dependencies were added in the `build-requirements.txt` of that +repo that are not in the FPF PyPI mirror (`./localwheels/` in this repository), then the maintainer needs to do the following (we are taking `securedrop-client` project as example): ### 0. Create updated build-requirements.txt for the project diff --git a/securedrop-export/debian/rules b/securedrop-export/debian/rules index 9494ffd6..7df10ba9 100755 --- a/securedrop-export/debian/rules +++ b/securedrop-export/debian/rules @@ -7,8 +7,10 @@ override_dh_virtualenv: dh_virtualenv \ --python /usr/bin/python3 \ --setuptools \ - --index-url https://pypi.securedrop.org/simple \ --extra-pip-arg "--ignore-installed" \ + --extra-pip-arg "--no-index" \ + --extra-pip-arg "--find-links" \ + --extra-pip-arg "$(WHEELS_DIR)" \ --extra-pip-arg "--no-deps" \ --extra-pip-arg "--no-cache-dir" \ --extra-pip-arg "--no-use-pep517" \