Skip to content

Commit

Permalink
Merge pull request #666 from freedomofpress/reproducible-rpm
Browse files Browse the repository at this point in the history
Builds dom0 RPM reproducibly, removes docker requirement for building RPMs
  • Loading branch information
emkll authored Mar 29, 2021
2 parents d6ae7bf + 3fde237 commit e182525
Show file tree
Hide file tree
Showing 10 changed files with 132 additions and 140 deletions.
27 changes: 19 additions & 8 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,16 +15,27 @@ jobs:
sudo apt install file
make flake8
make black
buildrpm-f25:
machine: true
buildrpm:
docker:
- image: circleci/python:3.7-buster
steps:
- checkout
- run: make dom0-rpm-f25
buildrpm-f32:
machine: true
- run: sudo apt-get update && make install-deps
- run: make dom0-rpm
reprotest:
docker:
- image: debian:buster
steps:
- checkout
- run: make dom0-rpm-f32
- run: apt-get update && apt-get install -y sudo git make
- run: make install-deps
# Patch reprotest in-place to skip 'setarch' prefix, which fails under containers.
# See for more info in https://github.com/freedomofpress/securedrop-debian-packaging/pull/213/commits/18770bd44ec54da86f436515da4452c05111605e
# We should probably try to upstream this as an option.
- run:
command:
sudo sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py
- run: make reprotest-ci
launcher-tests-buster:
docker:
- image: circleci/python:3.7-buster
Expand All @@ -45,6 +56,6 @@ workflows:
securedrop_workstation_ci:
jobs:
- lint
- buildrpm-f25
- buildrpm-f32
- buildrpm
- reprotest
- launcher-tests-buster
4 changes: 0 additions & 4 deletions .dockerignore

This file was deleted.

5 changes: 3 additions & 2 deletions MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ include dom0/*.j2
include dom0/*.yml
include dom0/*.conf
include dom0/remove-tags
include dom0/sdw-admin
include dom0/securedrop-login
include dom0/securedrop-launcher.desktop
include dom0/securedrop-handle-upgrade
Expand All @@ -14,7 +13,6 @@ include config.json.example
include README.md
include LICENSE
include VERSION
include Makefile
include sd-proxy/*
include sd-whonix/*
include sd-app/*
Expand All @@ -26,3 +24,6 @@ include launcher/sdw_updater_gui/*.py
include launcher/sdw_notify/*.py
include launcher/sdw_util/*.py
include usb-autoattach/*
exclude scripts/build-dom0-rpm
exclude scripts/clone-to-dom0
exclude scripts/prep-dev
26 changes: 19 additions & 7 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,26 @@ dev staging: assert-dom0 ## Configures and builds a dev or staging environment
$(MAKE) prep-dev
sdw-admin --apply

.PHONY: dom0-rpm
dom0-rpm: ## Builds rpm package to be installed on dom0
$(MAKE) dom0-rpm-f25

dom0-rpm-f25: ## Builds rpm package to be installed on dom0
@./scripts/build-dom0-rpm f25

dom0-rpm-f32: ## Builds rpm package to be installed on dom0
@./scripts/build-dom0-rpm f32
@./scripts/build-dom0-rpm

.PHONY: reprotest
reprotest: ## Runs reprotest for RPMs with all variations
reprotest -c "make dom0-rpm" . "rpm-build/RPMS/noarch/*.rpm"

.PHONY: reprotest-ci
reprotest-ci: ## Runs reprotest for RPMs, with CI-compatible skips in variations
# Disable a few variations, to support CircleCI container environments.
# Requires a sed hack to reprotest, see .circle/config.yml
TERM=xterm-256color reprotest --variations "+all, +kernel, -domain_host, -fileordering" \
-c "make dom0-rpm" . "rpm-build/RPMS/noarch/*.rpm"

.PHONY: install-deps
install-deps: ## Installs apt package dependencies, for building RPMs
sudo apt-get install --no-install-recommends -y \
python3 python3-setuptools file python3-rpm \
rpm rpm-common diffoscope reprotest disorderfs faketime

clone: assert-dom0 ## Builds rpm && pulls the latest repo from work VM to dom0
@./scripts/clone-to-dom0
Expand Down
10 changes: 4 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ After that initial manual step, the code in your development VM may be copied in
[dom0]$ export SECUREDROP_DEV_VM=sd-dev # set to your dev VM
[dom0]$ export SECUREDROP_DEV_DIR=/home/user/projects/securedrop-workstation # set to your working directory
[dom0]$ cd ~/securedrop-workstation/
[dom0]$ make clone # build RPM package (requires Docker) and copy repo to dom0
[dom0]$ make clone # build RPM package and copy repo to dom0
```

If you plan to work on the [SecureDrop Client](https://github.com/freedomofpress/securedrop-client) code, also run this command in `dom0`:
Expand Down Expand Up @@ -315,7 +315,7 @@ make clone
make dev
```

The `make clone` command will build a new version of the RPM package that contains the provisioning logic in your development VM (e.g., `sd-dev`) and copy it to `dom0`. The RPM is built using a Docker container, so Docker must be installed in your development VM.
The `make clone` command will build a new version of the RPM package that contains the provisioning logic in your development VM (e.g., `sd-dev`) and copy it to `dom0`.

### Building the Templates

Expand All @@ -332,10 +332,8 @@ https://github.com/freedomofpress/securedrop-debian-packaging/
make dom0-rpm
```

This uses a base docker image as defined in https://github.com/freedomofpress/containers/.
If you need to bump versions of the rpmbuild tooling, make an update to that
repo's metadata, and increment the version as defined in the `Makefile`. See the
`RPM_BUILD_VER` variable.
The build assumes use of Debian Stable as the build environment. You can install
the necessary dependencies from system packages via the `make install-deps` target.

## Using the *SecureDrop Client*

Expand Down
52 changes: 39 additions & 13 deletions rpm-build/SPECS/securedrop-workstation-dom0-config.spec
Original file line number Diff line number Diff line change
@@ -1,16 +1,33 @@
Name: securedrop-workstation-dom0-config
Version: 0.5.3
%global srcname securedrop-workstation-dom0-config
%global version 0.5.3
%global __python3 /usr/bin/python3
%global python3_sitelib /usr/lib/%{_python_version}/site-packages
# For reproducible builds:
#
# * _buildhost is hostname, will vary based on environment
# * _source_date_epoch will be defined via env var
# * _custom_docdir is a workaround for %docs not supporting SOURCE_DATE_EPOCH
# * optflags is for multi-arch support: otherwise rpmbuild sets 'OPTFLAGS: -O2 -g -march=i386 -mtune=i686'
%global _buildhost %{srcname}
%global _source_date_epoch %{getenv:SOURCE_DATE_EPOCH}
%global _custom_docdir /usr/share/doc/%{srcname}
%global optflags -O2 -g

Name: %{srcname}
Version: %{version}
Release: 1%{?dist}
Summary: SecureDrop Workstation

Group: Library
License: GPLv3+
URL: https://github.com/freedomofpress/securedrop-workstation
Source0: securedrop-workstation-dom0-config-0.5.3.tar.gz
Source0: %{srcname}-%%{version}.tar.gz

BuildArch: noarch
BuildRequires: python3-setuptools
BuildRequires: python3-devel
# Disable declaration of build dependencies, because
# we build on Debian stable.
#BuildRequires: python3-setuptools
#BuildRequires: python3-devel

# This package installs all standard VMs in Qubes
Requires: qubes-mgmt-salt-dom0-virtual-machines
Expand All @@ -27,14 +44,15 @@ configuration over time.
# root policy.
%undefine py_auto_byte_compile

%prep
%setup -n securedrop-workstation-dom0-config-0.5.3
# Ensure that SOURCE_DATE_EPOCH is honored. Does not appear to affect
# use of %doc macro, so we use _custom_docdir instead.
%define use_source_date_epoch_as_buildtime 1

%build
%{__python3} setup.py build
%prep
%setup -n %{srcname}-%{version}

%install
%{__python3} setup.py install --no-compile --skip-build --root %{buildroot}
%{__python3} setup.py install --install-lib %{python3_sitelib} --no-compile --root %{buildroot}
install -m 755 -d %{buildroot}/opt/securedrop/launcher
install -m 755 -d %{buildroot}/opt/securedrop/launcher/sdw_updater_gui
install -m 755 -d %{buildroot}/opt/securedrop/launcher/sdw_notify
Expand All @@ -54,6 +72,9 @@ install -m 755 -d %{buildroot}/srv/salt/launcher/sdw_updater_gui
install -m 755 -d %{buildroot}/srv/salt/launcher/sdw_notify
install -m 755 -d %{buildroot}/srv/salt/launcher/sdw_util
install -m 755 -d %{buildroot}/%{_bindir}
install -m 755 -d %{buildroot}/usr/share/%{srcname}
# Create doc dir manually, because %doc macro doesn't honor SOURCE_DATE_EPOCH.
install -m 755 -d %{buildroot}/%{_custom_docdir}
install -m 644 dom0/*.sls %{buildroot}/srv/salt/
install -m 644 dom0/*.top %{buildroot}/srv/salt/
install -m 644 dom0/*.j2 %{buildroot}/srv/salt/
Expand All @@ -73,7 +94,6 @@ install -m 644 sd-workstation/* %{buildroot}/srv/salt/sd/sd-workstation/
install -m 644 sys-firewall/* %{buildroot}/srv/salt/sd/sys-firewall/
install -m 644 usb-autoattach/99-sd-devices.rules %{buildroot}/srv/salt/sd/usb-autoattach/
install -m 755 usb-autoattach/sd-attach-export-device %{buildroot}/srv/salt/sd/usb-autoattach/
install -m 644 Makefile %{buildroot}/usr/share/%{name}/Makefile
install -m 755 scripts/* %{buildroot}/usr/share/%{name}/scripts/
# For the updater scripts, we want to provision them via rpm *and* also salt, since there's a salt step that will provision this
install -m 644 launcher/*.py %{buildroot}/opt/securedrop/launcher/
Expand All @@ -84,13 +104,19 @@ install -m 644 launcher/sdw_notify/*.py %{buildroot}/opt/securedrop/launcher/sdw
install -m 644 launcher/sdw_notify/*.py %{buildroot}/srv/salt/launcher/sdw_notify/
install -m 644 launcher/sdw_util/*.py %{buildroot}/opt/securedrop/launcher/sdw_util/
install -m 644 launcher/sdw_util/*.py %{buildroot}/srv/salt/launcher/sdw_util/
install -m 644 config.json.example %{buildroot}/usr/share/%{srcname}/
install -m 644 README.md LICENSE %{buildroot}/%{_custom_docdir}/
find %{buildroot} -type d -iname '*.egg-info' -print0 | xargs -0 -r rm -rf
find %{buildroot} -exec touch -m -d @%{_source_date_epoch} {} +

%files
%doc README.md LICENSE
%attr(755, root, root) /opt/securedrop/launcher/sdw-launcher.py
%attr(755, root, root) /opt/securedrop/launcher/sdw-notify.py
%attr(755, root, root) %{_bindir}/sdw-admin
%{python3_sitelib}/securedrop_workstation_dom0_config*
%{_datadir}/%{name}
%{_custom_docdir}/LICENSE
%{_custom_docdir}/README.md
/usr/share/%{srcname}/config.json.example
/opt/securedrop/launcher/**/*.py
/srv/salt/sd*
/srv/salt/dom0-xfce-desktop-file.j2
Expand Down
103 changes: 42 additions & 61 deletions scripts/build-dom0-rpm
Original file line number Diff line number Diff line change
@@ -1,67 +1,48 @@
#!/bin/bash
#
#
# Builds RPM for dom0 bootstrap logic
# $1 - Digest Hash of base docker container to use

set -u
# Builds RPMs for installation in dom0. RPMs are fully reproducible.
# Targets F25 & F32 for Qubes 4.0 and 4.1 support.
set -e
set -u
set -o pipefail

export RPM_DIST=${1-"f25"}

export F25_BASE_CONTAINER_HASH="322cb01bbca26972c98051bacd3ab8555cec059496d64d35ee78b15de9ea0d06"
export F32_BASE_CONTAINER_HASH="d6a6d60fda1b22b6d5fe3c3b2abe2554b60432b7b215adc11a2b5fae16f50188"
export F25_PKGR_VER="0.6.0.1-1.fc25"
export F32_PKGR_VER="0.6.0.4-1.fc32"
export LOCAL_IMG="fpf.local/rpmbuilder-${RPM_DIST}:latest"
export ROOT_DIR="$(git rev-parse --show-toplevel)"
export USER_RPMDIR="/home/user/rpmbuild"

# Set distribution-specific variables for generic Dockerfile
if [[ ${RPM_DIST} == "f32" ]] ; then
echo "Building for Fedora 32"
export CONTAINER_HASH="${F32_BASE_CONTAINER_HASH}"
export PKG_VER="${F32_PKGR_VER}"
else
echo "Building for Fedora 25"
export CONTAINER_HASH="${F25_BASE_CONTAINER_HASH}"
export PKG_VER="${F25_PKGR_VER}"

# Check for dependencies
if ! hash rpmbuild 2> /dev/null ; then
echo "ERROR: missing rpmbuild, run: make install-deps"
exit 1
fi

function build_local_base {
docker build --build-arg=CONTAINER_HASH="${CONTAINER_HASH}" \
--build-arg=FEDORA_PKGR_VER="${PKG_VER}" \
--build-arg=USERID="$(id -u)" \
-t "${LOCAL_IMG}" \
-f "scripts/rpmbuilder.Dockerfile" scripts/ 2>&1
}

function docker_cmd_wrapper() {
docker run -t --rm \
--network=none \
-v "${ROOT_DIR}:/sd" \
-v "${ROOT_DIR}/rpm-build:${USER_RPMDIR}" \
"${LOCAL_IMG}" \
$@
}



build_local_base

docker_cmd_wrapper /usr/bin/python3 setup.py sdist

# Remove any cached tarballs. We must do this because the container image config
# needlessly marks the rpmbuild dir as a volume. If we don't remove tarballs
# before building, the subsequent cp command will fail.
docker_cmd_wrapper find "${USER_RPMDIR}" -type f -iname '*.tar.gz' -delete

# The tarball will exist in the /sd workdir, copy it to the RPM build dir.
docker_cmd_wrapper find /sd -type f -iname '*.tar.gz' -exec cp -u -t "${USER_RPMDIR}/SOURCES/" {} +

docker_cmd_wrapper rpmbuild -ba "${USER_RPMDIR}/SPECS/securedrop-workstation-dom0-config.spec"

local_rpms="$(find rpm-build/ -type f -iname '*.rpm')"

printf "\nRPM packages can be found at:\n\n%s\n" "$local_rpms"
# Prepare tarball, rpmbuild will use it
mkdir -p dist/
git clean -fdX rpm-build/ dist/
/usr/bin/python3 setup.py sdist

# Use the epoch time of the highest semver tag available.
# SOURCE_DATE_EPOCH="$(git tag | sort -V | tail -n 1 | xargs git log -1 --format=%at)"
# Use the epoch time of the most recent commit. If works in dev,
# as well as building from signed tags.
SOURCE_DATE_EPOCH="$(git log -1 --format=%at HEAD)"
export SOURCE_DATE_EPOCH

# Place tarball where rpmbuild will find it
cp dist/*.tar.gz rpm-build/SOURCES/

# Build for Qubes 4.0.x and 4.1.x, for which dom0 is based on
# F25 and F32, respectively.
for i in 25 32; do
# dom0 defaults to python3.5 in F25
python_version="python3.5"
if [[ $i = 32 ]]; then
python_version="python3.8"
fi
dist=".fc${i}"
rpmbuild \
--quiet \
--define "_topdir $PWD/rpm-build" \
--define "dist $dist" \
--define "_python_version $python_version" \
-bb --clean "rpm-build/SPECS/securedrop-workstation-dom0-config.spec"
done

printf '\nBuild complete! RPMs and their checksums are:\n\n'
find rpm-build/ -type f -iname '*.rpm' -print0 | sort -zV | xargs -0 sha256sum
5 changes: 4 additions & 1 deletion scripts/prep-dev
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@ set -o pipefail
dom0_dev_dir="$HOME/securedrop-workstation"

function find_latest_rpm() {
find "${dom0_dev_dir}/rpm-build/RPMS/" -type f -iname '*.rpm' | xargs -d '\n' ls -t | head -n 1
# Look up which version of dom0 we're using.
# Qubes 4.0 is fc25, Qubes 4.1 will be fc32.
fedora_version="$(rpm --eval '%{fedora}')"
find "${dom0_dev_dir}/rpm-build/RPMS/" -type f -iname "*fc${fedora_version}.noarch.rpm" -print0 | xargs -0 ls -t | head -n 1
}

latest_rpm="$(find_latest_rpm)"
Expand Down
35 changes: 0 additions & 35 deletions scripts/rpmbuilder.Dockerfile

This file was deleted.

Loading

0 comments on commit e182525

Please sign in to comment.