Merge pull request #5484 from freedomofpress/5480-old-dhve-builtin-venv

Revert to old dh-virtualenv, use built-in venv module

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/main.yml

@@ -29,13 +29,13 @@
 - name: Copy requirements.txt to prep directory
   command: cp "{{ securedrop_pip_requirements }}" "{{ securedrop_app_code_prep_dir }}/requirements.txt"
 
-- include: sass.yml
-
-- name: Install SecureDrop Python requirements in container
+- name: Control the version of setuptools used in the default construction of virtual environments
   shell: |
-    pip3 install --no-deps --no-binary :all: --require-hashes -r {{ securedrop_app_code_prep_dir }}/requirements.txt
-  tags:
-    - pip
+    pip3 download --no-deps --require-hashes -r "{{ securedrop_app_code_prep_dir }}/requirements.txt" --dest /tmp/securedrop-app-code-requirements-download
+    rm -f /usr/share/python-wheels/setuptools-*.whl
+    mv /tmp/securedrop-app-code-requirements-download/setuptools-*.whl /usr/share/python-wheels/
+
+- include: sass.yml
 
 - include: translations.yml
 
@@ -117,9 +117,6 @@
   args:
     chdir: "{{ securedrop_app_code_deb_dir }}"
   environment:
-    DH_PIP_EXTRA_ARGS: "--verbose --ignore-installed --no-deps --no-binary=:all: --no-cache-dir"
-    DH_UPGRADE_SETUPTOOLS: "46.0.0"
-    DH_VIRTUALENV_ARGUMENTS: "{{  '--python=/usr/bin/python3.8 --setuptools  46.0.0' if securedrop_build_focal_support else '--python=/usr/bin/python3.5 --setuptools' }}"
     DH_VIRTUALENV_INSTALL_ROOT: "/opt/venvs"
 
 - name: Find newly built Debian package

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/translations.yml

@@ -1,10 +1,18 @@
 ---
 
+- name: Install SecureDrop Python requirements in virtualenv for translation work
+  shell: |
+    python3 -m venv /tmp/securedrop-app-code-i18n-ve
+    /tmp/securedrop-app-code-i18n-ve/bin/pip3 install --no-deps --no-binary :all: --require-hashes -r {{ securedrop_app_code_prep_dir }}/requirements.txt
+  tags:
+    - pip
+
 - name: Compile PO to MO.
   shell: >-
     cp config.py.example config.py ;
     trap 'rm config.py' EXIT ;
-    python3 ./i18n_tool.py --verbose translate-messages --compile
+    . /tmp/securedrop-app-code-i18n-ve/bin/activate ;
+    /tmp/securedrop-app-code-i18n-ve/bin/python3 ./i18n_tool.py --verbose translate-messages --compile
   args:
     chdir: "{{ securedrop_code_filtered }}"
   environment:

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/templates/control.j2

@@ -11,7 +11,7 @@ Architecture: amd64
 Conflicts: libapache2-mod-wsgi,supervisor
 Replaces: libapache2-mod-wsgi,supervisor
 {% if securedrop_build_focal_support %}
-Depends: ${dist:Depends}, ${misc:Depends}, ${python3:Depends}, apache2, apparmor-utils, coreutils, gnupg2, haveged, libapache2-mod-xsendfile, libpython3.8, paxctld, python3, redis-server, securedrop-config, securedrop-keyring, sqlite3
+Depends: ${dist:Depends}, ${misc:Depends}, ${python3:Depends}, apache2, apparmor-utils, coreutils, gnupg2, haveged, libapache2-mod-xsendfile, libpython3.8, paxctld, python3, python3-distutils, redis-server, securedrop-config, securedrop-keyring, sqlite3
 {% else %}
 Depends: ${dist:Depends}, ${misc:Depends}, ${python3:Depends}, apache2, apparmor-utils, coreutils, gnupg2, haveged, libapache2-mod-xsendfile, libpython3.5, paxctld, python3 (>= 3.5), python3 (<< 3.6), redis-server, securedrop-config, securedrop-keyring, sqlite3
 {% endif %}

install_files/securedrop-app-code/debian/rules

@@ -5,7 +5,7 @@ DEB_DH_INSTALL_ARGS=-X .git
 SECUREDROP_BUILD_PLATFORM=$(shell lsb_release -sc)
 
 %:
-	dh $@ --buildsystem=dh_virtualenv --with systemd
+	dh $@ --with python-virtualenv --with systemd
 
 override_dh_gencontrol:
 	dh_gencontrol -- $(SUBSTVARS)
@@ -24,6 +24,15 @@ override_dh_strip_nondeterminism:
 	find -type f -name RECORD -exec sed -i -e '/.*\.pyc.*/d' {} +
 	dh_strip_nondeterminism $@
 
+override_dh_virtualenv:
+	dh_virtualenv \
+		--python=/usr/bin/python3 \
+		--builtin-venv \
+		--extra-pip-arg "--verbose" \
+		--extra-pip-arg "--no-deps" \
+		--extra-pip-arg "--no-binary=:all:" \
+		--extra-pip-arg "--no-cache-dir"
+
 #
 # Have to override the automatic service handling since we have more
 # than one.

install_files/securedrop-ossec-agent/DEBIAN/control.j2

@@ -7,7 +7,7 @@ Package: securedrop-ossec-agent
 Version: 3.6.0+1.6.0~rc1
 Architecture: amd64
 {% if securedrop_build_focal_support %}
-Depends: libevent-2.1.7,libpcre2-8-0,ossec-agent,securedrop-keyring,securedrop-config
+Depends: libevent-2.1-7,libpcre2-8-0,ossec-agent,securedrop-keyring,securedrop-config
 {% else %}
 Depends: libevent-1.4-2,libpcre2-8-0,ossec-agent,securedrop-keyring,securedrop-config
 {% endif %}

install_files/securedrop-ossec-server/DEBIAN/control.j2

@@ -7,7 +7,7 @@ Package: securedrop-ossec-server
 Version: 3.6.0+1.6.0~rc1
 Architecture: amd64
 {% if securedrop_build_focal_support %}
-Depends: libevent-2.1.7,libpcre2-8-0,ossec-server,securedrop-keyring,securedrop-config
+Depends: libevent-2.1-7,libpcre2-8-0,ossec-server,securedrop-keyring,securedrop-config
 {% else %}
 Depends: libevent-1.4-2,libpcre2-8-0,ossec-server,securedrop-keyring,securedrop-config
 {% endif %}

molecule/builder-focal/Dockerfile

@@ -27,6 +27,7 @@ RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \
         python3-all \
         python3-pip \
         python3-setuptools \
+        python3-venv \
         rsync \
         ruby \
         sqlite \

molecule/builder-focal/image_hash

@@ -1,2 +1,2 @@
-# sha256 digest quay.io/freedomofpress/sd-docker-builder-focal:2020_08_13
-47f7e1af8cc54bf9cb0afdef599b76d063c614943e54443ea660d6b53615814c
+# sha256 digest quay.io/freedomofpress/sd-docker-builder-focal:2020_09_11
+10d4b7f28a3de6f2bd9e6a55dd5fdb448fb90442d15a013f73942b4d2e5ce259

molecule/builder-xenial/Dockerfile

@@ -7,14 +7,13 @@ LABEL image_name="xenial-sd-builder-app"
 
 RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \
         apache2-dev \
-        apt-transport-https \
         aptitude \
         coreutils \
-        curl \
         debhelper \
         devscripts \
         dh-python \
         dh-systemd \
+        dh-virtualenv \
         gdb \
         git \
         gnupg2 \
@@ -28,20 +27,14 @@ RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \
         python3-all \
         python3-pip \
         python3-setuptools \
+        python3-venv \
         rsync \
         ruby \
         sqlite \
         sudo \
         tzdata \
         unzip \
-        virtualenv
-
-#install dh-virtualenv from debian unstable
-RUN curl https://ftp-master.debian.org/keys/archive-key-10.asc -o /tmp/debian-stable-archive-pubkey.asc && \
-        apt-key add /tmp/debian-stable-archive-pubkey.asc
-RUN echo "deb https://deb.debian.org/debian unstable main contrib" > /etc/apt/sources.list.d/debian-unstable.list
-COPY aptpreferences.conf /etc/apt/preferences.d/debian-unstable
-RUN apt-get update && apt-get install -y -t unstable dh-virtualenv
-RUN apt-get clean && rm -rf /var/lib/apt/lists/*
+        && apt-get clean \
+        && rm -rf /var/lib/apt/lists/*
 
 RUN paxctl -cm /usr/bin/python3.5 && mkdir -p /tmp/build

molecule/builder-xenial/image_hash

@@ -1,2 +1,2 @@
-# sha256 digest quay.io/freedomofpress/sd-docker-builder-xenial:2020_09_02
-d62c99bb967e6dd1b276e642a29000656489313c2da2997f6bfbf466509f9b74
+# sha256 digest quay.io/freedomofpress/sd-docker-builder-xenial:2020_09_11
+b88a9e0ba180b55d0f22e84a81d3efc5da2263596a22ca5da043f84a2d741d88

molecule/builder-xenial/tests/test_build_dependencies.py

@@ -4,7 +4,7 @@
 
 SECUREDROP_TARGET_PLATFORM = os.environ.get("SECUREDROP_TARGET_PLATFORM")
 SECUREDROP_PYTHON_VERSION = os.environ.get("SECUREDROP_PYTHON_VERSION", "3.5")
-SECUREDROP_DH_VIRTUALENV_VERSION = os.environ.get("SECUREDROP_DH_VIRTUALENV_VERSION", "1.2.1")
+
 testinfra_hosts = [
         "docker://{}-sd-app".format(SECUREDROP_TARGET_PLATFORM)
 ]
@@ -19,15 +19,6 @@ def test_sass_gem_installed(host):
     assert c.rc == 0
 
 
-def test_pip_dependencies_installed(host):
-    """
-    Ensure the development pip dependencies are installed
-    """
-    c = host.run("pip3 list installed")
-    assert "Flask-Babel" in c.stdout
-    assert c.rc == 0
-
-
 @pytest.mark.xfail(reason="This check conflicts with the concept of pegging"
                           "dependencies")
 def test_build_all_packages_updated(host):
@@ -55,12 +46,9 @@ def test_python_version(host):
 
 def test_dh_virtualenv(host):
     """
-    The version of dh-virtualenv in Xenial repos isn't new enough to work
-    with setuptools >= 50, so we pull it in from Debian. Confirm the expected
-    recent version of dh-virtualenv is found. Since we're tracking Debian unstable
-    for this dependency, this check will fail if unstable surprises us with a new
-    version.
+    Confirm the expected version of dh-virtualenv is found.
     """
+    expected_version = "0.11" if host.system_info.codename == "xenial" else "1.2.1"
+    version_string = "dh_virtualenv {}".format(expected_version)
     c = host.run("dh_virtualenv --version")
-    version_string = "dh_virtualenv {}".format(SECUREDROP_DH_VIRTUALENV_VERSION)
     assert c.stdout.startswith(version_string)

molecule/builder-xenial/tests/test_securedrop_deb_package.py

@@ -187,11 +187,10 @@ def test_deb_package_contains_mo_file(host, deb):
     Ensures the `securedrop-app-code` package has at least one
     compiled mo file.
     """
-    deb_package = host.file(deb.format(
-        securedrop_test_vars.securedrop_version))
-    c = host.run("dpkg-deb --contents {}".format(deb_package.path))
+    deb_package = host.file(deb.format(securedrop_test_vars.securedrop_version))
     # Only relevant for the securedrop-app-code package:
     if "securedrop-app-code" in deb_package.path:
+        c = host.run("dpkg-deb --contents {}".format(deb_package.path))
         assert re.search(r"^.*messages\.mo$", c.stdout, re.M)
 
 

molecule/testinfra/staging/app/test_appenv.py

@@ -4,13 +4,11 @@
 testinfra_hosts = ["app-staging"]
 sdvars = pytest.securedrop_test_vars
 
-sdbin = "/opt/venvs/securedrop-app-code/bin"
-
 
 @pytest.mark.parametrize('exp_pip_pkg', sdvars.pip_deps)
 def test_app_pip_deps(host, exp_pip_pkg):
     """ Ensure pip dependencies are installed """
-    pip = host.pip_package.get_packages(pip_path=os.path.join(sdbin, "pip"))
+    pip = host.pip_package.get_packages(pip_path=os.path.join(sdvars.securedrop_venv_bin, "pip"))
     assert pip[exp_pip_pkg['name']]['version'] == exp_pip_pkg['version']
 
 
@@ -48,6 +46,18 @@ def test_app_code_pkg(host):
     assert host.package("securedrop-app-code").is_installed
 
 
+def test_app_code_venv(host):
+    """
+    Ensure the securedrop-app-code virtualenv is correct.
+    """
+    cmd = """test -z $VIRTUAL_ENV && . {}/bin/activate && test "$VIRTUAL_ENV" = "{}" """.format(
+        sdvars.securedrop_venv, sdvars.securedrop_venv
+    )
+
+    result = host.run(cmd)
+    assert result.rc == 0
+
+
 def test_supervisor_not_installed(host):
     """ ensure supervisor package is not installed """
     assert host.package("supervisor").is_installed is False

molecule/testinfra/staging/vars/app-staging.yml

@@ -12,8 +12,8 @@ wanted_apache_headers:
   - 'Header unset Etag'
 
 securedrop_venv: /opt/venvs/securedrop-app-code
-securedrop_venv_bin: "{{ securedrop_venv }}/bin"
-securedrop_venv_site_packages: "{{ securedrop_venv }}/lib/python3.5/site-packages"
+securedrop_venv_bin: /opt/venvs/securedrop-app-code/bin
+securedrop_venv_site_packages: /opt/venvs/securedrop-app-code/lib/python3.5/site-packages
 securedrop_code: /var/www/securedrop
 securedrop_data: /var/lib/securedrop
 securedrop_user: www-data

molecule/testinfra/staging/vars/staging.yml

@@ -12,8 +12,8 @@ wanted_apache_headers:
   - 'Header unset Etag'
 
 securedrop_venv: /opt/venvs/securedrop-app-code
-securedrop_venv_bin: "/opt/venvs/securedrop-app-code/bin"
-securedrop_venv_site_packages: "/opt/venvs/securedrop-app-code/lib/python3.5/site-packages"
+securedrop_venv_bin: /opt/venvs/securedrop-app-code/bin
+securedrop_venv_site_packages: /opt/venvs/securedrop-app-code/lib/python3.5/site-packages
 securedrop_code: /var/www/securedrop
 securedrop_data: /var/lib/securedrop
 securedrop_user: www-data