Merge pull request #4865 from freedomofpress/pip-updates-sprint-38

Updates pip dependencies: Ansible, Pyyaml, Jinja2, Molecule, Werkzeug

.circleci/config.yml

@@ -74,7 +74,7 @@ jobs:
           name: Run all linters but shellcheck
           command: |
             fromtag=$(docker images |grep securedrop-test-xenial-py3 |head -n1 |awk '{print $2}')
-            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "sudo /opt/venvs/securedrop-app-code/bin/pip3 install -r requirements/python3/develop-requirements.txt && make -C .. ansible-config-lint app-lint docs-lint flake8 html-lint typelint yamllint"
+            DOCKER_BUILD_ARGUMENTS="--cache-from securedrop-test-xenial-py3:${fromtag:-latest}" securedrop/bin/dev-shell bash -c "/opt/venvs/securedrop-app-code/bin/pip3 install --require-hashes -r requirements/python3/develop-requirements.txt && make -C .. ansible-config-lint app-lint docs-lint flake8 html-lint typelint yamllint"
 
       - run:
           name: Run shellcheck

.github/PULL_REQUEST_TEMPLATE.md

@@ -41,3 +41,10 @@ Any special considerations for deployment? Consider both:
 ### If you made changes to documentation:
 
 - [ ] Doc linting (`make docs-lint`) passed locally
+
+### If you added or updated a code dependency:
+
+Choose one of the following:
+
+- [ ] I have performed a diff review and pasted the contents to [the packaging wiki](https://github.com/freedomofpress/securedrop-debian-packaging/wiki)
+- [ ] I would like someone else to do the diff review

Makefile

@@ -28,7 +28,7 @@ update-admin-pip-requirements:  ## Update admin requirements.
 .PHONY: update-python3-requirements
 update-python3-requirements:  ## Update Python 3 requirements with pip-compile.
 	@echo "███ Updating Python 3 requirements files..."
-	@$(DEVSHELL) pip-compile \
+	@$(DEVSHELL) pip-compile --generate-hashes \
 		--output-file requirements/python3/develop-requirements.txt \
 		../admin/requirements-ansible.in \
 		../admin/requirements.in \

admin/requirements.in

@@ -1,2 +1,2 @@
 prompt_toolkit
-pyyaml
+pyyaml>=5.1.2

admin/requirements.txt

@@ -4,8 +4,8 @@
 #
 #    pip-compile --generate-hashes --output-file=requirements.txt requirements-ansible.in requirements.in
 #
-ansible==2.6.14 \
-    --hash=sha256:412f130f4c5d1953ccd95f01b5a4675cbff4ba225762bafb74a2f3bb6c807827
+ansible==2.6.19 \
+    --hash=sha256:dbcfc9ddf620d05e1147b4c713738045a67c32be7260b11cbdbd84e92b77ca06
 asn1crypto==0.24.0 \
     --hash=sha256:2f1adbb7546ed199e3c90ef23ec95c5cf3585bac7d11fb7eb562a3fe89c64e87 \
     --hash=sha256:9d5c20441baf0cb60a4ac34cc447c6c189024b6b4c6cd7877034f4965c464e49 \
@@ -146,15 +146,20 @@ pynacl==1.2.1 \
     --hash=sha256:f5ce9e26d25eb0b2d96f3ef0ad70e1d3ae89b5d60255c462252a3e456a48c053 \
     --hash=sha256:fabf73d5d0286f9e078774f3435601d2735c94ce9e514ac4fb945701edead7e4 \
     # via paramiko
-pyyaml==3.12 \
-    --hash=sha256:16b20e970597e051997d90dc2cddc713a2876c47e3d92d59ee198700c5427736 \
-    --hash=sha256:3262c96a1ca437e7e4763e2843746588a965426550f3797a79fca9c6199c431f \
-    --hash=sha256:592766c6303207a20efc445587778322d7f73b161bd994f227adaa341ba212ab \
-    --hash=sha256:5ac82e411044fb129bae5cfbeb3ba626acb2af31a8d17d175004b70862a741a7 \
-    --hash=sha256:827dc04b8fa7d07c44de11fabbc888e627fa8293b695e0f99cb544fdfa1bf0d1 \
-    --hash=sha256:bc6bced57f826ca7cb5125a10b23fd0f2fff3b7c4701d64c439a300ce665fff8 \
-    --hash=sha256:c01b880ec30b5a6e6aa67b09a2fe3fb30473008c85cd6a67359a1b15ed6d83a4 \
-    --hash=sha256:e863072cdf4c72eebf179342c94e6989c67185842d9997960b3e69290b2fa269
+pyyaml==5.1.2 \
+    --hash=sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9 \
+    --hash=sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4 \
+    --hash=sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8 \
+    --hash=sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696 \
+    --hash=sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34 \
+    --hash=sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9 \
+    --hash=sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73 \
+    --hash=sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299 \
+    --hash=sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b \
+    --hash=sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae \
+    --hash=sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681 \
+    --hash=sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41 \
+    --hash=sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8
 six==1.11.0 \
     --hash=sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9 \
     --hash=sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb \

docs/development/contributor_guidelines.rst

@@ -58,7 +58,7 @@ root of the repository:
 
   .. code:: sh
 
-     pip install -r securedrop/requirements/develop-requirements.txt
+     pip install --require-hashes -r securedrop/requirements/python3/develop-requirements.txt
 
 Python
 ~~~~~~

docs/development/documentation_guidelines.rst

@@ -23,7 +23,7 @@ To get started editing the docs:
 
    .. code:: sh
 
-      pip install -r securedrop/requirements/develop-requirements.txt
+      pip install --require-hashes -r securedrop/requirements/python3/develop-requirements.txt
 
 #. Build the docs for viewing in your web browser:
 

docs/development/setup_development.rst

@@ -285,6 +285,6 @@ Ensure your virtualenv is activated and install the packages.
 
 .. code:: sh
 
-    pip install -r securedrop/requirements/python3/develop-requirements.txt
+    pip install --require-hashes -r securedrop/requirements/python3/develop-requirements.txt
 
 .. note:: You will need to run this everytime new packages are added.

docs/development/testing_configuration_tests.rst

@@ -14,7 +14,7 @@ Installation
 
 .. code:: sh
 
-    pip install -r securedrop/requirements/develop-requirements.txt
+    pip install --require-hashes -r securedrop/requirements/python3/develop-requirements.txt
 
 
 Running the Config Tests

install_files/ansible-base/callback_plugins/ansible_version_check.py

@@ -21,7 +21,7 @@ class CallbackModule(CallbackBase):
     def __init__(self):
         # Can't use `on_X` because this isn't forwards compatible
         # with Ansible 2.0+
-        required_version = '2.6.14'  # Keep synchronized with requirements files
+        required_version = '2.6.19'  # Keep synchronized with requirements files
         if not ansible.__version__.startswith(required_version):
             print_red_bold(
                 "SecureDrop restriction: only Ansible {version}.*"

molecule/testinfra/staging/app/test_tor_config.py

@@ -8,13 +8,13 @@
 @pytest.mark.parametrize('package', [
     'tor',
 ])
-def test_tor_packages(Package, package):
+def test_tor_packages(host, package):
     """
     Ensure Tor packages are installed. Does not include the Tor keyring
     package, since we want only the SecureDrop Release Signing Key
     to be used even for Tor packages.
     """
-    assert Package(package).is_installed
+    assert host.package(package).is_installed
 
 
 def test_tor_service_running(host):

molecule/testinfra/staging/common/test_grsecurity.py

@@ -52,7 +52,7 @@ def test_generic_kernels_absent(host, package):
     c = host.run("dpkg -l {}".format(package))
     assert c.rc == 1
     error_text = "dpkg-query: no packages found matching {}".format(package)
-    assert c.stderr == error_text
+    assert error_text in c.stderr.strip()
 
 
 def test_grsecurity_lock_file(host):
@@ -71,8 +71,8 @@ def test_grsecurity_kernel_is_running(host):
     Make sure the currently running kernel is specific grsec kernel.
     """
     c = host.run('uname -r')
-    assert c.stdout.endswith('-grsec')
-    assert c.stdout == '{}-grsec'.format(KERNEL_VERSION)
+    assert c.stdout.strip().endswith('-grsec')
+    assert c.stdout.strip() == '{}-grsec'.format(KERNEL_VERSION)
 
 
 @pytest.mark.parametrize('sysctl_opt', [
@@ -130,7 +130,7 @@ def test_grub_pc_marked_manual(host):
     """
     c = host.run('apt-mark showmanual grub-pc')
     assert c.rc == 0
-    assert c.stdout == "grub-pc"
+    assert c.stdout.strip() == "grub-pc"
 
 
 def test_apt_autoremove(host):

molecule/testinfra/staging/common/test_tor_mirror.py

@@ -28,7 +28,7 @@ def test_tor_keyring_absent(host):
     c = host.run("dpkg -l {}".format(package))
     assert c.rc == 1
     error_text = "dpkg-query: no packages found matching {}".format(package)
-    assert c.stderr.rstrip() == error_text
+    assert error_text in c.stderr.strip()
 
 
 @pytest.mark.parametrize('tor_key_info', [

molecule/testinfra/staging/common/test_user_config.py

@@ -65,7 +65,7 @@ def test_sudoers_tmux_env(host):
             (tmux attach || tmux_attach_via_proc || tmux new-session)
         fi"""
     )
-    assert host_file.content_string == expected_content
+    assert host_file.content_string.strip() == expected_content
 
 
 def test_tmux_installed(host):

securedrop/dockerfiles/xenial/python3/Dockerfile

@@ -57,7 +57,8 @@ RUN python3 -m venv /opt/venvs/securedrop-app-code && \
 
 RUN if test $USER_NAME != root ; then useradd --no-create-home --home-dir /tmp --uid $USER_ID $USER_NAME && echo "$USER_NAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers ; fi && \
     cp -r /root/.local /tmp/ && chmod +x /tmp/.local/tbb/tor-browser_en-US/Browser/firefox && chmod -R 777 /tmp/.local && \
-    chown -R $USER_NAME.$USER_NAME /tmp/.local/
+    chown -R $USER_NAME.$USER_NAME /tmp/.local/ && \
+    chown -R $USER_NAME.$USER_NAME /opt/venvs/securedrop-app-code/
 
 STOPSIGNAL SIGKILL
 

securedrop/requirements/python3/develop-requirements.in

@@ -7,7 +7,7 @@ docker-py
 dnspython
 flake8
 html-linter
-molecule>=2.20.1
+molecule>=2.22
 mypy
 # Needed for ansible network filter
 # http://docs.ansible.com/ansible/latest/playbooks_filters_ipaddr.html