diff --git a/.circleci/config.yml b/.circleci/config.yml index 128ed7b05a..e1bb87109b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,7 +5,7 @@ jobs: docker: - image: gcr.io/cloud-builders/docker steps: - - run: apt-get install -y make + - run: apt-get install -y make python - checkout - run: name: Rebase on-top of github target diff --git a/install_files/ansible-base/roles/app-test/defaults/main.yml b/install_files/ansible-base/roles/app-test/defaults/main.yml index a33dbcec1b..87908fa4fc 100644 --- a/install_files/ansible-base/roles/app-test/defaults/main.yml +++ b/install_files/ansible-base/roles/app-test/defaults/main.yml @@ -8,24 +8,3 @@ securedrop_app_test_apparmor_complain: False # Source Interface config to enable logging in the staging environment. apache_user: www-data -# Additional apt packages required for running the application tests. -test_apt_dependencies: - - xvfb - -# Additional Python dependencies required for running the application tests. -test_pip_requirements: "{{ securedrop_code }}/requirements/test-requirements.txt" - -# Specify TBB version to download and install -tbb_release: 7.5.6 -tbb_locale: en-US -tbb_arch: 64 -tbb_directory: "/home/{{ ansible_user|default(ansible_ssh_user|default(lookup('env', 'USER'))) }}/.local/tbb" -tbb_signing_key: EF6E286DDA85EA2A4BA7DE684E2C6E8793298290 -keyserver: "hkp://ipv4.pool.sks-keyservers.net" -tbb_apt_dependencies: - - libasound2 - - libdbus-glib-1-2 - - libfontconfig1 - - libgtk2.0-0 - - libxrender1 - diff --git a/install_files/ansible-base/roles/app-test/tasks/dev_setup_xvfb_for_functional_tests.yml b/install_files/ansible-base/roles/app-test/tasks/dev_setup_xvfb_for_functional_tests.yml deleted file mode 100644 index 3ec81b71cd..0000000000 --- a/install_files/ansible-base/roles/app-test/tasks/dev_setup_xvfb_for_functional_tests.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -- name: Install pip dependencies for running the unit and functional tests. - pip: - requirements: "{{ test_pip_requirements }}" - tags: - - pip - -- name: Install testing package dependencies. - apt: - name: "{{ item }}" - state: present - with_items: "{{ test_apt_dependencies }}" - tags: - - apt - -- name: Copy xvfb init script. - copy: - src: xvfb - dest: /etc/init.d/xvfb - owner: root - mode: "0700" - tags: - - xvfb - - permissions - -- name: Update rc.d to run xvfb at boot. - command: update-rc.d xvfb defaults - register: xvfb_setup - changed_when: "'System start/stop links for /etc/init.d/xvfb already exist' not in xvfb_setup.stdout" - notify: start xvfb - tags: - - xvfb - -- name: Set DISPLAY environment variable for xvfb. - copy: - src: xvfb_display.sh - dest: /etc/profile.d/xvfb_display.sh - owner: root - mode: "0444" - tags: - - xvfb - - environment - - permissions diff --git a/install_files/ansible-base/roles/app-test/tasks/disable_mprotect_ff_plugin_container.yml b/install_files/ansible-base/roles/app-test/tasks/disable_mprotect_ff_plugin_container.yml deleted file mode 100644 index 4c7ae1c0cf..0000000000 --- a/install_files/ansible-base/roles/app-test/tasks/disable_mprotect_ff_plugin_container.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -- name: disable mprotect on firefox and the firefox plugin container - command: paxctl -cm {{ tbb_dir }}/tor-browser_{{ tbb_locale }}/Browser/{{ item }} - # Read-only task, so don't report changed - changed_when: false - # The first time this task is run, the grub binaries won't have - # their headers converted yet, which will cause paxctl to exit 1. - # We'll catch that error and respond accordingly in the next task. - failed_when: false - register: paxctl_firefox_header_check - with_items: - - firefox - - plugin-container - when: ansible_kernel.endswith('-grsec') - -# Flags as snagged from app-staging: '-----m-x-e--' -- name: Adjust paxctl headers on firefox binaries. - command: paxctl -zcm {{ item.item }} - with_items: "{{ paxctl_firefox_header_check.results }}" - when: - # Chained conditional; only inspect command results if running under grsecurity. - - ansible_kernel.endswith('-grsec') - - "item.stdout != '- PaX flags: -----m-x---- [{{ item.item }}]' or item.rc != 0" diff --git a/install_files/ansible-base/roles/app-test/tasks/install_tbb.yml b/install_files/ansible-base/roles/app-test/tasks/install_tbb.yml deleted file mode 100644 index 0ddf7a2003..0000000000 --- a/install_files/ansible-base/roles/app-test/tasks/install_tbb.yml +++ /dev/null @@ -1,74 +0,0 @@ ---- -- name: Install TBB apt dependencies. - sudo: true - apt: - name: "{{ item }}" - state: latest - update_cache: yes - cache_valid_time: 3600 - with_items: "{{ tbb_apt_dependencies }}" - -- name: Create tbb directory. - sudo: true - file: - path: "{{ tbb_directory }}" - state: directory - owner: "{{ securedrop_user }}" - group: "{{ securedrop_user }}" - mode: "0770" - -- name: Download TBB {{ tbb_release }} tarball and signature. - get_url: - url: "{{ item }}" - dest: "{{ tbb_directory }}/{{ item|basename }}" - with_items: - - https://dist.torproject.org/torbrowser/{{ tbb_release }}/tor-browser-linux{{ tbb_arch }}-{{ tbb_release }}_{{ tbb_locale }}.tar.xz - - https://dist.torproject.org/torbrowser/{{ tbb_release }}/tor-browser-linux{{ tbb_arch }}-{{ tbb_release }}_{{ tbb_locale }}.tar.xz.asc - -- name: Install gnupg-curl - sudo: true - apt: - name: gnupg-curl - state: present - -- name: Download TBB {{ tbb_release }} signing key. - sudo: true - command: gpg --keyserver {{ keyserver }} --recv-key {{ tbb_signing_key }} - register: gpg_import_tor_browser_devs_key_result - changed_when: "'imported: 1' in gpg_import_tor_browser_devs_key_result.stderr" - -- name: Verify TBB {{ tbb_release }} signature. - command: > - gpg --verify - tor-browser-linux{{ tbb_arch }}-{{ tbb_release }}_{{ tbb_locale }}.tar.xz.asc - tor-browser-linux{{ tbb_arch }}-{{ tbb_release }}_{{ tbb_locale }}.tar.xz - register: gpg_verify_result - changed_when: false - args: - chdir: "{{ tbb_directory }}" - -- name: Extract TBB {{ fpsd_crawler_tbb_release }} archive. - unarchive: - copy: no - src: "{{ tbb_directory }}/tor-browser-linux{{ tbb_arch }}-{{ tbb_release }}_{{ tbb_locale }}.tar.xz" - dest: "{{ tbb_directory }}" - owner: "{{ securedrop_user }}" - group: "{{ securedrop_user }}" - mode: "0770" - -- name: Download geckodriver for compatibility with Tor - get_url: - dest: "/opt/geckodriver-v0.17.0-linux64.tar.gz" - url: https://github.com/mozilla/geckodriver/releases/download/v0.17.0/geckodriver-v0.17.0-linux64.tar.gz - sha256sum: 3154274c050d724eb2f4e8986a58ed37c0138b48304692bf7eeed827a5e82319 - tags: - - apt - -- name: extract geckodriver - unarchive: - src: /opt/geckodriver-v0.17.0-linux64.tar.gz - dest: /bin/ - -- name: install geckodriver - shell: | - chmod +x /bin/geckodriver diff --git a/install_files/ansible-base/roles/app-test/tasks/main.yml b/install_files/ansible-base/roles/app-test/tasks/main.yml index 684f0fb164..3182eb668a 100644 --- a/install_files/ansible-base/roles/app-test/tasks/main.yml +++ b/install_files/ansible-base/roles/app-test/tasks/main.yml @@ -9,12 +9,5 @@ tags: - aa-complain -- include: dev_setup_xvfb_for_functional_tests.yml - -- include: setup_firefox_for_selenium.yml - -- include: disable_mprotect_ff_plugin_container.yml - - include: modern_gettext.yml -- include: install_tbb.yml diff --git a/install_files/ansible-base/roles/app-test/tasks/setup_firefox_for_selenium.yml b/install_files/ansible-base/roles/app-test/tasks/setup_firefox_for_selenium.yml deleted file mode 100644 index 4459093189..0000000000 --- a/install_files/ansible-base/roles/app-test/tasks/setup_firefox_for_selenium.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -- name: Check whether Firefox is installed. - command: firefox --version - changed_when: false - # Command will fail if Firefox is absent; don't halt. - failed_when: false - register: firefox_installed_result - -- name: Uninstall dist-provided version of Firefox. - apt: - name: firefox - state: absent - purge: yes - when: - - not firefox_installed_result|failed - - firefox_installed_result.stdout|default('') != "Mozilla Firefox 46.0.1" - -# Selenium 3 makes breaking changes with the 2.X API, and requires the -# installation of the Mozilla geckodriver. Since the Aaron Swartz Day Hackathon -# is approaching, which will involve many new external contributors, we've -# decided to play it as safe as possible by downgrading Firefox to the latest -# version (46.0.1) that is compatible with the last 2.X series Selenium release -# (2.53.6). After the Hackathon, we'll resolve the geckodriver business and -# remove the following three tasks (as well as add firefox back to the -# `test_apt_dependencies` list). -- name: Download Firefox 46.0.1 for compatibility with Selenium 2.53.6. - get_url: - # Since the whole tasklisk is run as root, the ansible_env.HOME fact is - # /root. Since this command doesn't need to be run as root and is part of a - # crutch anyway, I've just hardcoded /tmp - dest: "/opt/firefox_46.0.1+build1-0ubuntu0.14.04.3_amd64.deb" - url: https://launchpad.net/~ubuntu-mozilla-security/+archive/ubuntu/ppa/+build/9727836/+files/firefox_46.0.1+build1-0ubuntu0.14.04.3_amd64.deb - sha256sum: 88d25053306d33658580973b063cd459a56e3596a3a298c1fb8ab1d52171d860 - tags: - - apt - -- name: Install dependencies for Firefox 46.0.1. - apt: - name: "{{ item }}" - with_items: - - libasound2 - - libcairo-gobject2 - - libgtk-3-0 - - libstartup-notification0 - tags: - - apt - -- name: Install Firefox 46.0.1 for compatibility with Selenium 2.53.6. - apt: - deb: "/opt/firefox_46.0.1+build1-0ubuntu0.14.04.3_amd64.deb" - tags: - - apt - -- name: Set apt hold on Firefox version (via apt). - command: apt-mark hold firefox - register: apt_hold_firefox_result - # apt-mark will return output to report changed status; subsequent runs - # will report "firefox was already set on hold." - changed_when: "'firefox set on hold' in apt_hold_firefox_result.stdout" - -- name: Set apt hold on Firefox version (via aptitude). - command: aptitude hold firefox - # `aptitude hold ` doesn't report meaningful changed status, - # so mark the task as not changed. - changed_when: false diff --git a/securedrop/tests/functional/functional_test.py b/securedrop/tests/functional/functional_test.py index c9d9a54d31..ba131011ee 100644 --- a/securedrop/tests/functional/functional_test.py +++ b/securedrop/tests/functional/functional_test.py @@ -11,7 +11,6 @@ import time import json import traceback -import subprocess import shutil import requests @@ -202,7 +201,6 @@ def init_gpg(self): gpg.import_keys(open(keyfile).read()) return gpg - def setup(self, session_expiration=30): self.localtesting = False @@ -268,7 +266,8 @@ def setup(self, session_expiration=30): # This user is required for our tests cases to login self.admin_user = { "name": "journalist", - "password": "correct horse battery staple profanity oil chewy", + "password": ("correct horse battery staple" + " profanity oil chewy"), "secret": "JHCOGO7VCER3EJ4L"} self.admin_user['totp'] = pyotp.TOTP(self.admin_user['secret']) self.sleep_time = 2 @@ -319,7 +318,7 @@ def start_journalist_server(app): self.driver = self._create_webdriver() else: # We will use a normal firefox esr for the pages-layout tests - self.driver = self._create_webdriver2(self.new_profile) # pylint: disable=no-member # noqa + self.driver = self._create_webdriver2(self.new_profile) # noqa # pylint: disable=no-member self._javascript_toggle() # Polls the DOM to wait for elements. To read more about why diff --git a/securedrop/tests/functional/journalist_navigation_steps.py b/securedrop/tests/functional/journalist_navigation_steps.py index 667f41dd8d..86df59d056 100644 --- a/securedrop/tests/functional/journalist_navigation_steps.py +++ b/securedrop/tests/functional/journalist_navigation_steps.py @@ -1,5 +1,4 @@ import pytest -import json import re import tempfile import gzip @@ -8,7 +7,7 @@ import random import requests -from os.path import abspath, realpath, dirname, join +from os.path import dirname from selenium.common.exceptions import NoSuchElementException from selenium.webdriver.common.keys import Keys @@ -55,9 +54,9 @@ def return_downloaded_content(self, url, cookies): proxies = None if ".onion" in url: proxies = { - 'http': 'socks5h://127.0.0.1:9150', - 'https': 'socks5h://127.0.0.1:9150' - } + 'http': 'socks5h://127.0.0.1:9150', + 'https': 'socks5h://127.0.0.1:9150' + } r = requests.get(url, cookies=cookies, proxies=proxies, stream=True) if r.status_code != 200: raise Exception("Failed to download the data.") @@ -641,7 +640,7 @@ def _journalist_downloads_message(self): # Downloading files with Selenium is tricky because it cannot automate # the browser's file download dialog. We can directly request the file - # using requests, but we need to pass the cookies for the logged in user + # using requests, but we need to pass the cookies for logged in user # for Flask to allow this. def cookie_string_from_selenium_cookies(cookies): result = {} diff --git a/testinfra/app-code/test_xvfb.py b/testinfra/app-code/test_xvfb.py deleted file mode 100644 index 4b416e391a..0000000000 --- a/testinfra/app-code/test_xvfb.py +++ /dev/null @@ -1,121 +0,0 @@ -def test_xvfb_is_installed(Package): - """ - Ensure apt requirements for Xvfb are present. - """ - assert Package("xvfb").is_installed - - -def test_firefox_is_installed(Package, Command): - """ - The app test suite requires a very specific version of Firefox, for - compatibility with Selenium. Make sure to check the explicit - version of Firefox, not just that any version of Firefox is installed. - """ - p = Package("firefox") - assert p.is_installed - - c = Command("firefox --version") - # Reminder: the rstrip is only necessary for local-context actions, - # but it's a fine practice in all contexts. - assert c.stdout.rstrip() == "Mozilla Firefox 46.0.1" - - -def test_xvfb_service_config(File, Sudo): - """ - Ensure xvfb service configuration file is present. - Using Sudo context manager because the expected mode is 700. - Not sure it's really necessary to have this script by 700; 755 - sounds sufficient. - """ - with Sudo(): - f = File("/etc/init.d/xvfb") - assert f.is_file - assert oct(f.mode) == "0700" - assert f.user == "root" - assert f.group == "root" - # Let's hardcode the entire init script and check for exact match. - # The pytest output will display a diff if anything is missing. - xvfb_init_content = """ -# This is the /etc/init.d/xvfb script. We use it to launch xvfb at boot in the -# development environment so we can easily run the functional tests. - -XVFB=/usr/bin/Xvfb -XVFBARGS=":1 -screen 0 1024x768x24 -ac +extension GLX +render -noreset" -PIDFILE=/var/run/xvfb.pid -case "$1" in - start) - echo -n "Starting virtual X frame buffer: Xvfb" - start-stop-daemon --start --quiet --pidfile $PIDFILE --make-pidfile --background --exec $XVFB -- $XVFBARGS - echo "." - ;; - stop) - echo -n "Stopping virtual X frame buffer: Xvfb" - start-stop-daemon --stop --quiet --pidfile $PIDFILE - echo "." - ;; - restart) - $0 stop - $0 start - ;; - *) - echo "Usage: /etc/init.d/xvfb {start|stop|restart}" - exit 1 -esac - -exit 0 -""".lstrip().rstrip() # noqa - with Sudo(): - assert f.contains('^XVFB=/usr/bin/Xvfb$') - assert f.contains('^XVFBARGS=":1 -screen 0 1024x768x24 ' - '-ac +extension GLX +render -noreset"$') - assert f.content.rstrip() == xvfb_init_content - - -def test_xvfb_service_enabled(Command, Sudo): - """ - Ensure xvfb is configured to start on boot via update-rc.d. - The `-n` option to update-rc.d is dry-run. - - Using Sudo context manager because the service file is mode 700. - Not sure it's really necessary to have this script by 700; 755 - sounds sufficient. - """ - with Sudo(): - c = Command('update-rc.d -n xvfb defaults') - assert c.rc == 0 - wanted_text = 'System start/stop links for /etc/init.d/xvfb already exist.' - assert wanted_text in c.stdout - - -def test_xvfb_display_config(File): - """ - Ensure DISPLAY environment variable is set on boot, for running - headless tests via Xvfb. - """ - f = File('/etc/profile.d/xvfb_display.sh') - assert f.is_file - assert oct(f.mode) == "0444" - assert f.user == "root" - assert f.group == "root" - assert f.contains("export DISPLAY=:1\n") - - -def test_xvfb_service_running(Process, Sudo): - """ - Ensure that xvfb service is running. - - We can't use the Service module because it expects a "status" - subcommand for the init script, and our custom version doesn't have - one. So let's make sure the process is running. - """ - # Sudo isn't necessary to read out of /proc on development, but is - # required when running under Grsecurity, which app-staging does. - # So let's escalate privileges to ensure we can determine service state. - with Sudo(): - p = Process.get(user="root", comm="Xvfb") - wanted_args = str('/usr/bin/Xvfb :1 -screen 0 1024x768x24 ' - '-ac +extension GLX +render -noreset') - assert p.args == wanted_args - # We only expect a single process, no children. - workers = Process.filter(ppid=p.pid) - assert len(workers) == 0