Merge pull request #3833 from freedomofpress/3206-xenial-staging-targ…

…et-clean-install

Achieves clean install against Xenial

docs/development/xenial_support.rst

@@ -38,10 +38,6 @@ Known bugs with Xenial support
 Below is a high-level overview of known problems to be addressed
 for delivering Xenial compatibility.
 
-Ansible provisioning
-    Changes required to support the "xenial" apt sources throughout
-    config logic, particularly in templates.
-
 Packaging
     Dependencies for the ``securedrop-app-code`` deb package have changed;
     ``apache2`` should be explicitly required; ``apache2-mpm-worker``
@@ -61,9 +57,9 @@ PAM logic
     resolves. More research required.
 
 Config tests
-    The testinfra config test suite expects Trusty throughout. We'll need
-    to update that logic to refer to LSB codename instead, and assert
-    that the target platform is one of either Trusty or Xenial.
+    The testinfra config test suite runs slightly different checks for
+    Trusty and Xenial where appropriate. Care should be taken to preserve
+    functionality of the config tests against both distros.
 
 More detailed research notes on evaluating Xenial support can be found
 in the following GitHub issues:

install_files/ansible-base/roles/app-test/files/xvfb-systemd

@@ -0,0 +1,9 @@
+[Unit]
+Description=X Virtual Frame Buffer Service
+After=network.target
+
+[Service]
+ExecStart=/usr/bin/Xvfb :1 -screen 0 1024x768x24 -ac +extension GLX +render -noreset
+
+[Install]
+WantedBy=multi-user.target

install_files/ansible-base/roles/app-test/tasks/dev_setup_xvfb_for_functional_tests.yml

@@ -13,24 +13,48 @@
   tags:
     - apt
 
-- name: Copy xvfb init script.
+- name: Copy xvfb init script (Trusty)
   copy:
     src: xvfb
     dest: /etc/init.d/xvfb
     owner: root
     mode: "0700"
+  # Trusty uses upstart/sysv-style init scripts
+  when: ansible_distribution_release == "trusty"
   tags:
     - xvfb
     - permissions
 
-- name: Update rc.d to run xvfb at boot.
+- name: Update rc.d to run xvfb at boot (Trusty)
   command: update-rc.d xvfb defaults
   register: xvfb_setup
   changed_when: "'System start/stop links for /etc/init.d/xvfb already exist' not in xvfb_setup.stdout"
+  when: ansible_distribution_release == "trusty"
   notify: start xvfb
   tags:
     - xvfb
 
+- name: Copy xvfb init script (Xenial)
+  copy:
+    src: xvfb-systemd
+    dest: /etc/systemd/system/xvfb.service
+    owner: root
+    mode: "0644"
+  # Xenial uses systemd, so upstart/sysv init scripts won't work
+  when: ansible_distribution_release == "xenial"
+  notify: start xvfb
+  tags:
+    - xvfb
+    - permissions
+
+- name: Update xvfb service to run at boot (Xenial)
+  service:
+    name: xvfb
+    enabled: yes
+  when: ansible_distribution_release == "xenial"
+  tags:
+    - xvfb
+
 - name: Set DISPLAY environment variable for xvfb.
   copy:
     src: xvfb_display.sh

install_files/ansible-base/roles/app-test/tasks/modern_gettext.yml

@@ -8,6 +8,7 @@
     repo: deb http://archive.ubuntu.com/ubuntu/ xenial main
     state: present
     update_cache: yes
+  when: ansible_distribution_release != "xenial"
   tags:
     - apt
 
@@ -23,5 +24,6 @@
     repo: deb http://archive.ubuntu.com/ubuntu/ xenial main
     state: absent
     update_cache: yes
+  when: ansible_distribution_release != "xenial"
   tags:
     - apt

install_files/ansible-base/roles/app/defaults/main.yml

@@ -45,11 +45,9 @@ securedrop_app_https_ssl_ciphers:
   - ECDHE-ECDSA-AES128-SHA256
   - ECDHE-RSA-AES128-SHA256
 
-# Apt packages for configuring Apache service.
-apache_packages:
-  - apache2-mpm-worker
-  - libapache2-mod-wsgi
-  - libapache2-mod-xsendfile
+# Distro-specific vars to be included dynamically for
+# Trusty and Xenial support.
+apache_packages: []
 
 apache_templates:
   - ports.conf

install_files/ansible-base/roles/app/tasks/install_and_harden_apache.yml

@@ -1,4 +1,7 @@
 ---
+- name: Import apache packages vars for distro.
+  include_vars: "{{ ansible_distribution+'_'+ansible_distribution_release }}.yml"
+
 - name: Install apache packages.
   apt:
     pkg: "{{ item }}"
@@ -57,6 +60,7 @@
   apache2_module:
     state: absent
     name: "{{ item }}"
+    force: yes
   with_items: "{{ apache_disabled_modules }}"
   notify:
     - restart apache2

install_files/ansible-base/roles/app/vars/Ubuntu_trusty.yml

@@ -0,0 +1,5 @@
+---
+apache_packages:
+  - apache2-mpm-worker
+  - libapache2-mod-wsgi
+  - libapache2-mod-xsendfile

install_files/ansible-base/roles/app/vars/Ubuntu_xenial.yml

@@ -0,0 +1,5 @@
+---
+apache_packages:
+  - apache2
+  - libapache2-mod-wsgi
+  - libapache2-mod-xsendfile

molecule/testinfra/staging/app-code/test_xvfb.py

@@ -23,15 +23,20 @@ def test_firefox_is_installed(Package, Command):
     assert c.stdout.rstrip() == "Mozilla Firefox 46.0.1"
 
 
-def test_xvfb_service_config(File, Sudo):
+def test_xvfb_service_config_trusty(host):
     """
     Ensure xvfb service configuration file is present.
     Using Sudo context manager because the expected mode is 700.
     Not sure it's really necessary to have this script by 700; 755
     sounds sufficient.
     """
-    with Sudo():
-        f = File("/etc/init.d/xvfb")
+    # We're checking the upstart/sysv-style init script, which is only
+    # relevant for Trusty.
+    if host.system_info.codename == "xenial":
+        return True
+
+    with host.sudo():
+        f = host.file("/etc/init.d/xvfb")
     assert f.is_file
     assert oct(f.mode) == "0700"
     assert f.user == "root"
@@ -67,14 +72,14 @@ def test_xvfb_service_config(File, Sudo):
 
 exit 0
 """.lstrip().rstrip()  # noqa
-    with Sudo():
+    with host.sudo():
         assert f.contains('^XVFB=/usr/bin/Xvfb$')
         assert f.contains('^XVFBARGS=":1 -screen 0 1024x768x24 '
                           '-ac +extension GLX +render -noreset"$')
         assert f.content.rstrip() == xvfb_init_content
 
 
-def test_xvfb_service_enabled(Command, Sudo):
+def test_xvfb_service_enabled_trusty(host):
     """
     Ensure xvfb is configured to start on boot via update-rc.d.
     The `-n` option to update-rc.d is dry-run.
@@ -83,8 +88,13 @@ def test_xvfb_service_enabled(Command, Sudo):
     Not sure it's really necessary to have this script by 700; 755
     sounds sufficient.
     """
-    with Sudo():
-        c = Command('update-rc.d -n xvfb defaults')
+    # We're checking the upstart/sysv-style init script, which is only
+    # relevant for Trusty.
+    if host.system_info.codename == "xenial":
+        return True
+
+    with host.sudo():
+        c = host.command('update-rc.d -n xvfb defaults')
     assert c.rc == 0
     wanted_text = 'System start/stop links for /etc/init.d/xvfb already exist.'
     assert wanted_text in c.stdout
@@ -103,22 +113,27 @@ def test_xvfb_display_config(File):
     assert f.contains("export DISPLAY=:1\n")
 
 
-def test_xvfb_service_running(Process, Sudo):
+def test_xvfb_service_running_trusty(host):
     """
     Ensure that xvfb service is running.
 
     We can't use the Service module because it expects a "status"
     subcommand for the init script, and our custom version doesn't have
     one. So let's make sure the process is running.
     """
+    # We're checking the upstart/sysv-style init script, which is only
+    # relevant for Trusty.
+    if host.system_info.codename == "xenial":
+        return True
+
     # Sudo isn't necessary to read out of /proc on development, but is
     # required when running under Grsecurity, which app-staging does.
     # So let's escalate privileges to ensure we can determine service state.
-    with Sudo():
-        p = Process.get(user="root", comm="Xvfb")
+    with host.sudo():
+        p = host.process.get(user="root", comm="Xvfb")
         wanted_args = str('/usr/bin/Xvfb :1 -screen 0 1024x768x24 '
                           '-ac +extension GLX +render -noreset')
         assert p.args == wanted_args
         # We only expect a single process, no children.
-        workers = Process.filter(ppid=p.pid)
+        workers = host.process.filter(ppid=p.pid)
         assert len(workers) == 0

molecule/testinfra/staging/app/apache/test_apache_system_config.py

@@ -6,7 +6,6 @@
 
 
 @pytest.mark.parametrize("package", [
-    "apache2-mpm-worker",
     "libapache2-mod-wsgi",
     "libapache2-mod-xsendfile",
 ])
@@ -17,6 +16,17 @@ def test_apache_apt_packages(Package, package):
     assert Package(package).is_installed
 
 
+def test_apache_apt_packages_trusty(Package, SystemInfo):
+    """
+    Ensure required Apache packages are installed. Only checks Trusty-specific
+    packages; other tests handle more general apt dependencies for Apache.
+    """
+    # Skip if testing against Xenial
+    if SystemInfo.release != "trusty":
+        return True
+    assert Package("apache2-mpm-worker").is_installed
+
+
 def test_apache_security_config_deprecated(File):
     """
     Ensure that /etc/apache2/security is absent, since it was setting

molecule/testinfra/staging/app/iptables-app-staging.j2

@@ -24,7 +24,7 @@
 -A OUTPUT -o lo -p tcp -m tcp --sport 8080 -m owner --uid-owner {{ securedrop_user_id }} -m state --state RELATED,ESTABLISHED -m comment --comment "Restrict the apache user outbound connections" -j ACCEPT
 -A OUTPUT -s 127.0.0.1/32 -d 127.0.0.1/32 -o lo -p tcp -m owner --uid-owner {{ securedrop_user_id }} -m state --state NEW,RELATED,ESTABLISHED -m comment --comment "for redis worker all application user local loopback user" -j ACCEPT
 -A OUTPUT -m owner --uid-owner {{ securedrop_user_id }} -m comment --comment "Drop all other traffic by the securedrop user" -j LOGNDROP
--A OUTPUT -m owner --gid-owner 108 -m comment --comment "Drop all other outbound traffic for ssh user" -j LOGNDROP
+-A OUTPUT -m owner --gid-owner {{ ssh_group_gid }} -m comment --comment "Drop all other outbound traffic for ssh user" -j LOGNDROP
 -A OUTPUT -d {{ dns_server }}/32 -p tcp -m tcp --dport 53 -m owner --uid-owner 0 -m state --state NEW,RELATED,ESTABLISHED -m comment --comment "tcp/udp dns" -j ACCEPT
 -A OUTPUT -d {{ dns_server }}/32 -p udp -m udp --dport 53 -m owner --uid-owner 0 -m state --state NEW,RELATED,ESTABLISHED -m comment --comment "tcp/udp dns" -j ACCEPT
 -A OUTPUT -p udp -m udp --sport 123 --dport 123 -m owner --uid-owner 0 -m state --state NEW,RELATED,ESTABLISHED -m comment --comment ntp -j ACCEPT

molecule/testinfra/staging/app/test_apparmor.py

@@ -109,12 +109,24 @@ def test_apparmor_total_profiles(Command, Sudo):
     with Sudo():
         total_expected = str((len(sdvars.apparmor_enforce)
                              + len(sdvars.apparmor_complain)))
-        assert Command.check_output("aa-status --profiled") == total_expected
+        # Trusty has ~10, Xenial about ~20 profiles, so let's expect
+        # *at least* the sum.
+        assert Command.check_output("aa-status --profiled") >= total_expected
 
 
-def test_aastatus_unconfined(Command, Sudo):
+def test_aastatus_unconfined(host):
     """ Ensure that there are no processes that are unconfined but have
         a profile """
-    unconfined_chk = "0 processes are unconfined but have a profile defined"
-    with Sudo():
-        assert unconfined_chk in Command("aa-status").stdout
+
+    # Trusty should show 0 unconfined processes. In Xenial, haveged
+    # is unconfined by default, due to introduction of a new profile.
+    # We should consider enforcing that, too.
+    expected_unconfined = 0
+    if host.system_info.codename == "xenial":
+        expected_unconfined = 1
+
+    unconfined_chk = str("{} processes are unconfined but have"
+                         " a profile defined".format(expected_unconfined))
+    with host.sudo():
+        aa_status_output = host.check_output("aa-status")
+        assert unconfined_chk in aa_status_output

molecule/testinfra/staging/app/test_tor_config.py

@@ -16,31 +16,54 @@ def test_tor_packages(Package, package):
     assert Package(package).is_installed
 
 
-def test_tor_service_running(Command, File, Sudo):
+def test_tor_service_running_trusty(host):
     """
     Ensure tor is running and enabled. Tor is required for SSH access,
-    so it must be enabled to start on boot.
+    so it must be enabled to start on boot. Checks upstart/sysv-style
+    services, used by Trusty.
     """
     # TestInfra tries determine the service manager intelligently, and
     # inappropriately assumes Upstart on Trusty, due to presence of the
     # `initctl` command. The tor service is handled via a SysV-style init
     # script, so let's just shell out and verify the running and enabled
     # states explicitly.
-    with Sudo():
-        assert Command.check_output("service tor status") == \
+    if host.system_info.codename == "xenial":
+        return True
+
+    with host.sudo():
+        assert host.check_output("service tor status") == \
                " * tor is running"
-        tor_enabled = Command.check_output("find /etc/rc?.d -name S??tor")
+        tor_enabled = host.check_output("find /etc/rc?.d -name S??tor")
 
     assert tor_enabled != ""
 
     tor_targets = tor_enabled.split("\n")
     assert len(tor_targets) == 4
     for target in tor_targets:
-        t = File(target)
+        t = host.file(target)
         assert t.is_symlink
         assert t.linked_to == "/etc/init.d/tor"
 
 
+def test_tor_service_running_xenial(host):
+    """
+    Ensure tor is running and enabled. Tor is required for SSH access,
+    so it must be enabled to start on boot. Checks systemd-style services,
+    used by Xenial.
+    """
+    # TestInfra tries determine the service manager intelligently, and
+    # inappropriately assumes Upstart on Trusty, due to presence of the
+    # `initctl` command. The tor service is handled via a SysV-style init
+    # script, so let's just shell out and verify the running and enabled
+    # states explicitly.
+    if host.system_info.codename == "trusty":
+        return True
+
+    s = host.service("tor")
+    assert s.is_running
+    assert s.is_enabled
+
+
 @pytest.mark.parametrize('torrc_option', [
     'SocksPort 0',
     'SafeLogging 1',

molecule/testinfra/staging/common/test_platform.py

@@ -1,3 +1,9 @@
+# We expect Ubuntu, either Trusty or Xenial, the two LTSes
+# currently targeted for support.
+SUPPORTED_CODENAMES = ('trusty', 'xenial')
+SUPPORTED_RELEASES = ('14.04', '16.04')
+
+
 def test_ansible_version(host):
     """
     Check that a supported version of Ansible is being used.
@@ -11,14 +17,14 @@ def test_ansible_version(host):
     assert c.startswith("ansible 2.")
 
 
-def test_platform(SystemInfo):
+def test_platform(host):
     """
     SecureDrop requires Ubuntu Trusty 14.04 LTS. The shelf life
     of that release means we'll need to migrate to Xenial LTS
     at some point; until then, require hosts to be running
     Ubuntu.
     """
-    assert SystemInfo.type == "linux"
-    assert SystemInfo.distribution == "ubuntu"
-    assert SystemInfo.codename == "trusty"
-    assert SystemInfo.release == "14.04"
+    assert host.system_info.type == "linux"
+    assert host.system_info.distribution == "ubuntu"
+    assert host.system_info.codename in SUPPORTED_CODENAMES
+    assert host.system_info.release in SUPPORTED_RELEASES

molecule/testinfra/staging/common/test_system_hardening.py

@@ -74,7 +74,9 @@ def test_swap_disabled(Command):
         assert not re.search("^/", c, re.M)
         # Expect that ONLY the headers will be present in the output.
         rgx = re.compile("Filename\s*Type\s*Size\s*Used\s*Priority")
-        assert re.search(rgx, c)
+        # On Xenial, swapon 2.27.1 shows blank output, with no headers, so
+        # check for empty output as confirmation of no swap.
+        assert any((re.search(rgx, c), c == ""))
 
 
 def test_twofactor_disabled_on_tty(host):