Automatically builds deb packages in staging environment

.travis.yml

@@ -23,7 +23,7 @@ install:
 - pip uninstall pytest -y
 - pip install -r testinfra/requirements.txt
 script:
-- echo localhost > inventory
+- printf "[development]\nlocalhost\n[travis]\n[development:children]\ntravis" > inventory
 - ansible-playbook -i inventory -vv --syntax-check install_files/ansible-base/securedrop-travis.yml
 - ansible-playbook -i inventory -vv --connection=local --sudo --skip-tags=non-development install_files/ansible-base/securedrop-travis.yml
   # For some reason, redis-server does not start automatically when installed

Vagrantfile

@@ -23,7 +23,7 @@ Vagrant.configure("2") do |config|
       ansible.playbook = "install_files/ansible-base/securedrop-development.yml"
       ansible.verbose = 'v'
       ansible.groups = {
-        'development' => %(development),
+        'development' => %w(development),
         'securedrop_application_server' => %w(development),
         'securedrop:children' => %w(development),
       }
@@ -83,10 +83,10 @@ Vagrant.configure("2") do |config|
       ansible.limit = 'all'
       ansible.raw_arguments = Shellwords.shellsplit(ENV['ANSIBLE_ARGS']) if ENV['ANSIBLE_ARGS']
       ansible.groups = {
-        'securedrop_application_server' => %(app-staging),
-        'securedrop_monitor_server' => %(mon-staging),
+        'securedrop_application_server' => %w(app-staging),
+        'securedrop_monitor_server' => %w(mon-staging),
         'staging:children' => %w(securedrop_application_server securedrop_monitor_server),
-        'securedrop:children' => %w(securedrop_application_server securedrop_monitor_server),
+        'securedrop:children' => %w(staging),
       }
     end
   end
@@ -138,8 +138,8 @@ Vagrant.configure("2") do |config|
       ansible.verbose = 'v'
       ansible.raw_arguments = Shellwords.shellsplit(ENV['ANSIBLE_ARGS']) if ENV['ANSIBLE_ARGS']
       ansible.groups = {
-        'development' => %(build),
-        'securedrop_application_server' => %(build),
+        'development' => %w(build),
+        'securedrop_application_server' => %w(build),
         'securedrop:children' => %w(development),
       }
     end

docs/development/virtual_environments.rst

@@ -128,6 +128,19 @@ to fill out your local copy of
    ./manage.py add-admin
    pytest -v tests/
 
+To rebuild the local packages for the app code: ::
+
+   ANSIBLE_ARGS="--tags rebuild" vagrant provision /staging/
+
+The Debian packages will be rebuilt from the current state of your
+local git repository and then installed on the staging servers.
+
+.. tip::
+    You will also need to build the OSSEC packages in a separate repo.
+    Clone the ossec repo from https://github.com/freedomofpress/ossec and run
+    ``vagrant up``, then copy the deb packages into the ``build/``
+    directory in the securedrop repo.
+
 Prod
 ----
 

install_files/ansible-base/build-deb-pkgs.yml

@@ -1,9 +1,43 @@
 ---
-- hosts: build
+# Ensure that the build machine is present. If it hasn't been created,
+# then Vagrant will not add it to the generated inventory file. Since
+# the build machine is required for staging runs, error out with a helpful
+# message, pointing the developer to the docs.
+- name: Ensure Build VM is available.
+  hosts: localhost
+  connection: local
+  gather_facts: no
+  sudo: no
+  tasks:
+    - name: Check for "build" machine in inventory.
+      fail:
+        msg: >
+          The "build" VM is not reachable, but is required for building the
+          app code Debian packages on the staging machines. Run `vagrant up build`,
+          then `vagrant provision /staging/`. Thereafter you can run
+          `ANSIBLE_ARGS="--tags rebuild" vagrant provision /staging/` to rebuild
+          and reinstall the application code.
+      when: "'build' not in groups.all"
+  tags: rebuild
+
+# All plays in this playbook target the build host, but most be invoked separately,
+# since we're using wrapper roles for the build. If you add multiple child roles with
+# the same parent in the same play, Ansible conflates the dependencies, and only the
+# last wrapper role will run. This is a known bug and will be fixed in v2.0.
+- name: Build SecureDrop application Debian package from local repository.
+  hosts: build
+  vars_files:
+    # The order of these files files is important, since each successive
+    # include overrides vars in the previous files. Ordinarily we'd
+    # use group memberships to manage the variable resolution order, but
+    # since we're using Vagrant with Ansible, group management becomes tricky.
+    - group_vars/securedrop.yml
+    - group_vars/securedrop_application_server.yml
+    - group_vars/development.yml
   roles:
     - { role: build-securedrop-app-code-deb-pkg, tags: [ "app-deb" ] }
     - { role: build-generic-pkg, tags: ["securedrop-ossec-server"], package_name: "securedrop-ossec-server" }
     - { role: build-generic-pkg, tags: ["securedrop-ossec-agent"], package_name: "securedrop-ossec-agent" }
     - { role: build-generic-pkg, tags: ["securedrop-keyring"], package_name: "securedrop-keyring" }
-
   sudo: yes
+  tags: rebuild

install_files/ansible-base/group_vars/development.yml

@@ -37,3 +37,8 @@ smtp_relay_fingerprint: "6D:87:EE:CB:D0:37:2F:88:B8:29:06:FB:35:F4:65:00:7F:FD:8
 sasl_username: "test"
 sasl_domain: "ossec.test"
 sasl_password: "password123"
+
+# Don't install app-code package from the FPF apt repo, since we want to run
+# the application code directly out of the local repo.
+securedrop_app_install_from_repo: False
+securedrop_app_configure_apache: False

install_files/ansible-base/group_vars/securedrop_application_server.yml

@@ -40,9 +40,10 @@ securedrop_ossec_agent_deb: "{{ securedrop_repo }}/securedrop-ossec-agent-2.8.2-
 
 ### Used by the install_local_deb_pkgs role ###
 local_deb_packages:
+  - "securedrop-ossec-agent-2.8.2+{{ securedrop_app_code_version }}-amd64.deb"
   - "{{ securedrop_app_code_deb }}.deb"
   - ossec-agent-2.8.2-amd64.deb
-  - "securedrop-ossec-agent-2.8.2+{{ securedrop_app_code_version }}-amd64.deb"
+  - securedrop-keyring-0.1.0+{{ securedrop_app_code_version }}-amd64.deb
 
 ### Used by the app role ###
 

install_files/ansible-base/group_vars/securedrop_monitor_server.yml

@@ -20,8 +20,9 @@ securedrop_ossec_server_deb: "{{ securedrop_repo }}/securedrop-ossec-server-2.8.
 
 ### Used by the install_local_deb_pkgs role ###
 local_deb_packages:
-  - ossec-server-2.8.2-amd64.deb
   - "securedrop-ossec-server-2.8.2+{{ securedrop_app_code_version }}-amd64.deb"
+  - ossec-server-2.8.2-amd64.deb
+  - securedrop-keyring-0.1.0+{{ securedrop_app_code_version }}-amd64.deb
 
 ### Used by the mon role ###
 

install_files/ansible-base/group_vars/staging.yml

@@ -64,3 +64,11 @@ allow_direct_access: true
 # To use uncomment the following line and enter the correct decrypted zip
 # filename between the quotes.
 #backup_zip: ""
+
+install_local_packages: true
+
+# Don't install app-code package from the FPF apt repo, since we want to run
+# the application code directly out of the local repo. Do, however, make sure
+# the Apache service is configured correctly.
+securedrop_app_install_from_repo: False
+securedrop_app_configure_apache: True

install_files/ansible-base/roles/app-test/tasks/dev_setup_xvfb_for_functional_tests.yml

@@ -27,7 +27,7 @@
     # Since the whole tasklisk is run as root, the ansible_env.HOME fact is
     # /root. Since this command doesn't need to be run as root and is part of a
     # crutch anyway, I've just hardcoded /home/vagrant.
-    dest: /home/vagrant/
+    dest: /home/vagrant/firefox_46.0.1+build1-0ubuntu0.14.04.3_amd64.deb
     url: https://launchpad.net/~ubuntu-mozilla-security/+archive/ubuntu/ppa/+build/9727836/+files/firefox_46.0.1+build1-0ubuntu0.14.04.3_amd64.deb
     sha256sum: 88d25053306d33658580973b063cd459a56e3596a3a298c1fb8ab1d52171d860
   tags:
@@ -50,6 +50,19 @@
   tags:
     - apt
 
+- name: Set apt hold on Firefox version (via apt).
+  command: apt-mark hold firefox
+  register: apt_hold_firefox_result
+  # apt-mark will return output to report changed status; subsequent runs
+  # will report "firefox was already set on hold."
+  changed_when: "'firefox set on hold' in apt_hold_firefox_result.stdout"
+
+- name: Set apt hold on Firefox version (via aptitude).
+  command: aptitude hold firefox
+  # `aptitude hold <package>` doesn't report meaningful changed status,
+  # so mark the task as not changed.
+  changed_when: false
+
 - name: Copy xvfb init script.
   copy:
     src: xvfb

install_files/ansible-base/roles/app/defaults/main.yml

@@ -0,0 +1,9 @@
+---
+# By default the role should install the SD app-code package from the repo,
+# but staging hosts will override this setting to prefer locally-built
+# deb packages. Development will also skip entirely.
+securedrop_app_install_from_repo: True
+
+# Whether to configure the Apache service for the application. Required
+# in prod and staging, but not relevant for development.
+securedrop_app_configure_apache: True

install_files/ansible-base/roles/app/tasks/main.yml

@@ -3,14 +3,12 @@
 - include: create_app_dirs.yml
 
 - include: app_install_fpf_deb_pkgs.yml
-  when: "'development' not in group_names"
-  tags: non-development
+  when: securedrop_app_install_from_repo
 
 - include: initialize_securedrop_app.yml
 
 - include: install_and_harden_apache.yml
-  when: "'development' not in group_names"
-  tags: non-development
+  when: securedrop_app_configure_apache
 
 - include: configure_securedrop_worker.yml
 

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/rsync-filter

@@ -1,4 +1,5 @@
 exclude config.py
+include config.py.example
 include *.py
 include COPYING
 include dictionaries/
@@ -8,7 +9,7 @@ include journalist_templates/*.html
 include management/
 include management/*.py
 include requirements/
-include requirements/securedrop-requirements.txt
+include requirements/**.txt
 include source_templates/
 include source_templates/*.html
 include static/

install_files/ansible-base/roles/install-fpf-repo/defaults/main.yml

@@ -1,2 +1,6 @@
 ---
 apt_repo_url: https://apt.freedom.press
+
+# By default, install packages from the apt-repo, but under
+# staging hosts we'll prefer locally-built deb packages
+install_local_packages: False

install_files/ansible-base/roles/install-fpf-repo/tasks/main.yml

@@ -21,3 +21,5 @@
   apt:
     pkg: securedrop-keyring
     state: present
+  # Skip installing the keyring package from the apt repo under staging.
+  when: not install_local_packages

install_files/ansible-base/roles/install-local-packages/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+# Declare empty list var by default; host-specific overrides will provide
+# appropriate package lists for Application and Monitor servers.
+local_deb_packages: []

install_files/ansible-base/roles/install-local-packages/tasks/check_debs.yml

@@ -0,0 +1,24 @@
+---
+# Developers will need to build the OSSEC debs from a separate repo.
+# That's not immediately obvious for most new developers, so let's
+# provide a friendly message pointing to the docs.
+- name: Check for local deb files prior to installation.
+  local_action:
+    module: stat
+    path: ../../build/{{ item }}
+  with_items: "{{ local_deb_packages }}"
+  # Local action, so we don't want elevated privileges
+  sudo: no
+  # Read-only task, so don't report changed.
+  changed_when: false
+  register: local_packages_existence_check
+
+- name: Fail if local deb files are missing.
+  fail:
+    msg: >
+      Cannot find local Debian package "{{ item }}".
+      You will need to build the OSSEC packages in a separate repo
+      and copy them into the "build/" directory.
+      See the Developer docs for details: https://docs.securedrop.org/en/latest/development/virtual_environments.html#staging
+  when: item.stat.exists == false
+  with_items: "{{ local_packages_existence_check.results }}"

install_files/ansible-base/roles/install-local-packages/tasks/hold_debs.yml

@@ -0,0 +1,26 @@
+---
+# The Ansible config runs an aptitude safe-upgrade on every playbook invocation,
+# which will clobber the locally built packages with prod packages from the apt
+# repo. Let's set the packages to "hold" so they don't auto-upgrade.
+
+- name: Read package names from deb packages.
+  command: dpkg-deb -f /root/{{ item }} Package
+  register: local_deb_packages_name_check
+  changed_when: false
+  with_items: "{{ local_deb_packages }}"
+
+# Using apt-mark only applies to `apt` or `apt-get` actions, so we'll use a two-pass
+# approach to hold the locally built packages, so that neither apt nor aptitude
+# tries to upgrade them on subsequent playbook runs.
+- name: Mark packages as held, so they aren't upgraded automatically (via apt).
+  command: apt-mark hold {{ item.stdout }}
+  register: apt_mark_hold_result
+  changed_when: not apt_mark_hold_result.stdout.endswith('already set on hold.')
+  with_items: "{{ local_deb_packages_name_check.results }}"
+
+- name: Mark packages as held, so they aren't upgraded automatically (via aptitude).
+  command: aptitude hold {{ item.stdout }}
+  # `aptitude hold <package>` doesn't give any meaningful output to handle idempotence,
+  # so let's mark the task as no changes.
+  changed_when: false
+  with_items: "{{ local_deb_packages_name_check.results }}"

install_files/ansible-base/roles/install-local-packages/tasks/install_debs.yml

@@ -0,0 +1,29 @@
+# This will look for packages in install_files/ansible-base,
+# so the "build" directory is relative to playbook, which is
+# why the parent dirs ("..") are necessary.
+- name: Copy locally built deb packages to server (Staging only).
+  copy:
+    src: ../../build/{{ item }}
+    dest: /root/
+  with_items: "{{ local_deb_packages }}"
+
+# Using a two-pass approach for installing local deb packages.
+# The first pass uses `apt`, which will intelligently resolve dependencies;
+# a useful attribute, particular for the initial provisioning run. On subsequent
+# runs, however, the apt module will skip installation, since the version in
+# the DEBIAN/control file hasn't changed.
+- name: Install locally built deb packages (via apt).
+  apt:
+    deb: /root/{{ item.1 }}
+    force: yes
+  ignore_errors: yes
+  with_indexed_items: "{{ local_deb_packages }}"
+
+# Using `dpkg` via `command` to ensure installation ensure installation
+# every time, regardless of whether packages changed. SecureDrop deb package 
+# builds are not deterministic, so the `copy` task above will always report 
+# changed. Once the `apt` task above has installed the packages, only the 
+# `dpkg -i` calls will reinstall, ensuring the latest local code changes are used.
+- name: Install locally built deb packages (via dpkg).
+  command: dpkg -i /root/{{ item }}
+  with_items: "{{ local_deb_packages }}"

install_files/ansible-base/roles/install-local-packages/tasks/main.yml

@@ -1,20 +1,6 @@
 ---
-# This will look for packages in install_files/ansible-base,
-# so the "build" directory is relative to playbook, which is
-# why the parent dirs ("..") are necessary.
-- name: Copy locally built deb packages to server (Staging only).
-  copy:
-    src: ../../build/{{ item }}
-    dest: /root/
-  with_items: "{{ local_deb_packages }}"
+- include: check_debs.yml
 
-# There's a known bug in Ansible that causes installing
-# .deb packages via the apt module to fail when invoked
-# with `with_items`. A temporary workaround is to use
-# `with_indexed_items` and reference the value, which tricks
-# the `apt` module into installing each deb package separately,
-# rather than joining into a single comma-separated string.
-- name: Install locally built deb packages.
-  apt:
-    deb: /root/{{ item.1 }}
-  with_indexed_items: local_deb_packages
+- include: install_debs.yml
+
+- include: hold_debs.yml

install_files/ansible-base/roles/ossec-agent/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+# Override capability for installing locally built deb packages in the staging
+# environment. By default, packages are installed via the FPF apt repo.
+install_local_packages: False

install_files/ansible-base/roles/ossec-agent/tasks/agent_config.yml

@@ -3,6 +3,7 @@
   apt:
     name: securedrop-ossec-agent
     state: present
+  when: not install_local_packages
   tags:
     - apt
 

install_files/ansible-base/roles/ossec-server/defaults/main.yml

@@ -3,3 +3,6 @@ smtp_relay_cert_dir: /etc/ssl/certs
 smtp_relay_cert_override_dir: '/etc/ssl/certs_local'
 smtp_relay_cert_override_file: ''
 
+# Override capability for installing locally built deb packages in the staging
+# environment. By default, packages are installed via the FPF apt repo.
+install_local_packages: False

install_files/ansible-base/roles/ossec-server/tasks/mon_configure_ossec_gpg_alerts.yml

@@ -5,6 +5,7 @@
     state: latest
     update_cache: yes
     cache_valid_time: 3600
+  when: not install_local_packages
   tags:
     - apt
 

install_files/ansible-base/securedrop-prod.yml

@@ -26,8 +26,8 @@
     - prod-specific.yml
   roles:
     - { role: common, tags: common }
-    - { role: grsecurity, when: grsecurity, tags: [grsec, grsecurity] }
     - { role: tor-hidden-services, tags: tor }
+    - { role: grsecurity, when: grsecurity, tags: [grsec, grsecurity] }
   sudo: yes
 
 - name: Configure SecureDrop Monitor Server.