freedomofpress · conorsch · Feb 2, 2019 · Jan 16, 2019 · Jan 23, 2019 · Jan 17, 2019
@@ -91,7 +91,11 @@ build-debs-notest: ## Builds and tests debian packages (sans tests)
 
 .PHONY: build-debs-xenial
 build-debs-xenial: ## Builds and tests debian packages (includes Xenial overrides, TESTING ONLY)
-	@./devops/scripts/build-debs.sh xenial
+	@./devops/scripts/build-debs.sh test xenial
+
+.PHONY: build-debs-xenial-notest
+	build-debs-xenial-notest: ## Builds and tests debian packages (includes Xenial overrides, sans tests, TESTING ONLY)
+		@./devops/scripts/build-debs.sh notest xenial
 
 .PHONY: build-gcloud-docker
 build-gcloud-docker: ## Build docker container for gcloud sdk

@@ -1,2 +1,4 @@
 *.deb
+trusty/*.deb
+xenial/*.deb
 *.tar.gz
@@ -56,7 +56,7 @@ function copy_securedrop_repo() {
 # Main logic
 copy_securedrop_repo
 if [[ "$target_platform" = "xenial" ]]; then
-    ssh_gce "make build-debs-xenial"
+    ssh_gce "make build-debs-xenial-notest"
 else
     ssh_gce "make build-debs-notest"
 fi

@@ -11,8 +11,12 @@ set -o pipefail
 
 virtualenv_bootstrap
 
+RUN_TESTS="${1:-test}"
+TARGET_PLATFORM="${2:-trusty}"
+SCENARIO_NAME="builder-${TARGET_PLATFORM}"
+
 if [[ "${CIRCLE_BRANCH:-}" != docs-* ]]; then
-    case "${1:-test}" in
+    case "$RUN_TESTS" in
         notest)
             molecule_action=converge
             ;;
@@ -21,12 +25,7 @@ if [[ "${CIRCLE_BRANCH:-}" != docs-* ]]; then
             ;;
     esac
 
-    if [[ "${1:-trusty}" = "xenial" ]]; then
-	molecule converge -s builder -- -e securedrop_build_xenial_support=True;
-    else
-    	molecule "${molecule_action}" -s builder
-    fi
-
+    molecule "${molecule_action}" -s "${SCENARIO_NAME}"
 else
     echo Not running on docs branch...
 fi
@@ -41,18 +41,25 @@ Debian packages on the staging machines:
 .. code:: sh
 
    make build-debs
-   vagrant up /staging/
-   vagrant ssh app-staging
+   make staging
+   # Use the proper backend for your developer environment:
+   molecule login -s virtualbox-staging -h app-staging
+   # or:
+   molecule login -s libvirt-staging -h app-staging
    sudo su
    cd /var/www/securedrop
    ./manage.py add-admin
    pytest -v tests/
 
-To rebuild the local packages for the app code and update on staging: ::
+To rebuild the local packages for the app code and update on Trusty staging: ::
 
    make build-debs
-   vagrant up /staging/
-   vagrant provision
+   make staging
+
+To rebuild the local packages for the app code and update on Xenial staging: ::
+
+   make build-debs-xenial
+   make staging-xenial
 
 The Debian packages will be rebuilt from the current state of your
 local git repository and then installed on the staging servers.
@@ -111,9 +118,10 @@ the OSSEC-related configuration in
 ``install_files/ansible-base/staging.yml`` so you receive the OSSEC
 alert emails.
 
-Direct SSH access is available via Vagrant for staging hosts, so you can use
-``vagrant ssh app-staging`` and ``vagrant ssh mon-staging`` to start an
-interactive session on either server.
+Direct SSH access is available for staging hosts, so you can use
+``molecule login -s <scenario> -h app-staging``, where ``<scenario>``
+is either ``virtualbox-staging`` or ``libvirt-staging``, depending
+on your environment.
 
 .. _production_vms:
 
@@ -173,7 +181,7 @@ Install the required Vagrant plugins for converting and using libvirt boxes:
 Log out, then log in again. Verify that libvirt is installed and KVM is available:
 
 .. code:: sh
-  
+
    libvirtd --version
    kvm-ok
 
@@ -228,13 +236,16 @@ Set the default Vagrant provider to ``libvirt``:
 
 Convert Vagrant boxes to libvirt
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Convert the bento/14.04 Vagrant box from ``virtualbox`` to ``libvirt`` format:
+Convert the VirtualBox images for both Trusty & Xenial from ``virtualbox`` to ``libvirt`` format:
 
 .. code:: sh
 
    vagrant box add --provider virtualbox bento/ubuntu-14.04
    vagrant mutate bento/ubuntu-14.04 libvirt
 
+   vagrant box add --provider virtualbox bento/ubuntu-16.04
+   vagrant mutate bento/ubuntu-16.04 libvirt
+
 You can now use the libvirt-backed VM images to develop against
 the SecureDrop multi-machine environment.
 

@@ -18,7 +18,7 @@ base image. To set one up, run the following commands:
 
 .. code:: sh
 
-   vagrant box add bento/ubuntu-16.04  # choose the virtualbox option
+   vagrant box add --provider virtualbox bento/ubuntu-16.04
    vagrant mutate bento/ubuntu-16.04 mutate libvirt
 
 
@@ -74,24 +74,6 @@ Known bugs with Xenial support
 Below is a high-level overview of known problems to be addressed
 for delivering Xenial compatibility.
 
-Packaging
-    Dependencies for the ``securedrop-app-code`` deb package have changed;
-    ``apache2`` should be explicitly required; ``apache2-mpm-worker``
-    should be omitted.
-
-Firewall
-    The ``_apt`` user should be permitted to perform DNS and outbound TCP
-    calls on ports 80 and 443, rather than the ``root`` user.
-
-AppArmor
-    Explicit rules required for Apache mpm worker/event changes. ``gpg2``
-    policy should permit links via ``/var/lib/securedrop/keys/* l`` or similar.
-
-PAM logic
-    The PAM common-auth customizations include declarations for
-    ``pam_ecryptfs.so`` which prove problematic; commenting out ostensibly
-    resolves. More research required.
-
 Config tests
     The testinfra config test suite runs slightly different checks for
     Trusty and Xenial where appropriate. Care should be taken to preserve

@@ -25,7 +25,8 @@ apparmor_profiles:
   - usr.sbin.apache2
 
 # Installing the securedrop-app-code.deb package
-securedrop_app_code_deb: "securedrop-app-code-{{ securedrop_app_code_version }}-amd64" # do not enter .deb extension
+securedrop_staging_install_target_distro: trusty
+securedrop_app_code_deb: "securedrop-app-code_{{ securedrop_app_code_version }}+{{ securedrop_staging_install_target_distro }}_amd64" # do not enter .deb extension
 
 # Apt package dependencies for running the SecureDrop application.
 appserver_dependencies:
@@ -40,8 +41,6 @@ appserver_dependencies:
   - supervisor
   - libpython2.7-dev
 
-tor_apt_repo_url: https://tor-apt.freedom.press
-
 # Enable Tor over SSH by default
 enable_ssh_over_tor: true
 

@@ -7,7 +7,7 @@ ip_info:
 ### Used by the install_local_deb_pkgs role ###
 local_deb_packages:
   - "securedrop-keyring-0.1.2+{{ securedrop_app_code_version }}-amd64.deb"
-  - "securedrop-config-0.1.1+{{ securedrop_app_code_version }}-amd64.deb"
+  - "securedrop-config-0.1.2+{{ securedrop_app_code_version }}-amd64.deb"
   - "securedrop-ossec-agent-3.0.0+{{ securedrop_app_code_version }}-amd64.deb"
   - "{{ securedrop_app_code_deb }}.deb"
   - "ossec-agent-3.0.0-amd64.deb"

@@ -7,7 +7,7 @@ ip_info:
 ### Used by the install_local_deb_pkgs role ###
 local_deb_packages:
   - "securedrop-keyring-0.1.2+{{ securedrop_app_code_version }}-amd64.deb"
-  - "securedrop-config-0.1.1+{{ securedrop_app_code_version }}-amd64.deb"
+  - "securedrop-config-0.1.2+{{ securedrop_app_code_version }}-amd64.deb"
   - "securedrop-ossec-server-3.0.0+{{ securedrop_app_code_version }}-amd64.deb"
   - ossec-server-3.0.0-amd64.deb
 

@@ -1,7 +1,7 @@
 ---
 # Destination directory on Ansible controller where built deb packages
 # will be stored after fetching from build host.
-securedrop_local_build: "../../build"
+securedrop_local_build: "../../build/{{ ansible_distribution_release }}/"
 
 securedrop_generic_build_path: "/tmp/build/"
 

@@ -11,7 +11,7 @@ repo_src_path: "{{ build_path }}/ossec-{{ purpose }}"
 ossec_build_dir: "{{ build_path }}/ossec-{{ purpose }}-{{ ossec_version }}-amd64"
 
 # Destination on Ansible controller where built deb packages will be placed.
-ossec_build_deb_package_local_dest_dir: "../../build/"
+ossec_build_deb_package_local_dest_dir: "../../build/{{ ansible_distribution_release }}/"
 
 ossec_build_rsync_generic_opts:
   - "--exclude=*.git"

@@ -10,6 +10,13 @@
     - "{{ repo_src_path }}"
     - "{{ build_path }}"
 
+- name: Install python-requests for URL fetching
+  apt:
+    name: python-requests
+    state: present
+    update_cache: yes
+    cache_valid_time: 3600
+
 - name: Copy over current directory for repo build
   synchronize:
     # All Debian package source trees are located in `install_files/<package>`

@@ -20,3 +20,4 @@ securedrop_local_build: "../../build"
 # Whether to build packages targeting Xenial. Some of the dependency
 # logic changes, so we'll adjust the control file accordingly.
 securedrop_build_xenial_support: False
+securedrop_package_dist: "{{ 'xenial' if securedrop_build_xenial_support else 'trusty' }}"
@@ -1,6 +1,6 @@
-securedrop-app-code (0.12.0~rc1) trusty; urgency=medium
+securedrop-app-code (0.12.0~rc1+trusty) trusty; urgency=medium
 
-  * See changelog.md 
+  * See changelog.md
 
  -- SecureDrop Team <[email protected]>  Thu, 13 Dec 2018 02:02:05 +0000
 
@@ -12,13 +12,13 @@ securedrop-app-code (0.11.1) trusty; urgency=medium
 
 securedrop-app-code (0.11.0) trusty; urgency=medium
 
-  * See changelog.md 
+  * See changelog.md
 
  -- SecureDrop Team <[email protected]>  Tue, 11 Dec 2018 21:44:44 +0000
 
 securedrop-app-code (0.10.0) trusty; urgency=medium
 
-  * See changelog.md 
+  * See changelog.md
 
  -- SecureDrop Team <[email protected]>  Tue, 23 Oct 2018 17:53:14 +0000
 

@@ -0,0 +1,5 @@
+securedrop-app-code (0.12.0~rc1+xenial) xenial; urgency=medium
+
+  * See changelog.md
+
+ -- SecureDrop Team <[email protected]>  Thu, 13 Dec 2018 02:02:05 +0000
@@ -11,8 +11,6 @@
     delete: yes
     recursive: yes
 
-- debug: var=securedrop_code
-
 - name: Copy app code to build directory.
   synchronize:
     src: "{{ role_path }}/../../../../securedrop/"
@@ -46,20 +44,28 @@
   with_items: "{{ apparmor_profiles }}"
   tags: apparmor
 
-# Admittedly, this is a hack. It's off by default, but provides a toggle
-# for enabling Xenial support. So far, evaluation of Xenial has identified
-# package dependency logic as a blocker for installing the locally built
-# deb packages.
-- name: Support Xenial apache dependencies.
-  command: sed -i -e 's/apache2-mpm-worker,/apache2,/' {{ securedrop_app_code_deb_dir }}/DEBIAN/control
-  when: securedrop_build_xenial_support
+- name: Replace placeholder changelog to dist-specific changelog
+  copy:
+    src: "changelog-{{ securedrop_package_dist }}"
+    dest: "{{ securedrop_app_code_deb_dir }}/debian/changelog"
 
 - name: Build securedrop-app-code Debian package.
-  command: dpkg-deb --build {{ securedrop_app_code_deb_dir }}
+  command: dpkg-buildpackage -us -uc
+  args:
+    chdir: "{{ securedrop_app_code_deb_dir }}"
+
+- name: Find newly built Debian package
+  find:
+    paths:
+      - "{{ securedrop_app_code_deb_dir }}/../"
+    patterns:
+      - '*.deb'
+  register: securedrop_deb_package_find_results
 
 - name: Fetch newly built Debian packages back to localhost.
   fetch:
-    src: "{{ securedrop_app_code_deb_dir }}.deb"
+    src: "{{ item.path }}"
     dest: "{{ ossec_build_deb_package_local_dest_dir }}"
     flat: yes
     fail_on_missing: yes
+  with_items: "{{ securedrop_deb_package_find_results.files }}"
@@ -1,6 +1,5 @@
-deb http://security.ubuntu.com/ubuntu trusty-security main
-deb-src http://security.ubuntu.com/ubuntu trusty-security main
-deb http://security.ubuntu.com/ubuntu trusty-security universe
-deb-src http://security.ubuntu.com/ubuntu trusty-security universe
-deb [arch=amd64] {{ apt_repo_url }} trusty main
-deb {{ tor_apt_repo_url }} trusty main
+deb http://security.ubuntu.com/ubuntu {{ ansible_distribution_release }}-security main
+deb-src http://security.ubuntu.com/ubuntu {{ ansible_distribution_release }}-security main
+deb http://security.ubuntu.com/ubuntu {{ ansible_distribution_release }}-security universe
+deb-src http://security.ubuntu.com/ubuntu {{ ansible_distribution_release }}-security universe
+deb [arch=amd64] {{ apt_repo_url }} {{ ansible_distribution_release }} main
@@ -3,7 +3,7 @@
 # why the parent dirs ("..") are necessary.
 - name: Copy locally built deb packages to server (Staging only).
   copy:
-    src: ../../build/{{ item }}
+    src: ../../build/{{ securedrop_staging_install_target_distro }}/{{ item }}
     dest: /root/
   with_items: "{{ local_deb_packages }}"
 

@@ -2,7 +2,3 @@
 tor_hidden_services_parent_dir: /var/lib/tor/services
 tor_user: debian-tor
 enable_ssh_over_tor: true
-
-# Enabling support for trusty by default. Can be overridden to evaluate
-# xenial support.
-tor_apt_target_distro: "{{ ansible_distribution_release }}"
@@ -7,12 +7,10 @@
     - apt
     - tor
 
-- name: Setup Tor apt repo.
-  apt_repository:
-    filename: tor_apt_freedom_press
-    repo: deb {{ tor_apt_repo_url }} {{ tor_apt_target_distro }} main
-    state: present
-  register: add_tor_apt_repo
+- name: Remove Tor apt repo, so FPF repo is used.
+  file:
+    path: /etc/apt/sources.list.d/tor_apt_freedom_press.list
+    state: absent
   tags:
     - apt
     - tor

@@ -4,18 +4,21 @@ RunAsDaemon 1
 
 {% if 'securedrop_application_server' in group_names %}
 HiddenServiceDir /var/lib/tor/services/source
+HiddenServiceVersion 2
 HiddenServicePort 80 127.0.0.1:80
 {% if securedrop_app_https_on_source_interface|default(False) %}
 HiddenServicePort 443 127.0.0.1:443
 {% endif %}
 
 HiddenServiceDir /var/lib/tor/services/journalist
+HiddenServiceVersion 2
 HiddenServicePort 80 127.0.0.1:8080
 HiddenServiceAuthorizeClient stealth journalist
 {% endif %}
 
 {% if enable_ssh_over_tor %}
 HiddenServiceDir /var/lib/tor/services/ssh
+HiddenServiceVersion 2
 HiddenServicePort 22 127.0.0.1:22
 HiddenServiceAuthorizeClient stealth admin
 {% endif %}