diff --git a/Makefile b/Makefile index fc62269c91..47b4de5484 100644 --- a/Makefile +++ b/Makefile @@ -179,7 +179,7 @@ dev: ## Run the development server in a Docker container. .PHONY: staging staging: ## Create a local staging environment in virtual machines (Xenial) @echo "███ Creating staging environment on Ubuntu Xenial..." - @$(SDROOT)/devops/scripts/create-staging-env + @$(SDROOT)/devops/scripts/create-staging-env xenial @echo .PHONY: staging-focal diff --git a/devops/scripts/create-staging-env b/devops/scripts/create-staging-env index f55178cfc8..3b9a2c7f86 100755 --- a/devops/scripts/create-staging-env +++ b/devops/scripts/create-staging-env @@ -15,7 +15,7 @@ set -o pipefail . ./devops/scripts/boot-strap-venv.sh -securedrop_staging_scenario="$(./devops/scripts/select-staging-env)" +securedrop_staging_scenario="$(./devops/scripts/select-staging-env "${1}")" if [ -z "$TEST_DATA_FILE" ] then @@ -27,7 +27,7 @@ else fi fi -printf "Creating staging environment via '%s'...\n" "${securedrop_staging_scenario}" +printf "Creating staging environment via '%s'...\\n" "${securedrop_staging_scenario}" # Run it! virtualenv_bootstrap diff --git a/devops/scripts/select-staging-env b/devops/scripts/select-staging-env index 98483bb6cf..df67de7c6a 100755 --- a/devops/scripts/select-staging-env +++ b/devops/scripts/select-staging-env @@ -23,7 +23,6 @@ if [[ -n "${VAGRANT_DEFAULT_PROVIDER:-}" ]] ; then # environment uses Xenial template VMs only, so we also suppress the platform suffix. elif printenv | grep -q ^QUBES_ ; then securedrop_vm_provider="qubes" - securedrop_platform_suffix="" elif [[ "${OSTYPE:-}" == "linux-gnu" ]]; then # Default to Libvirt for Linux users, which works well with Tails VM virtualization. securedrop_vm_provider="libvirt" diff --git a/install_files/ansible-base/roles/app/tasks/initialize_securedrop_app.yml b/install_files/ansible-base/roles/app/tasks/initialize_securedrop_app.yml index 8d0892bcb7..70732a84c2 100644 --- a/install_files/ansible-base/roles/app/tasks/initialize_securedrop_app.yml +++ b/install_files/ansible-base/roles/app/tasks/initialize_securedrop_app.yml @@ -11,6 +11,7 @@ command: > su -s /bin/bash -c 'gpg --homedir {{ securedrop_data }}/keys + --no-default-keyring --keyring {{ securedrop_data }}/keys/pubring.gpg --import {{ securedrop_data }}/{{ securedrop_app_gpg_public_key }}' {{ securedrop_user }} register: gpg_app_key_import changed_when: "'imported: 1' in gpg_app_key_import.stderr" diff --git a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.apache2 b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.apache2 index 897a3266f3..345a944820 100644 --- a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.apache2 +++ b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.apache2 @@ -93,6 +93,7 @@ /sbin/ldconfig rix, /sbin/ldconfig.real rix, /tmp/** rwm, + /usr/bin/dash rix, /usr/bin/file rix, /usr/bin/gpg rix, /usr/bin/gpg-agent rix, @@ -101,6 +102,8 @@ /usr/bin/pinentry-gtk-2 rix, /usr/bin/shred rix, /usr/bin/srm rix, + /usr/bin/touch rix, + /usr/bin/uname rix, /usr/lib{,32,64}/** mr, /usr/share/file/magic r, /usr/share/file/magic.mgc r, diff --git a/install_files/ansible-base/tasks/reboot.yml b/install_files/ansible-base/tasks/reboot.yml index ad0244d5c4..743365a783 100644 --- a/install_files/ansible-base/tasks/reboot.yml +++ b/install_files/ansible-base/tasks/reboot.yml @@ -7,7 +7,7 @@ when: not securedrop_staging_qubes_env|default(False) - name: Gracefully halt Qubes staging VM - command: qvm-shutdown --wait {{ "sd-staging-app" if "app" in inventory_hostname else "sd-staging-mon" }} + command: qvm-shutdown --wait {{ "sd-staging-app" if "app" in inventory_hostname else "sd-staging-mon" }}-{{ securedrop_staging_install_target_distro }} become: no delegate_to: localhost when: securedrop_staging_qubes_env|default(False) @@ -17,7 +17,7 @@ - name: Boot Qubes staging VM shell: > qvm-start - {{ "sd-staging-app" if "app" in inventory_hostname else "sd-staging-mon" }} + {{ "sd-staging-app" if "app" in inventory_hostname else "sd-staging-mon" }}-{{ securedrop_staging_install_target_distro }} && sleep 30 become: no delegate_to: localhost diff --git a/molecule/qubes-staging/create.yml b/molecule/qubes-staging-focal/create.yml similarity index 100% rename from molecule/qubes-staging/create.yml rename to molecule/qubes-staging-focal/create.yml diff --git a/molecule/qubes-staging/destroy.yml b/molecule/qubes-staging-focal/destroy.yml similarity index 100% rename from molecule/qubes-staging/destroy.yml rename to molecule/qubes-staging-focal/destroy.yml diff --git a/molecule/qubes-staging/molecule.yml b/molecule/qubes-staging-focal/molecule.yml similarity index 86% rename from molecule/qubes-staging/molecule.yml rename to molecule/qubes-staging-focal/molecule.yml index e7e105c1bd..c1401105aa 100644 --- a/molecule/qubes-staging/molecule.yml +++ b/molecule/qubes-staging-focal/molecule.yml @@ -11,15 +11,15 @@ driver: platforms: - name: app-staging - vm_base: sd-staging-app-base - vm_name: sd-staging-app + vm_base: sd-staging-app-base-focal + vm_name: sd-staging-app-focal groups: - securedrop_application_server - staging - name: mon-staging - vm_base: sd-staging-mon-base - vm_name: sd-staging-mon + vm_base: sd-staging-mon-base-focal + vm_name: sd-staging-mon-focal groups: - securedrop_monitor_server - staging @@ -39,7 +39,7 @@ provisioner: env: ANSIBLE_CONFIG: ../../install_files/ansible-base/ansible.cfg scenario: - name: qubes-staging + name: qubes-staging-focal # Skip unnecessary "prepare" step in create sequence create_sequence: - create diff --git a/molecule/qubes-staging-focal/qubes-vars.yml b/molecule/qubes-staging-focal/qubes-vars.yml new file mode 100644 index 0000000000..43031e15cb --- /dev/null +++ b/molecule/qubes-staging-focal/qubes-vars.yml @@ -0,0 +1,29 @@ +--- +# Support dynamic lookups for Qubes host IPs. The staging vars +# in the Ansible config still assume hardcoded Vagrant-only IPs. +app_ip: "{{ hostvars['app-staging']['ansible_default_ipv4'].address }}" +monitor_ip: "{{ hostvars['mon-staging']['ansible_default_ipv4'].address }}" + +# Use hardcoded username from the manual VM provisioning step. +ssh_users: sdadmin + +# Override the default logic to determine remote host connection info. +# Since we're using the "delegated" driver in Molecule, there's no inventory +# file in play for the connection, only the "instance config" file. +# Molecule will try to connect to the hostname, e.g. "app-staging". +# Let's look up the IP address already written to the instance config file, +# and wait for that address when the VMs are rebooting. +remote_host_ref: >- + {{ lookup('file', lookup('env', 'MOLECULE_INSTANCE_CONFIG')) + | from_yaml + | selectattr('instance', 'eq', ansible_host) + | map(attribute='address') + | first + | default (ansible_host) + }} + +securedrop_staging_install_target_distro: focal + +# Inform the Ansible logic we're targeting Qubes staging VMs, +# helps to customize the reboot logic. +securedrop_staging_qubes_env: True diff --git a/molecule/qubes-staging/ssh_config.j2 b/molecule/qubes-staging-focal/ssh_config.j2 similarity index 100% rename from molecule/qubes-staging/ssh_config.j2 rename to molecule/qubes-staging-focal/ssh_config.j2 diff --git a/molecule/qubes-staging-xenial/create.yml b/molecule/qubes-staging-xenial/create.yml new file mode 100644 index 0000000000..77234875ac --- /dev/null +++ b/molecule/qubes-staging-xenial/create.yml @@ -0,0 +1,85 @@ +--- +- name: Create + hosts: localhost + connection: local + vars: + molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}" + molecule_instance_config: "{{ lookup('env', 'MOLECULE_INSTANCE_CONFIG') }}" + molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}" + tasks: + - name: Check that Qubes admin tools are installed + shell: > + which qvm-clone + || { echo 'qvm-clone not found, install qubes-core-admin-client'; + exit 1; } + changed_when: false + + - name: Clone base image for staging VMs + # The "ignore-errors" flag sidesteps an issue with qvm-sync-appmenus. We don't need + # app menus for the SD VMs, so an error there need not block provisioning. + command: qvm-clone {{ item.vm_base }} {{ item.vm_name }} --ignore-errors + register: clone_result + failed_when: >- + clone_result.rc != 0 and "qvm-clone: error: VM "+item.vm_name+" already exists" not in clone_result.stderr_lines + changed_when: >- + clone_result.rc == 0 and clone_result.stdout == "" + with_items: "{{ molecule_yml.platforms }}" + + - name: Start Qubes VMs + command: qvm-start {{ item.vm_name }} + register: start_result + failed_when: >- + start_result.rc != 0 and "domain "+item.vm_name+" is already running" not in start_result.stderr_lines + changed_when: >- + start_result.rc == 0 and start_result.stdout == "" + with_items: "{{ molecule_yml.platforms }}" + + - name: Wait for VMs to boot + pause: + seconds: 15 + when: start_result.changed + + - name: Get IP address for instances + command: qvm-ls --raw-data --field ip {{ item.vm_name }} + register: server_info + changed_when: false + # Not necessary, using pipe lookup to avoid convoluted Jinja logic. + when: false + with_items: "{{ molecule_yml.platforms }}" + + # Mandatory configuration for Molecule to function. + + - name: Populate instance config dict + set_fact: + instance_conf_dict: + instance: "{{ item.name }}" + address: "{{ lookup('pipe', 'qvm-ls --raw-data --field ip '+item.vm_name) }}" + identity_file: "~/.ssh/id_rsa" + port: "22" + # Hardcoded username, must match the username manually configured during + # base VM creation (see developer documentation). + user: "sdadmin" + with_items: "{{ molecule_yml.platforms }}" + register: instance_config_dict + when: start_result.changed | bool + + - name: Convert instance config dict to a list + set_fact: + instance_conf: "{{ instance_config_dict.results | map(attribute='ansible_facts.instance_conf_dict') | list }}" + when: start_result.changed | bool + + - name: render ssh_config for instances + template: + src: ssh_config.j2 + dest: "/tmp/molecule-qubes-ssh-config" + when: start_result.changed | bool + + - debug: var=instance_conf + + - name: Dump instance config + copy: + # NOTE(retr0h): Workaround for Ansible 2.2. + # https://github.com/ansible/ansible/issues/20885 + content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}" + dest: "{{ molecule_instance_config }}" + when: start_result.changed | bool diff --git a/molecule/qubes-staging-xenial/destroy.yml b/molecule/qubes-staging-xenial/destroy.yml new file mode 100644 index 0000000000..1c33e3facd --- /dev/null +++ b/molecule/qubes-staging-xenial/destroy.yml @@ -0,0 +1,45 @@ +--- + +- name: Destroy + hosts: localhost + connection: local + vars: + molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}" + molecule_instance_config: "{{ lookup('env',' MOLECULE_INSTANCE_CONFIG') }}" + molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}" + molecule_ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" + tasks: + - name: Check that Qubes admin tools are installed + shell: > + which qvm-clone + || { echo 'qvm-clone not found, install qubes-core-admin-client'; + exit 1; } + changed_when: false + + - name: Halt molecule instance(s) + command: qvm-shutdown --wait "{{ item.vm_name }}" + register: server + failed_when: >- + server.rc != 0 and "qvm-shutdown: error: no such domain: '"+item.vm_name+"'" not in server.stderr_lines + with_items: "{{ molecule_yml.platforms }}" + + - name: Destroy molecule instance(s) + command: qvm-remove --force "{{ item.vm_name }}" + register: server + failed_when: >- + server.rc != 0 and "qvm-remove: error: no such domain: '"+item.vm_name+"'" not in server.stderr_lines + with_items: "{{ molecule_yml.platforms }}" + + # Mandatory configuration for Molecule to function. + + - name: Populate instance config + set_fact: + instance_conf: {} + + - name: Dump instance config + copy: + # NOTE(retr0h): Workaround for Ansible 2.2. + # https://github.com/ansible/ansible/issues/20885 + content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}" + dest: "{{ molecule_instance_config }}" + when: server.changed | bool diff --git a/molecule/qubes-staging-xenial/molecule.yml b/molecule/qubes-staging-xenial/molecule.yml new file mode 100644 index 0000000000..82a21fffcf --- /dev/null +++ b/molecule/qubes-staging-xenial/molecule.yml @@ -0,0 +1,57 @@ +--- +driver: + name: delegated + options: + managed: True + login_cmd_template: 'ssh {instance} -F /tmp/molecule-qubes-ssh-config' + ansible_connection_options: + connection: ssh + ansible_ssh_common_args: -F /tmp/molecule-qubes-ssh-config + ansible_become_pass: securedrop + +platforms: + - name: app-staging + vm_base: sd-staging-app-base-xenial + vm_name: sd-staging-app-xenial + groups: + - securedrop_application_server + - staging + + - name: mon-staging + vm_base: sd-staging-mon-base-xenial + vm_name: sd-staging-mon-xenial + groups: + - securedrop_monitor_server + - staging + +provisioner: + name: ansible + lint: + name: ansible-lint + config_options: + defaults: + callback_whitelist: "profile_tasks, timer" + interpreter_python: auto + options: + e: "@qubes-vars.yml" + playbooks: + converge: ../../install_files/ansible-base/securedrop-staging.yml + env: + ANSIBLE_CONFIG: ../../install_files/ansible-base/ansible.cfg +scenario: + name: qubes-staging-xenial + # Skip unnecessary "prepare" step in create sequence + create_sequence: + - create + test_sequence: + - destroy + - create + - converge +verifier: + name: testinfra + lint: + name: flake8 + directory: ../testinfra + options: + n: auto + v: 2 diff --git a/molecule/qubes-staging/qubes-vars.yml b/molecule/qubes-staging-xenial/qubes-vars.yml similarity index 100% rename from molecule/qubes-staging/qubes-vars.yml rename to molecule/qubes-staging-xenial/qubes-vars.yml diff --git a/molecule/qubes-staging-xenial/ssh_config.j2 b/molecule/qubes-staging-xenial/ssh_config.j2 new file mode 100644 index 0000000000..4a51d69162 --- /dev/null +++ b/molecule/qubes-staging-xenial/ssh_config.j2 @@ -0,0 +1,8 @@ +{% for host in instance_conf %} +Host {{ host.instance }} + HostName {{ host.address }} + Port {{ host.port }} + IdentityFile {{ host.identity_file }} + PreferredAuthentications publickey + User {{ host.user }} +{%endfor%}