diff --git a/Makefile b/Makefile index 20e257ff0e..6b303ab766 100644 --- a/Makefile +++ b/Makefile @@ -232,34 +232,15 @@ docker-vnc: ## Open a VNC connection to a running Docker instance. @echo .PHONY: upgrade-start -upgrade-start: ## Boot an upgrade test environment using libvirt. +upgrade-start: ## Create a local apt server for testing upgrades in VMs @echo "███ Starting upgrade test environment..." - @SD_UPGRADE_BASE=$(STABLE_VER) molecule converge -s upgrade - @echo - -.PHONY: upgrade-start-qa -upgrade-start-qa: ## Boot an upgrade test env using libvirt in remote apt mode. - @echo "███ Starting upgrade test environment for remote apt..." - @SD_UPGRADE_BASE=$(STABLE_VER) QA_APTTEST=yes molecule converge -s upgrade + molecule converge -s upgrade @echo .PHONY: upgrade-destroy upgrade-destroy: ## Destroy an upgrade test environment. @echo "███ Destroying upgrade test environment..." - @SD_UPGRADE_BASE=$(STABLE_VER) molecule destroy -s upgrade - @echo - -.PHONY: upgrade-test-local -upgrade-test-local: ## Upgrade a running test environment with local apt packages. - @echo "Testing upgrade with local apt packages..." - @molecule side-effect -s upgrade - @echo - -.PHONY: upgrade-test-qa -upgrade-test-qa: ## Upgrade a running test environment with apt packages from the QA repo. - @echo "Testing upgrade with packages from QA apt repo..." - @QA_APTTEST=yes molecule converge -s upgrade -- --diff -t apt - @QA_APTTEST=yes molecule side-effect -s upgrade + molecule destroy -s upgrade @echo ############## diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index ec7423e8e2..0000000000 --- a/Vagrantfile +++ /dev/null @@ -1,122 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -Vagrant.require_version ">= 2.1.2" - -Vagrant.configure("2") do |config| - - # Vagrant 1.7.0+ removes the insecure_private_key by default - # and substitutes a dynamically generated SSH key for each box. - # Unfortunately this breaks Ansible provisioning with Vagrant, - # so the key insertion feature should be disabled. - config.ssh.insert_key = false - - # The prod hosts are just like production but are virtualized. - # All access to SSH and the web interfaces is only over Tor. - config.vm.define 'mon-prod', autostart: false do |prod| - if ENV['SECUREDROP_SSH_OVER_TOR'] - config.ssh.host = find_ssh_aths("mon-ssh-aths") - config.ssh.proxy_command = tor_ssh_proxy_command - config.ssh.port = 22 - end - prod.vm.hostname = "mon-prod" - prod.vm.box = "bento/ubuntu-20.04" - prod.vm.network "private_network", ip: "10.0.1.5" - prod.vm.synced_folder './', '/vagrant', disabled: true - prod.vm.provider "libvirt" do |lv, override| - lv.video_type = "virtio" - end - end - - config.vm.define 'app-prod', autostart: false do |prod| - if ENV['SECUREDROP_SSH_OVER_TOR'] - config.ssh.host = find_ssh_aths("app-ssh-aths") - config.ssh.proxy_command = tor_ssh_proxy_command - config.ssh.port = 22 - end - prod.vm.hostname = "app-prod" - prod.vm.box = "bento/ubuntu-20.04" - prod.vm.network "private_network", ip: "10.0.1.4" - prod.vm.synced_folder './', '/vagrant', disabled: true - prod.vm.provider "libvirt" do |lv, override| - lv.memory = 1024 - lv.video_type = "virtio" - end - prod.vm.provision "ansible" do |ansible| - ansible.playbook = "install_files/ansible-base/securedrop-prod.yml" - ansible.verbose = 'v' - # the production playbook verifies that staging default values are not - # used will need to skip the this role to run in Vagrant - ansible.raw_arguments = Shellwords.shellsplit(ENV['ANSIBLE_ARGS']) if ENV['ANSIBLE_ARGS'] - # Taken from the parallel execution tips and tricks - # https://docs.vagrantup.com/v2/provisioning/ansible.html - ansible.limit = 'all,localhost' - ansible.groups = { - 'securedrop_application_server' => %w(app-prod), - 'securedrop_monitor_server' => %w(mon-prod), - 'securedrop' => %w(app-prod mon-prod) - } - end - end - - config.vm.define 'apt-local', autostart: false do |prod| - prod.vm.hostname = "apt-local" - prod.vm.box = "bento/ubuntu-20.04" - prod.vm.network "private_network", ip: "10.0.1.7" - prod.vm.synced_folder './', '/vagrant', disabled: true - prod.vm.provider "libvirt" do |lv, override| - lv.memory = 1024 - lv.video_type = "virtio" - end - prod.vm.provision "ansible" do |ansible| - ansible.playbook = "devops/apt-local.yml" - ansible.galaxy_role_file = "molecule/upgrade/requirements.yml" - ansible.galaxy_roles_path = ".galaxy_roles" - ansible.verbose = 'v' - # the production playbook verifies that staging default values are not - # used will need to skip the this role to run in Vagrant - ansible.raw_arguments = Shellwords.shellsplit(ENV['ANSIBLE_ARGS']) if ENV['ANSIBLE_ARGS'] - # Taken from the parallel execution tips and tricks - # https://docs.vagrantup.com/v2/provisioning/ansible.html - ansible.limit = 'all,localhost' - end - end - -end - - -# Get .onion URL for connecting to instances over Tor. -# The Ansible playbooks fetch these values back to the -# Admin Workstation (localhost) so they can be manually -# added to the inventory file. Possible values for filename -# are "app-ssh-aths" and "mon-ssh-aths". -def find_ssh_aths(filename) - repo_root = File.expand_path(File.dirname(__FILE__)) - aths_file = File.join(repo_root, "install_files", "ansible-base", filename) - if FileTest.file?(aths_file) - File.open(aths_file).each do |line| - # Take second value for URL; format for the ATHS file is: - # /^HidServAuth \w{16}.onion \w{22} # client: admin$/ - return line.split()[1] - end - else - puts "Failed to find ATHS file: #{filename}" - puts "Cannot connect via SSH." - exit(1) - end -end - -# Build proxy command for connecting to prod instances over Tor. -def tor_ssh_proxy_command - def command?(command) - system("which #{command} > /dev/null 2>&1") - end - if command?("nc") - base_cmd = "nc -x" - else - puts "Failed to build proxy command for SSH over Tor." - puts "Install or 'netcat-openbsd'." - exit(1) - end - return "#{base_cmd} 127.0.0.1:9050 %h %p" -end diff --git a/install_files/ansible-base/securedrop-apt-local.yml b/install_files/ansible-base/securedrop-apt-local.yml index 20b1e0a5b5..b04888944f 100644 --- a/install_files/ansible-base/securedrop-apt-local.yml +++ b/install_files/ansible-base/securedrop-apt-local.yml @@ -1,16 +1,16 @@ --- -# Playbook to update SecureDrop VMs to install Focal packages from a local repo +# Playbook to update SecureDrop VMs to via local repo # # Steps to use this playbook: # -# 1. On host machine, build packages with `make build-debs-focal` -# 2. On host machine, provision local apt repo with `vagrant up apt-local` +# 1. On host machine, build packages with `make build-debs` +# 2. On host machine, provision local apt repo with `make upgrade-start` # 3. Switch to Admin Workstation -# 3. Continue with prod provisioning as far as `./securedrop-admin sdconfig` +# 3. Continue with prod provisioning as far as `./securedrop-admin tailsconfig` # 5. Run `source admin/.venv3/bin/activate` (so ansible commands work) # 6. Run `cd install_files/ansible-base` # 7. Run `ansible-playbook -vv --diff securedrop-apt-local.yml` -# 8. Proceed with `./securedrop-admin install` +# 8. Proceed with upgrading packages inside VMs - name: Configure prod host to prioritize local packages. environment: diff --git a/molecule/libvirt-prod-focal/molecule.yml b/molecule/libvirt-prod-focal/molecule.yml new file mode 100644 index 0000000000..3e21516843 --- /dev/null +++ b/molecule/libvirt-prod-focal/molecule.yml @@ -0,0 +1,40 @@ +--- +dependency: + name: galaxy +driver: + name: vagrant + provider: + name: libvirt +platforms: + - name: app-prod + box: bento/ubuntu-20.04 + raw_config_args: + - "cpu_mode = 'host-passthrough'" + - "video_type = 'virtio'" + instance_raw_config_args: + - "vm.synced_folder './', '/vagrant', disabled: true" + - "vm.network 'private_network', ip: '10.0.1.4'" + - "ssh.insert_key = false" + memory: 1024 + private_ip: 10.0.1.4 + groups: + - securedrop_application_server + - securedrop + + - name: mon-prod + box: bento/ubuntu-20.04 + raw_config_args: + - "cpu_mode = 'host-passthrough'" + - "video_type = 'virtio'" + instance_raw_config_args: + - "vm.synced_folder './', '/vagrant', disabled: true" + - "vm.network 'private_network', ip: '10.0.1.5'" + - "ssh.insert_key = false" + memory: 1024 + private_ip: 10.0.1.5 + groups: + - securedrop_monitor_server + - securedrop + +scenario: + name: libvirt-prod-focal diff --git a/molecule/shared/sd_clone.yml b/molecule/shared/sd_clone.yml deleted file mode 100755 index a9bedc0939..0000000000 --- a/molecule/shared/sd_clone.yml +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env ansible-playbook ---- -- name: Bring in SecureDrop production version - hosts: localhost - gather_facts: false - tasks: - - debug: - msg="Starting scenario from baseline {{ ORIG_SECUREDROP_VER }}" - - - name: Ensure molecule local ephemeral dir exists - file: - state: directory - path: "{{ molecule_scenario_dir }}/.molecule" - - - name: Clone in copy of original SD version - git: - repo: https://github.com/freedomofpress/securedrop.git - dest: "{{ molecule_ephemeral_directory }}/{{ sd_clone_dir }}" - version: "{{ ORIG_SECUREDROP_VER }}" - depth: 1 - - - name: Create hacky symlink for roles dir - file: - dest: "{{ molecule_scenario_dir }}/.molecule/roles" - src: "{{ molecule_ephemeral_directory }}/{{ sd_clone_dir }}/{{ sd_ansible_roles }}" - state: link - force: yes - - # Required for mutual compatibility during Ansible 2.7.x -> 2.9.x transition - # We moved to 2.9.x in v1.3.0, so don't run if historical version is >= to 1.3.0. - - name: Patch Tor handler - lineinfile: - dest: "{{ molecule_ephemeral_directory }}/{{ sd_clone_dir }}/{{ sd_ansible_roles }}/tor-hidden-services/handlers/main.yml" - regexp: '^ include:' - line: ' include_tasks: "{% raw %}{{ role_path }}{% endraw %}/handlers/restart-tor-carefully.yml"' - when: > - ORIG_SECUREDROP_VER.startswith("1.2") or - ORIG_SECUREDROP_VER.startswith("1.1") or - ORIG_SECUREDROP_VER.startswith("1.0") - vars: - sd_clone_dir: "sd-orig" - sd_ansible_roles: "install_files/ansible-base/roles" - molecule_scenario_dir: "{{lookup('env', 'MOLECULE_SCENARIO_DIRECTORY')}}" - ORIG_SECUREDROP_VER: "{{ sd_version|default(lookup('file',playbook_dir+'/stable.ver',rstrip=True)) }}" - molecule_ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" diff --git a/molecule/upgrade/ansible-override-vars.yml b/molecule/upgrade/ansible-override-vars.yml deleted file mode 100644 index f1f745ae53..0000000000 --- a/molecule/upgrade/ansible-override-vars.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -install_local_packages: false -primary_network_iface: eth0 -ssh_users: vagrant -securedrop_app_install_from_repo: true -allow_direct_access: true -ssh_listening_address: 0.0.0.0 - -monitor_ip: "{{ hostvars['mon-staging']['ansible_'+primary_network_iface].ipv4.address }}" -monitor_hostname: "{{ hostvars['mon-staging'].ansible_hostname }}" -app_ip: "{{ hostvars['app-staging']['ansible_'+primary_network_iface].ipv4.address }}" -app_hostname: "{{ hostvars['app-staging'].ansible_hostname }}" -securedrop_code: "/var/www/securedrop" - -etc_hosts: - app-staging: - - reg: ^127.0.0.1 - line: 127.0.0.1 app-staging localhost - - reg: securedrop-monitor-server-alias$ - line: "{{ monitor_ip }} mon-staging securedrop-monitor-server-alias" - mon-staging: - - reg: ^127.0.0.1\s+mon - line: 127.0.0.1 mon-staging localhost - - reg: app-staging$ - line: "{{ app_ip }} app-staging" - -# Set Xenial as apt repo channel for hosting deb packages -rep_dist: xenial - -# Override path to repo root, to use the nested repo on the previous version. -# See molecule/shared/sd_clone.yml for details on the path structure. -sd_root_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/sd-orig" diff --git a/molecule/upgrade/apt.yml b/molecule/upgrade/apt.yml deleted file mode 100644 index 51d9041814..0000000000 --- a/molecule/upgrade/apt.yml +++ /dev/null @@ -1,73 +0,0 @@ ---- - -- name: Configure apt-server - hosts: aptservers - become: yes - tasks: - - name: Establish QA strategy - set_fact: - QA_APTTEST: "{{ lookup('env','QA_APTTEST')|bool }}" - - - import_tasks: local_apt_mirror.yml - - import_tasks: local_apt_with_debs.yml - when: not QA_APTTEST - - vars: - molecule_dir: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}" - dpkg_dir: /var/repos/debs - rep_component: main - rep_arch: i386 amd64 - release_file: "/var/repos/base/dists/{{ rep_dist }}/Release" - nginx_sites: - default: - - listen 80 - - root "/var/repos/base" - - location / { autoindex on; } - - location /gpg { alias /var/repos/base/; } - encrypted: - - listen 443 ssl - - server_name apt.freedom.press - - ssl_certificate /etc/ssl/certs/apt_freedom_press.pem - - ssl_certificate_key /etc/ssl/private/apt_freedom_press.priv - - root "/var/repos/base" - - location / { {{ "proxy_pass https://apt-test.freedom.press;" if QA_APTTEST else "autoindex on;" }} } - -- name: Configure apt-server - hosts: securedrop - gather_facts: false - become: yes - tasks: - - block: - - name: Redirect to local QA Apt server - set_fact: - apt_key_bits: - - url: "http://{{ hostvars['apt']['ansible_eth0'].ipv4.address }}/gpg/apt-test.pub" - id: 6D65484B - - url: "http://{{ hostvars['apt']['ansible_eth0'].ipv4.address }}/gpg/apt-test-fpf.pub" - id: 4A3BE4A92211B03C - apt_test_etc_line: "{{ hostvars['apt']['ansible_eth0'].ipv4.address }} apt.freedom.press" - - - name: Install testing apt key to keyring - apt_key: - id: "{{ item.id|default(omit) }}" - url: "{{ item.url|default(omit) }}" - data: "{{ item.data|default(omit) }}" - state: present - with_items: "{{ apt_key_bits }}" - - - name: Redirect apt.freedom.press to local apt server - lineinfile: - path: /etc/hosts - regexp: apt.freedom.press$ - line: "{{ apt_test_etc_line }}" - - - name: Add apt-test CA to CA trusted store - copy: - src: cacert.pub - dest: /usr/local/share/ca-certificates/fpf_test_ca.crt - notify: update ca - tags: apt - - handlers: - - name: update ca - command: update-ca-certificates diff --git a/devops/apt-local.yml b/molecule/upgrade/converge.yml similarity index 80% rename from devops/apt-local.yml rename to molecule/upgrade/converge.yml index 62adab0813..bd10ba9fe6 100644 --- a/devops/apt-local.yml +++ b/molecule/upgrade/converge.yml @@ -2,12 +2,14 @@ - name: Confirm local debs present hosts: localhost gather_facts: no + vars: + molecule_dir: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}" tasks: # Before running apt-server logic, confirm we have local # debs built. - name: Find local deb files find: - paths: "../build/focal/" + paths: "{{ molecule_dir }}/../../build/focal/" patterns: "*.deb" register: _find_debs_result @@ -21,13 +23,13 @@ hosts: apt-local become: yes tasks: - - import_tasks: ../molecule/upgrade/local_apt_mirror.yml - - import_tasks: ../molecule/upgrade/local_apt_with_debs.yml + - import_tasks: local_apt_mirror.yml + - import_tasks: local_apt_with_debs.yml vars: QA_APTTEST: False rep_dist: "focal" - molecule_dir: "../molecule/upgrade" + molecule_dir: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}" dpkg_dir: /var/repos/debs rep_origin: SecureDrop rep_component: main diff --git a/molecule/upgrade/create.yml b/molecule/upgrade/create.yml deleted file mode 100644 index af96c87eb3..0000000000 --- a/molecule/upgrade/create.yml +++ /dev/null @@ -1,59 +0,0 @@ ---- -- import_playbook: ../shared/sd_clone.yml sd_version="{{ lookup('env','SD_UPGRADE_BASE') }}" - -- name: Create - hosts: localhost - connection: local - vars: - molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}" - molecule_instance_config: "{{ lookup('env', 'MOLECULE_INSTANCE_CONFIG') }}" - molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}" - tasks: - - - name: Create molecule instance(s) - vagrant: - instance_name: "{{ item.name }}" - instance_interfaces: "{{ item.interfaces | default(omit) }}" - instance_raw_config_args: "{{ item.instance_raw_config_args | default(omit) }}" - - platform_box: "{{ item.box }}" - platform_box_version: "{{ omit if 'apt' in item.name else lookup('env','SD_UPGRADE_BASE') }}" - platform_box_url: "{{ 'file://'+playbook_dir+'/' if item.box_url|default(false) else '' }}{{ item.box_url | default(omit) }}" - - provider_name: "{{ molecule_yml.driver.provider.name }}" - provider_memory: "{{ item.memory | default(omit) }}" - provider_cpus: "{{ item.cpus | default(omit) }}" - provider_override_args: "{{ item.provider_override_args | default(omit) }}" - provider_raw_config_args: "{{ item.raw_config_args | default(omit) }}" - force_stop: yes - - state: up - register: server - with_items: "{{ molecule_yml.platforms }}" - - # Mandatory configuration for Molecule to function. - - - name: Populate instance config dict - set_fact: - instance_conf_dict: { - 'instance': "{{ item.Host }}", - 'address': "{{ item.HostName }}", - 'user': "{{ item.User }}", - 'port': "{{ item.Port }}", - 'identity_file': "{{ item.IdentityFile }}", } - with_items: "{{ server.results }}" - register: instance_config_dict - when: server.changed | bool - - - name: Convert instance config dict to a list - set_fact: - instance_conf: "{{ instance_config_dict.results | map(attribute='ansible_facts.instance_conf_dict') | list }}" - when: server.changed | bool - - - name: Dump instance config - copy: - # NOTE(retr0h): Workaround for Ansible 2.2. - # https://github.com/ansible/ansible/issues/20885 - content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}" - dest: "{{ molecule_instance_config }}" - when: server.changed | bool diff --git a/molecule/upgrade/destroy.yml b/molecule/upgrade/destroy.yml deleted file mode 100644 index b98563c051..0000000000 --- a/molecule/upgrade/destroy.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- - -- name: Destroy - hosts: localhost - connection: local - vars: - molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}" - molecule_instance_config: "{{ lookup('env',' MOLECULE_INSTANCE_CONFIG') }}" - molecule_scenario_dir: "{{ lookup('env','MOLECULE_SCENARIO_DIRECTORY') }}" - molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}" - molecule_ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" - tasks: - - name: Destroy molecule instance(s) - vagrant: - instance_name: "{{ item.name }}" - platform_box: "{{ item.box }}" - provider_name: "{{ molecule_yml.driver.provider.name }}" - force_stop: "{{ item.force_stop | default(True) }}" - - state: destroy - register: server - with_items: "{{ molecule_yml.platforms }}" - - - name: Destroy local molecule ephemeral dir - file: - state: absent - path: "{{ molecule_scenario_dir }}/.molecule" - - # Mandatory configuration for Molecule to function. - - - name: Populate instance config - set_fact: - instance_conf: {} - - - name: Dump instance config - copy: - # NOTE(retr0h): Workaround for Ansible 2.2. - # https://github.com/ansible/ansible/issues/20885 - content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}" - dest: "{{ molecule_instance_config }}" - when: server.changed | bool diff --git a/molecule/upgrade/molecule.yml b/molecule/upgrade/molecule.yml index f168435209..6a57669a9e 100644 --- a/molecule/upgrade/molecule.yml +++ b/molecule/upgrade/molecule.yml @@ -9,39 +9,17 @@ lint: | yamllint platforms: - - name: app-staging - box: fpf/securedrop-app-xenial - box_url: "../vagrant-packager/box_files/app_xenial_metadata.json" + - name: apt-local + box: bento/ubuntu-20.04 + raw_config_args: + - "cpu_mode = 'host-passthrough'" + - "video_type = 'virtio'" instance_raw_config_args: + - "vm.synced_folder './', '/vagrant', disabled: true" + - "vm.network 'private_network', ip: '10.0.1.7'" - "ssh.insert_key = false" - provider_override_args: - - "vm.synced_folder './', '/vagrant', disabled: true, type: 'nfs'" memory: 1024 - private_ip: 10.0.1.2 - groups: - - securedrop_application_server - - securedrop - - staging - - - name: mon-staging - box: fpf/securedrop-mon-xenial - box_url: "../vagrant-packager/box_files/mon_xenial_metadata.json" - instance_raw_config_args: - - "ssh.insert_key = false" - provider_override_args: - - "vm.synced_folder './', '/vagrant', disabled: true, type: 'nfs'" - memory: 1024 - private_ip: 10.0.1.3 - groups: - - securedrop_monitor_server - - securedrop - - staging - - - name: apt - box: bento/ubuntu-16.04 - provider_override_args: - - "vm.synced_folder './', '/vagrant', disabled: true, type: 'nfs'" - memory: 256 + private_ip: 10.0.1.7 groups: - aptservers @@ -49,33 +27,11 @@ provisioner: name: ansible lint: | ansible-lint - inventory: - links: - group_vars: ../../install_files/ansible-base/group_vars - host_vars: ../../install_files/ansible-base/host_vars env: - ANSIBLE_ROLES_PATH: ".molecule/roles" - options: - e: "@ansible-override-vars.yml" - config_options: - defaults: - interpreter_python: auto - ssh_connection: - pipelining: True - playbooks: - side_effect: side_effect.yml - connection_options: - ansible_ssh_common_args: -o IdentitiesOnly=yes + ANSIBLE_CONFIG: ../../install_files/ansible-base/ansible.cfg scenario: name: upgrade - create_sequence: - - create - - dependency - converge_sequence: - - create - - dependency - - converge verifier: name: testinfra lint: | diff --git a/molecule/upgrade/playbook.yml b/molecule/upgrade/playbook.yml deleted file mode 100644 index c028ec6ac2..0000000000 --- a/molecule/upgrade/playbook.yml +++ /dev/null @@ -1,97 +0,0 @@ ---- -- name: Confirm local debs present if required - hosts: localhost - gather_facts: no - vars: - molecule_dir: "{{ lookup('env', 'MOLECULE_SCENARIO_DIRECTORY') }}" - tasks: - # Before running apt-server logic, confirm we have local - # debs built. - - name: Establish QA strategy - set_fact: - QA_APTTEST: "{{ lookup('env','QA_APTTEST')|bool }}" - - - name: Find local deb files - find: - paths: "{{ molecule_dir }}/../../build/{{ rep_dist }}/" - patterns: "*.deb" - register: _upgrade_scenario_find_debs_result - when: not QA_APTTEST - - - name: Ensure debs were found - assert: - that: - - "_upgrade_scenario_find_debs_result.files|length >= 8" - msg: "No local debs found, run 'make build-debs'" - when: not QA_APTTEST - -- name: Re-run original Tor role - hosts: securedrop - max_fail_percentage: 0 - any_errors_fatal: yes - pre_tasks: - - name: Fix hosts file - lineinfile: - path: /etc/hosts - line: "{{ item.line }}" - regexp: "{{ item.reg }}" - with_items: "{{ etc_hosts[ansible_hostname] }}" - tags: hosts - roles: - - role: tor-hidden-services - tags: tor - - role: restrict-direct-access - tags: firewall - - role: ossec - tags: ossec - post_tasks: - - name: Make sure worker directory is created? - file: - state: directory - path: "/var/log/{{ item }}" - with_items: - - cron-apt - # Reuse existing "reboot" logic from site config - - import_tasks: ../../install_files/ansible-base/roles/reboot-if-first-install/tasks/reboot-no-check.yml - tags: reboot - - - name: Wait longer for machines to reboot - pause: - seconds: 60 - become: yes - -- name: Re-run OSSEC role - hosts: securedrop - max_fail_percentage: 0 - any_errors_fatal: yes - roles: - - role: ossec - tags: ossec - post_tasks: - - name: Ensure Tor is running - service: - name: tor - state: started - enabled: yes - tags: always - become: yes - -- import_playbook: apt.yml - tags: apt - -- name: Spit out details - hosts: localhost - gather_facts: false - tasks: - - name: Spit out Tor details - debug: - msg: - - "V2 Source interface available at {{ lookup('file', molecule_ephemeral_directory+'/sd-orig/'+source_ths_path) }}" - - "V3 Source interface available at {{ lookup('file', molecule_ephemeral_directory+'/sd-orig/'+source_ths_v3_path) }}" - tags: - - onion - - always - vars: - molecule_ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" - source_ths_path: /install_files/ansible-base/app-source-ths - source_ths_v3_path: /install_files/ansible-base/app-sourcev3-ths diff --git a/molecule/upgrade/prepare.yml b/molecule/upgrade/prepare.yml deleted file mode 100644 index 112da19982..0000000000 --- a/molecule/upgrade/prepare.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Prepare - hosts: all - gather_facts: False - tasks: - - name: Install python for Ansible - raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) - become: True - changed_when: False diff --git a/molecule/upgrade/side_effect.yml b/molecule/upgrade/side_effect.yml deleted file mode 100644 index 246255575b..0000000000 --- a/molecule/upgrade/side_effect.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -- name: Perform apt upgrades - hosts: securedrop - become: yes - tasks: - - name: Perform safe upgrade - apt: - update_cache: yes - upgrade: yes - -- name: Lay out app testing deps - hosts: securedrop_application_server - max_fail_percentage: 0 - any_errors_fatal: yes - roles: - - role: app-test - tags: app-test - - tasks: - - name: Reset database - command: ./manage.py reset - args: - chdir: /var/www/securedrop - - - name: Slap in latest loaddata script - copy: - src: ../../securedrop/loaddata.py - dest: /var/www/securedrop/loaddata.py - mode: 0555 - - - name: Insert journalist test user - command: /var/www/securedrop/loaddata.py - environment: - CRYPTOGRAPHY_ALLOW_OPENSSL_102: "True" - args: - chdir: /var/www/securedrop - become: yes diff --git a/molecule/vagrant-packager/ansible-override-vars.yml b/molecule/vagrant-packager/ansible-override-vars.yml deleted file mode 100644 index 8f748bfeb2..0000000000 --- a/molecule/vagrant-packager/ansible-override-vars.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -install_local_packages: false -primary_network_iface: eth0 -ssh_users: vagrant -securedrop_app_install_from_repo: true -allow_direct_access: true -ssh_listening_address: 0.0.0.0 -app_ip: "{{ hostvars['app-staging']['ansible_'+primary_network_iface].ipv4.address }}" -monitor_ip: "{{ hostvars['mon-staging']['ansible_'+primary_network_iface].ipv4.address }}" -apt_repo_url: https://apt.freedom.press - -# Override path to repo root, to use the nested repo on the previous version. -# See molecule/shared/sd_clone.yml for details on the path structure. -sd_root_dir: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}/sd-orig" diff --git a/molecule/vagrant-packager/box_files/Vagrantfile.app-staging b/molecule/vagrant-packager/box_files/Vagrantfile.app-staging deleted file mode 100644 index dcf02d3bf4..0000000000 --- a/molecule/vagrant-packager/box_files/Vagrantfile.app-staging +++ /dev/null @@ -1,18 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -Vagrant.configure("2") do |config| - - config.ssh.insert_key = false - config.vm.hostname = "app-staging" - - config.vm.box = "bento/ubuntu-14.04" - - config.vm.provider "libvirt" do |lv, override| - lv.memory = 1024 - override.vm.synced_folder './', '/vagrant', type: 'nfs', disabled: false - end - - config.vm.network "private_network", ip: "10.0.1.2" - -end diff --git a/molecule/vagrant-packager/box_files/Vagrantfile.mon-staging b/molecule/vagrant-packager/box_files/Vagrantfile.mon-staging deleted file mode 100644 index 8b090a6052..0000000000 --- a/molecule/vagrant-packager/box_files/Vagrantfile.mon-staging +++ /dev/null @@ -1,15 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -Vagrant.configure("2") do |config| - - config.ssh.insert_key = false - - config.vm.provider "libvirt" do |lv, override| - lv.memory = 1024 - override.vm.synced_folder './', '/vagrant', type: 'nfs', disabled: false - end - - config.vm.network "private_network", ip: "10.0.1.3" - -end diff --git a/molecule/vagrant-packager/box_files/app_xenial_metadata.json b/molecule/vagrant-packager/box_files/app_xenial_metadata.json deleted file mode 100644 index 3998094af5..0000000000 --- a/molecule/vagrant-packager/box_files/app_xenial_metadata.json +++ /dev/null @@ -1,193 +0,0 @@ -{ - "description": "Testing VM image for SecureDrop Application Server running Ubuntu Xenial.", - "name": "fpf/securedrop-app-xenial", - "versions": [ - { - "providers": [ - { - "checksum": "79cfa415df0bc83891dab59832564a89815f144ff1248995add90bc84e167ca5", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_0.12.0.box" - } - ], - "version": "0.12.0" - }, - { - "providers": [ - { - "checksum": "a59ffff93660460b0653d429031deb033819ccac30b8872d8efa9f7f5b57f6f3", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_0.12.1.box" - } - ], - "version": "0.12.1" - }, - { - "providers": [ - { - "checksum": "0ba57d1cf31e0ec49b107da46dce91b8a911653091f140ce019543349b6deb82", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_0.12.2.box" - } - ], - "version": "0.12.2" - }, - { - "providers": [ - { - "checksum": "30987d6f81b50822ef5234b1dc76a6640cf3e0f784b8f9a97aa21b22d5bb2950", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_0.13.0.box" - } - ], - "version": "0.13.0" - }, - { - "providers": [ - { - "checksum": "9e4e2cd973c0daa955d24597ab9ae4e0915b032fe7cd26fdd0f410d4e4300143", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_0.13.1.box" - } - ], - "version": "0.13.1" - }, - { - "providers": [ - { - "checksum": "79a8684d46c8e77c2793e45818101f985e8654020ff345b11f543b87110927d0", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_0.14.0.box" - } - ], - "version": "0.14.0" - }, - { - "providers": [ - { - "checksum": "7045b1a593c70e6256bb059db43723d121001337d8c060498b6e463a93102c65", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_1.0.0.box" - } - ], - "version": "1.0.0" - }, - { - "providers": [ - { - "checksum": "0a316e65c307ce67ee1056ae59a81378286d31e6527e65e246268b51c18cc148", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_1.1.0.box" - } - ], - "version": "1.1.0" - }, - { - "providers": [ - { - "checksum": "834debf3efea4f96b8527e3af1e6d79ae09c059365d1d0f1f26ebe3ee279439b", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_1.2.0.box" - } - ], - "version": "1.2.0" - }, - { - "providers": [ - { - "checksum": "652d4b10825e2fa7d1f9cabdace79d842cd6b6b8a22dcb5668f9d33b24613f71", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_1.2.1.box" - } - ], - "version": "1.2.1" - }, - { - "providers": [ - { - "checksum": "d38973ea121a96b40927cf6684812bac35ebcce17c5215e42874ad65a9b6ddd8", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_1.2.2.box" - } - ], - "version": "1.2.2" - }, - { - "providers": [ - { - "checksum": "32865fc67110bc3dd4d98bd61d1b54207bebae75fec5b0df9e3935b70d107467", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_1.3.0.box" - } - ], - "version": "1.3.0" - }, - { - "providers": [ - { - "checksum": "364916d4f48f2786da6d672e88862afb9badcf7f9c1bc601061a550ddcbddff7", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_1.4.0.box" - } - ], - "version": "1.4.0" - }, - { - "providers": [ - { - "checksum": "bcc366e11b54a53e6b6be193edb8a977b0624c56b430fce717998c2b28bb1daa", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_1.4.1.box" - } - ], - "version": "1.4.1" - }, - { - "providers": [ - { - "checksum": "3fc53176b6f0ef18fa720b05192371301711276530e32878cbd53971e4dbc39b", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_1.5.0.box" - } - ], - "version": "1.5.0" - }, - { - "providers": [ - { - "checksum": "b7208208febdaca0682afe60fac0709cf284c643dc9c5459a729e1df5584ec38", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_1.6.0.box" - } - ], - "version": "1.6.0" - }, - { - "providers": [ - { - "checksum": "63fa462c1990312c25c4b4ff3cfbbf8bb443c42bf050aa524ec9938689f79625", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/app-staging-xenial_1.7.1.box" - } - ], - "version": "1.7.1" - } - ] -} \ No newline at end of file diff --git a/molecule/vagrant-packager/box_files/mon_xenial_metadata.json b/molecule/vagrant-packager/box_files/mon_xenial_metadata.json deleted file mode 100644 index 08669f4c9c..0000000000 --- a/molecule/vagrant-packager/box_files/mon_xenial_metadata.json +++ /dev/null @@ -1,193 +0,0 @@ -{ - "description": "Testing VM image for SecureDrop Monitor Server running Ubuntu Xenial.", - "name": "fpf/securedrop-mon-xenial", - "versions": [ - { - "providers": [ - { - "checksum": "efa3c6aa38279c3c9caadea0480ac5d1e855ccbac7afe54aeb56d9075b0874e9", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_0.12.0.box" - } - ], - "version": "0.12.0" - }, - { - "providers": [ - { - "checksum": "e38864cc63bf26c4423c45986ab06f16c5638615b5850f00c69153b456cbb3a9", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_0.12.1.box" - } - ], - "version": "0.12.1" - }, - { - "providers": [ - { - "checksum": "648233f3f727b648a0f79e9d5a81473e2ca39b0d24e9184c51f8603160ce3232", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_0.12.2.box" - } - ], - "version": "0.12.2" - }, - { - "providers": [ - { - "checksum": "6b8a4f2acec2453c79bec78e5dfa1d5a4045f1a322d33b8571b6d41bfa74ae28", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_0.13.0.box" - } - ], - "version": "0.13.0" - }, - { - "providers": [ - { - "checksum": "cbe347281867def31e6f6383d40a072d1336bde1ead14fd886192bf548ef41f7", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_0.13.1.box" - } - ], - "version": "0.13.1" - }, - { - "providers": [ - { - "checksum": "752106cb530810d39ac860dcaf6d1829266560a3b0c578898698983f41d4154a", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_0.14.0.box" - } - ], - "version": "0.14.0" - }, - { - "providers": [ - { - "checksum": "5a2b003e4eb5f396f78efec5ac64257f1df32a0416564937d5bfc405f8f6f6b0", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_1.0.0.box" - } - ], - "version": "1.0.0" - }, - { - "providers": [ - { - "checksum": "54961b42d6213ddb9c196db2d51f5036bcbf625ccafd124153ad94f1e0d63ac5", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_1.1.0.box" - } - ], - "version": "1.1.0" - }, - { - "providers": [ - { - "checksum": "efe6bccc1d5aa84fad998cf42f038984ad60f4d9cd4a1fc12320327cdc7348bd", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_1.2.0.box" - } - ], - "version": "1.2.0" - }, - { - "providers": [ - { - "checksum": "78c6b5e777f5517315e44cbbfdeb527ef1f38369091432b8c1ce02653ce219dc", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_1.2.1.box" - } - ], - "version": "1.2.1" - }, - { - "providers": [ - { - "checksum": "156253981849acb027d4ec081c6ec42d0d3475e37ab936d389823d2d2fb8ef6f", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_1.2.2.box" - } - ], - "version": "1.2.2" - }, - { - "providers": [ - { - "checksum": "2cd099eb7c8ac45381ec106f8de3412f39dad15d3f4cc663b284e021a8605b83", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_1.3.0.box" - } - ], - "version": "1.3.0" - }, - { - "providers": [ - { - "checksum": "7b8f58292297f6ab8a004f8925451e1d93bb99a71611852d72eff9d6e221bad7", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_1.4.0.box" - } - ], - "version": "1.4.0" - }, - { - "providers": [ - { - "checksum": "551d8ea7323aeafe03010a63181857f8d546ed897e686efbb2c8f2a268201555", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_1.4.1.box" - } - ], - "version": "1.4.1" - }, - { - "providers": [ - { - "checksum": "2f2657811d77f90604c6f7d51a7f8d1e6ffd7adbe4e87cecd207aa597355d645", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_1.5.0.box" - } - ], - "version": "1.5.0" - }, - { - "providers": [ - { - "checksum": "e003896cc3b53fafa7f43a60b78cfac0a3a9587bc7ae0d5e581d3e0dbb479c03", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_1.6.0.box" - } - ], - "version": "1.6.0" - }, - { - "providers": [ - { - "checksum": "33068aab60f86e996625a0305f4620e5d31ad8379e87bfac2fb38c3b35434ffb", - "checksum_type": "sha256", - "name": "libvirt", - "url": "https://dev-bin.ops.securedrop.org/vagrant/mon-staging-xenial_1.7.1.box" - } - ], - "version": "1.7.1" - } - ] -} \ No newline at end of file diff --git a/molecule/vagrant-packager/box_files/vagrant_app-staging.yml b/molecule/vagrant-packager/box_files/vagrant_app-staging.yml deleted file mode 100644 index 9e63e9ceed..0000000000 --- a/molecule/vagrant-packager/box_files/vagrant_app-staging.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Molecule managed - ---- -instance: - interfaces: [] - name: app-staging - raw_config_args: - - 'vm.synced_folder ''./'', ''/vagrant'', disabled: true' - - 'vm.network ''private_network'', ip: ''10.0.1.2''' - - ssh.insert_key = false -platform: - box: bento/ubuntu-14.04 - box_url: null - box_version: null -provider: - name: libvirt - options: - cpus: 2 - memory: 1024 - raw_config_args: null diff --git a/molecule/vagrant-packager/box_files/vagrant_mon-staging.yml b/molecule/vagrant-packager/box_files/vagrant_mon-staging.yml deleted file mode 100644 index fd6e8e4c1c..0000000000 --- a/molecule/vagrant-packager/box_files/vagrant_mon-staging.yml +++ /dev/null @@ -1,20 +0,0 @@ -# Molecule managed - ---- -instance: - interfaces: [] - name: mon-staging - raw_config_args: - - 'vm.synced_folder ''./'', ''/vagrant'', disabled: true' - - 'vm.network ''private_network'', ip: ''10.0.1.3''' - - ssh.insert_key = false -platform: - box: bento/ubuntu-14.04 - box_url: null - box_version: null -provider: - name: libvirt - options: - cpus: 2 - memory: 1024 - raw_config_args: null diff --git a/molecule/vagrant-packager/build/.gitignore b/molecule/vagrant-packager/build/.gitignore deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/molecule/vagrant-packager/create.yml b/molecule/vagrant-packager/create.yml deleted file mode 100644 index c2c13ee16f..0000000000 --- a/molecule/vagrant-packager/create.yml +++ /dev/null @@ -1,58 +0,0 @@ ---- -- import_playbook: ../shared/sd_clone.yml - -- name: Create - hosts: localhost - connection: local - vars: - molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}" - molecule_instance_config: "{{ lookup('env', 'MOLECULE_INSTANCE_CONFIG') }}" - molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}" - tasks: - - - name: Create molecule instance(s) - vagrant: - instance_name: "{{ item.name }}" - instance_interfaces: "{{ item.interfaces | default(omit) }}" - instance_raw_config_args: "{{ item.instance_raw_config_args | default(omit) }}" - - platform_box: "{{ item.box }}" - platform_box_version: "{{ item.box_version | default(omit) }}" - platform_box_url: "{{ item.box_url | default(omit) }}" - - provider_name: "{{ molecule_yml.driver.provider.name }}" - provider_memory: "{{ item.memory | default(omit) }}" - provider_cpus: "{{ item.cpus | default(omit) }}" - provider_raw_config_args: "{{ item.raw_config_args | default(omit) }}" - force_stop: yes - - state: up - register: server - with_items: "{{ molecule_yml.platforms }}" - - # Mandatory configuration for Molecule to function. - - - name: Populate instance config dict - set_fact: - instance_conf_dict: { - 'instance': "{{ item.Host }}", - 'address': "{{ item.HostName }}", - 'user': "{{ item.User }}", - 'port': "{{ item.Port }}", - 'identity_file': "{{ item.IdentityFile }}", } - with_items: "{{ server.results }}" - register: instance_config_dict - when: server.changed | bool - - - name: Convert instance config dict to a list - set_fact: - instance_conf: "{{ instance_config_dict.results | map(attribute='ansible_facts.instance_conf_dict') | list }}" - when: server.changed | bool - - - name: Dump instance config - copy: - # NOTE(retr0h): Workaround for Ansible 2.2. - # https://github.com/ansible/ansible/issues/20885 - content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}" - dest: "{{ molecule_instance_config }}" - when: server.changed | bool diff --git a/molecule/vagrant-packager/destroy.yml b/molecule/vagrant-packager/destroy.yml deleted file mode 100644 index b98563c051..0000000000 --- a/molecule/vagrant-packager/destroy.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- - -- name: Destroy - hosts: localhost - connection: local - vars: - molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}" - molecule_instance_config: "{{ lookup('env',' MOLECULE_INSTANCE_CONFIG') }}" - molecule_scenario_dir: "{{ lookup('env','MOLECULE_SCENARIO_DIRECTORY') }}" - molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}" - molecule_ephemeral_directory: "{{ lookup('env', 'MOLECULE_EPHEMERAL_DIRECTORY') }}" - tasks: - - name: Destroy molecule instance(s) - vagrant: - instance_name: "{{ item.name }}" - platform_box: "{{ item.box }}" - provider_name: "{{ molecule_yml.driver.provider.name }}" - force_stop: "{{ item.force_stop | default(True) }}" - - state: destroy - register: server - with_items: "{{ molecule_yml.platforms }}" - - - name: Destroy local molecule ephemeral dir - file: - state: absent - path: "{{ molecule_scenario_dir }}/.molecule" - - # Mandatory configuration for Molecule to function. - - - name: Populate instance config - set_fact: - instance_conf: {} - - - name: Dump instance config - copy: - # NOTE(retr0h): Workaround for Ansible 2.2. - # https://github.com/ansible/ansible/issues/20885 - content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}" - dest: "{{ molecule_instance_config }}" - when: server.changed | bool diff --git a/molecule/vagrant-packager/molecule.yml b/molecule/vagrant-packager/molecule.yml deleted file mode 100644 index 9790ec880f..0000000000 --- a/molecule/vagrant-packager/molecule.yml +++ /dev/null @@ -1,68 +0,0 @@ ---- -driver: - name: vagrant - provider: - name: libvirt -lint: | - yamllint - -platforms: - - name: app-staging - box: bento/ubuntu-16.04 - instance_raw_config_args: - - "vm.synced_folder './', '/vagrant', disabled: true" - - "vm.network 'private_network', ip: '10.0.1.2'" - - "ssh.insert_key = false" - memory: 1024 - private_ip: 10.0.1.2 - groups: - - securedrop_application_server - - securedrop - - staging - - - name: mon-staging - box: bento/ubuntu-16.04 - instance_raw_config_args: - - "vm.synced_folder './', '/vagrant', disabled: true" - - "vm.network 'private_network', ip: '10.0.1.3'" - - "ssh.insert_key = false" - memory: 1024 - private_ip: 10.0.1.3 - groups: - - securedrop_monitor_server - - securedrop - - staging - -provisioner: - name: ansible - lint: | - ansible-lint - inventory: - links: - group_vars: ../../install_files/ansible-base/group_vars - host_vars: ../../install_files/ansible-base/host_vars - env: - ANSIBLE_ROLES_PATH: ".molecule/roles" - ANSIBLE_GATHER_TIMEOUT: "120" - options: - skip-tags: "local_build" - e: "@ansible-override-vars.yml" - config_options: - defaults: - interpreter_python: auto - ssh_connection: - pipelining: True - playbooks: - side_effect: side_effect.yml - -scenario: - name: vagrant-packager - test_sequence: - - destroy - - create - - converge - - side_effect -verifier: - name: testinfra - lint: | - flake8 diff --git a/molecule/vagrant-packager/package.py b/molecule/vagrant-packager/package.py deleted file mode 100755 index eb7041a4e5..0000000000 --- a/molecule/vagrant-packager/package.py +++ /dev/null @@ -1,262 +0,0 @@ -#!/usr/bin/env python3 -# -# -# -# -import hashlib -import json -import os -from os.path import join -import re -import shutil -import subprocess -import tarfile -import xml.etree.ElementTree as ET - - -SCENARIO_NAME = os.path.basename(os.path.dirname(os.path.abspath(__file__))) -SCENARIO_PATH = os.path.dirname(os.path.realpath(__file__)) -BOX_BUILD_DIR = join(SCENARIO_PATH, "build") -BOX_METADATA_DIR = join(SCENARIO_PATH, "box_files") -EPHEMERAL_DIRS = {} -TARGET_VERSION_FILE = os.path.join(SCENARIO_PATH, os.path.pardir, "shared", "stable.ver") - - -class LibVirtPackager(object): - - def __init__(self, vm): - # type: (str) -> None - self.cli_prefix = "virsh --connect qemu:///system {}" - self.vm_name = vm - - def _get_virsh_xml(self, cmd): - # type: (str) -> ET.Element - virsh_cmd_str = self.cli_prefix.format(cmd) - cmd_output = subprocess.check_output(virsh_cmd_str.split()) - return ET.fromstring(cmd_output) - - def vm_xml(self): - # type: () -> ET.Element - """ Get XML definition for virtual machine domain - """ - return self._get_virsh_xml("dumpxml "+self.vm_name) - - def default_image_location(self): - # type: () -> str - """ - Get full system path to the default system pool dir - """ - pool = self._get_virsh_xml("pool-dumpxml default") - - return pool.findall('./target/path')[0].text - - def image_rebase(self, img_location): - # type: (str) -> None - """ If an image has a backing store, merge the backing store into the - target image file - """ - if self.default_image_location() in img_location: - raise UserWarning("To prevent catastrophy, will not" - " run on image in the default pool. Copy it" - " first to someplace else") - - img_info = subprocess.check_output(["qemu-img", "info", img_location]) - rebase_cmd = """qemu-img rebase -b "" {}""".format(img_location) - - if "backing file:" in img_info.decode('utf-8'): - print("Running rebase now..") - subprocess.check_call(rebase_cmd, shell=True) - else: - print("No rebase necessary") - - def image_store_path(self): - # type: () -> str - """ Get location of VM's first storage disk file """ - vm_xml = self.vm_xml() - return vm_xml.findall('./devices/disk/source')[0].attrib['file'] - - def image_sparsify(self, src, dest, tmp_dir, inplace): - # type: (str, str, str, bool) -> None - """ Remove blank-space from the image. Note that setting inplace to - false will get you better bang for your buck but can make system - unstable. Pushed IO to the max on my machine and caused it to crash - """ - img_info = subprocess.check_output(["qemu-img", "info", src]) - - if "backing file:" in img_info.decode('utf-8'): - raise UserWarning("Cannot sparsify image w/ backing " - "store. Please rebase first.") - - if inplace: - subprocess.check_call(["virt-sparsify", - "--in-place", - src]) - shutil.move(src, dest) - else: - subprocess.check_call(["virt-sparsify", "--tmp", - tmp_dir, - src, - dest]) - - def sysprep(self, img_location): - # type: (str) -> None - """ Run the virt-sysprep tool over the image to prep the log for - re-dist. Removes things like logs and user history files - """ - sysprep_cmd = ("virt-sysprep --no-logfile --operations " - "defaults,-ssh-userdir,-ssh-hostkeys,-logfiles -a " + - img_location) - subprocess.check_call(sysprep_cmd.split()) - - def vagrant_metadata(self, img_location): - # type: (str) -> dict - """ Produce dictionary of necessary vagrant key/values """ - json = {} - - info_output = subprocess.check_output(["qemu-img", "info", - img_location]).decode('utf-8') - json['virtual_size'] = int((re.search(r"virtual size: (?P\d+)G", - info_output)).group("size")) - - json['format'] = (re.search(r"file format: (?P\w+)", - info_output)).group("format") - json['provider'] = 'libvirt' - - return json - - -def main(): - with open(TARGET_VERSION_FILE, 'r') as f: - TARGET_VERSION = f.read().strip() - - # Default to Xenial as base OS. - TARGET_DISTRIBUTION = os.environ.get("SECUREDROP_TARGET_DISTRIBUTION", "xenial") - - for srv in ["app-staging", "mon-staging"]: - - for temp_dir in ["build", "tmp"]: - try: - ephemeral_path = join(SCENARIO_PATH, ".molecule", - temp_dir) - EPHEMERAL_DIRS[temp_dir] = ephemeral_path - - os.makedirs(os.path.join(SCENARIO_PATH, ".molecule", temp_dir)) - except OSError: - pass - - vm = LibVirtPackager("{}_{}".format(SCENARIO_NAME, srv)) - - tmp_img_file = join(EPHEMERAL_DIRS["tmp"], "wip.img") - packaged_img_file = join(EPHEMERAL_DIRS["build"], "box.img") - - print("Copying VM image store locally") - subprocess.check_output(["sudo", "cp", - vm.image_store_path(), # source - tmp_img_file # dest - ]) - - print("Changing file ownership") - subprocess.check_output(["sudo", "chown", os.environ['USER'], - tmp_img_file]) - - # Run a sysprep on it - print("Run an image sysprep") - vm.sysprep(tmp_img_file) - - print("Rebase Image") - vm.image_rebase(tmp_img_file) - - # Sparsify the image file - print("Run sparsi-fication on the image") - vm.image_sparsify(src=tmp_img_file, - dest=packaged_img_file, - tmp_dir=EPHEMERAL_DIRS['tmp'], - inplace=True) - - # Write out metadata file - with open(join(EPHEMERAL_DIRS['build'], 'metadata.json'), - 'w') as mdata: - json.dump( - vm.vagrant_metadata(packaged_img_file), - mdata) - - # Copy in appropriate vagrant file to build dir - shutil.copyfile(join(BOX_METADATA_DIR, "Vagrantfile."+srv), - join(EPHEMERAL_DIRS['build'], 'Vagrantfile')) - - print("Creating tar file") - box_file = join( - BOX_BUILD_DIR, "{}-{}_{}.box".format(srv, TARGET_DISTRIBUTION, TARGET_VERSION) - ) - with tarfile.open(box_file, "w|gz") as tar: - for boxfile in ["box.img", "Vagrantfile", "metadata.json"]: - tar.add(join(EPHEMERAL_DIRS["build"], boxfile), - arcname=boxfile) - - print("Box created at {}".format(box_file)) - - print("Updating box metadata") - update_box_metadata(srv, box_file, TARGET_DISTRIBUTION, TARGET_VERSION) - - print("Clean-up tmp space") - shutil.rmtree(EPHEMERAL_DIRS['tmp']) - - -def sha256_checksum(filepath): - """ - Returns a SHA256 checksum for a given filepath. - """ - checksum = hashlib.sha256() - with open(filepath, 'rb') as f: - # Read by chunks, to avoid slurping the entire file into memory. - # Box files range from 500MB to 1.5GB. - for block in iter(lambda: f.read(checksum.block_size), b''): - checksum.update(block) - return checksum.hexdigest() - - -def update_box_metadata(server_name, box_file, platform, version): - """ - Updates the JSON file of Vagrant box metadata, including remote URL, - version number, and SHA256 checksum. - """ - # Strip off "staging" suffix from box names - server_name_short = re.sub(r'\-staging$', '', server_name) - json_file_basename = "{}_{}_metadata.json".format(server_name_short, platform) - json_file = os.path.join(BOX_METADATA_DIR, json_file_basename) - - # Read in current JSON metadata, so we can append the new info to it. - with open(json_file, "r") as f: - metadata_config = json.loads(f.read()) - - base_url = "https://dev-bin.ops.securedrop.org/vagrant" - box_name = os.path.basename(box_file) - box_url = "{}/{}".format(base_url, box_name) - box_checksum = sha256_checksum(box_file) - box_config = dict( - name="libvirt", - url=box_url, - checksum_type="sha256", - checksum=box_checksum, - ) - # Creating list of dicts to adhere to JSON format of Vagrant box metadata - providers_list = [] - providers_list.append(box_config) - version_config = dict( - version=version, - providers=providers_list, - ) - box_versions = metadata_config['versions'] - box_versions.append(version_config) - metadata_config['versions'] = box_versions - - # Write out final, modified data. Does not validate for uniqueness, - # so repeated runs on the same version will duplicate version info, - # which'll likely break the box fetching. Target file is version-controlled, - # though, so easy enough to correct in the event of a mistake. - with open(json_file, "w") as f: - f.write(json.dumps(metadata_config, indent=2, sort_keys=True)) - - -if __name__ == "__main__": - main() diff --git a/molecule/vagrant-packager/playbook.yml b/molecule/vagrant-packager/playbook.yml deleted file mode 100644 index a1afdfc8dc..0000000000 --- a/molecule/vagrant-packager/playbook.yml +++ /dev/null @@ -1,70 +0,0 @@ ---- -- name: Prepare servers for installation - hosts: securedrop - gather_facts: no - max_fail_percentage: 0 - any_errors_fatal: yes - become: yes - roles: - - { role: prepare-servers } - -- name: Add FPF apt repository and install base packages. - hosts: securedrop - max_fail_percentage: 0 - any_errors_fatal: yes - roles: - - { role: install-fpf-repo, tags: fpf } - - { role: grsecurity, when: grsecurity, tags: [grsec, grsecurity] } - - { role: common, tags: common } - - { role: tor-hidden-services, tags: tor } - become: yes - -- name: Configure OSSEC. - hosts: securedrop - max_fail_percentage: 0 - any_errors_fatal: yes - roles: - - role: ossec - tags: ossec - become: yes - -- name: Configure mailing utilities. - hosts: securedrop_monitor_server - max_fail_percentage: 0 - any_errors_fatal: yes - roles: - - role: postfix - tags: postfix - become: yes - -- name: Configure SecureDrop Application Server. - hosts: securedrop_application_server - max_fail_percentage: 0 - any_errors_fatal: yes - roles: - - { role: app, tags: app } - become: yes - - # This section will put the SSH and iptables rules in place - # It will then add any staging exemptions required - # at the end of each host section is when the handlers are run. - # So iptables will not be reloaded until the exemptions are applied - # for production the last task is apply iptables. This will break their - # connection. After that point the admin will to proxy traffic over tor. -- name: Lock down firewall configuration for Application and Monitor Servers. - hosts: securedrop - max_fail_percentage: 0 - any_errors_fatal: yes - roles: - - { role: restrict-direct-access, tags: [ common, restrict-direct-access ] } - become: yes - -- name: Reboot Application and Monitor Servers. - hosts: securedrop - max_fail_percentage: 0 - any_errors_fatal: yes - tasks: - # Reuse existing "reboot-and-wait" logic from site config - - import_tasks: ../../install_files/ansible-base/roles/reboot-if-first-install/tasks/reboot-no-check.yml - tags: reboot-no-check - become: yes diff --git a/molecule/vagrant-packager/prepare.yml b/molecule/vagrant-packager/prepare.yml deleted file mode 100644 index 112da19982..0000000000 --- a/molecule/vagrant-packager/prepare.yml +++ /dev/null @@ -1,9 +0,0 @@ ---- -- name: Prepare - hosts: all - gather_facts: False - tasks: - - name: Install python for Ansible - raw: test -e /usr/bin/python || (apt -y update && apt install -y python-minimal) - become: True - changed_when: False diff --git a/molecule/vagrant-packager/push.yml b/molecule/vagrant-packager/push.yml deleted file mode 100755 index 971e8231c3..0000000000 --- a/molecule/vagrant-packager/push.yml +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env ansible-playbook ---- - -- name: Push boxes to s3 bucket - hosts: localhost - become: no - tasks: - - name: Assume role - sts_assume_role: - role_arn: "arn:aws:iam::601214233086:role/sdroppackager" - role_session_name: "vagrantupload-{{ ansible_hostname | to_uuid }}" - register: assumed_role - - - name: Find boxes to push up - find: - paths: "{{ playbook_dir }}/build/" - patterns: '*.box' - register: local_boxes - - - name: Push boxes up - aws_s3: - aws_access_key: "{{ assumed_role.sts_creds.access_key }}" - aws_secret_key: "{{ assumed_role.sts_creds.secret_key }}" - security_token: "{{ assumed_role.sts_creds.session_token }}" - bucket: "{{ bucket_name }}" - object: "/{{ bucket_path }}/{{ item.path | basename }}" - permission: "public-read" - src: "{{ item.path }}" - mode: put - with_items: "{{ local_boxes.files }}" - - - name: Generate index - script: ./scripts/list_bucket.py - environment: - AWS_ACCESS_KEY_ID: "{{ assumed_role.sts_creds.access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ assumed_role.sts_creds.secret_key }}" - AWS_SESSION_TOKEN: "{{ assumed_role.sts_creds.session_token }}" - BUCKET: "{{ bucket_name }}" - BUCKET_PATH: "{{ bucket_path }}" - - vars: - bucket_name: dev-bin.ops.securedrop.org - bucket_path: vagrant diff --git a/molecule/vagrant-packager/scripts/list_bucket.py b/molecule/vagrant-packager/scripts/list_bucket.py deleted file mode 100755 index 9258730c2e..0000000000 --- a/molecule/vagrant-packager/scripts/list_bucket.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# -# -# -# -# Generate index.html of vagrant box files in our s3 bucket -# and upload said file. - -import boto3 -import os - - -class S3_Bucket_IndexCreator(object): - """ Class to initialize s3 bucket connection, grab contents, publish index """ - - def __init__(self, bucket, path): - self.s3 = boto3.resource('s3') - self.vagrant_bucket = self.s3.Bucket(name=bucket) - self.bucket = bucket - self.path = path - - def bucket_get_list(self): - """ Get bucket file listings and return python list """ - - return [obj.key.split('/')[-1] for obj in - self.vagrant_bucket.objects.filter(Prefix=self.path) if - "index.html" not in obj.key] - - def generate_html_index(self): - """Build a simple HTML index string from bucket listings""" - - str_files = ["""{file}""".format(file=f) for f in - self.bucket_get_list()] - - index_string = """ - Index of /{path}/ -

Index of /{path}/


{files}

""".format( - path=self.path, - files="
".join(str_files) - ) - - return index_string - - def upload_string_as_file(self, - contents, - filename="index.html", - content_type="text/html"): - - """ Take contents of a file as input and dump that to a file """ - object = self.s3.Object(self.bucket, '{}/{}'.format(self.path, filename)) - object.put(Body=contents, - ContentType=content_type) - - -if __name__ == "__main__": - BUCKET = os.environ.get('BUCKET', 'dev-bin.ops.securedrop.org') - BUCKET_PATH = os.environ.get('BUCKET_PATH', 'vagrant/') - - bucket_index_creation = S3_Bucket_IndexCreator(BUCKET, BUCKET_PATH) - index = bucket_index_creation.generate_html_index() - bucket_index_creation.upload_string_as_file(index) diff --git a/molecule/vagrant-packager/side_effect.yml b/molecule/vagrant-packager/side_effect.yml deleted file mode 100644 index 623aaf6646..0000000000 --- a/molecule/vagrant-packager/side_effect.yml +++ /dev/null @@ -1,50 +0,0 @@ ---- -- name: Wait a bit to make sure hosts are fully booted - hosts: localhost - gather_facts: no - tasks: - - name: Pause for a while - pause: - seconds: 120 - -- name: Strip away configured onion services files - hosts: securedrop - become: yes - tasks: - - name: Delete Tor related keys - file: - state: absent - path: "/var/lib/tor/services/{{ item }}" - with_items: - - ssh - - journalist - - source - notify: disable tor - handlers: - - name: disable tor - service: - enabled: yes - name: tor - state: stopped - -- name: Halt boxes - hosts: localhost - connection: local - tasks: - - name: Halt vagrant instances - vagrant: - instance_name: "{{ item.name }}" - instance_raw_config_args: "{{ item.instance_raw_config_args | default(omit) }}" - platform_box: "{{ item.box }}" - platform_box_version: "{{ item.box_version | default(omit) }}" - platform_box_url: "{{ item.box_url | default(omit) }}" - provider_name: "{{ molecule_yml.driver.provider.name }}" - provider_memory: "{{ item.memory | default(omit) }}" - provider_raw_config_args: "{{ item.raw_config_args | default(omit) }}" - force_stop: yes - state: halt - register: server - with_items: "{{ molecule_yml.platforms }}" - vars: - molecule_file: "{{ lookup('env', 'MOLECULE_FILE') }}" - molecule_yml: "{{ lookup('file', molecule_file) | molecule_from_yaml }}"