From 5a0c16baffaea033b9fd5c1dd2cefbd79a3e4730 Mon Sep 17 00:00:00 2001 From: mickael e Date: Fri, 11 Dec 2020 11:56:00 -0500 Subject: [PATCH 1/8] Updates apt sources for Focal Universe channel is required for `ntp` and `aptitude` --- .../ansible-base/roles/common/tasks/apt_sources.yml | 8 ++++++++ .../ansible-base/roles/common/tasks/main.yml | 12 ++++++++++++ .../roles/common/templates/sources.list.j2 | 12 ++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 install_files/ansible-base/roles/common/tasks/apt_sources.yml create mode 100644 install_files/ansible-base/roles/common/templates/sources.list.j2 diff --git a/install_files/ansible-base/roles/common/tasks/apt_sources.yml b/install_files/ansible-base/roles/common/tasks/apt_sources.yml new file mode 100644 index 0000000000..9ee323ccc9 --- /dev/null +++ b/install_files/ansible-base/roles/common/tasks/apt_sources.yml @@ -0,0 +1,8 @@ +- name: Configure apt sources. + template: + src: sources.list.j2 + dest: /etc/apt/sources.list + mode: "0644" + owner: root + tags: + - apt diff --git a/install_files/ansible-base/roles/common/tasks/main.yml b/install_files/ansible-base/roles/common/tasks/main.yml index a2904ba03e..915c68b2e4 100644 --- a/install_files/ansible-base/roles/common/tasks/main.yml +++ b/install_files/ansible-base/roles/common/tasks/main.yml @@ -1,6 +1,10 @@ --- - include_vars: "{{ ansible_distribution }}_{{ ansible_distribution_release }}.yml" +- include: apt_sources.yml + when: + - ansible_distribution_release == "focal" + - include: install_packages.yml - include: post_ubuntu_install_checks.yml @@ -12,6 +16,14 @@ - include: harden_dns.yml - include: cron_apt.yml + when: + - ansible_distribution_release == "xenial" + tags: + - reboot + +- include: unattended_upgrades.yml + when: + - ansible_distribution_release == "focal" tags: - reboot diff --git a/install_files/ansible-base/roles/common/templates/sources.list.j2 b/install_files/ansible-base/roles/common/templates/sources.list.j2 new file mode 100644 index 0000000000..a0a09fd05d --- /dev/null +++ b/install_files/ansible-base/roles/common/templates/sources.list.j2 @@ -0,0 +1,12 @@ +## newer versions of the distribution. +deb http://archive.ubuntu.com/ubuntu/ {{ ansible_distribution_release }} main + +## newer versions of the distribution. +deb http://archive.ubuntu.com/ubuntu/ {{ ansible_distribution_release }} universe + +## Major bug fix updates produced after the final release of the +## distribution. +deb http://archive.ubuntu.com/ubuntu/ {{ ansible_distribution_release }}-updates main + +### Security fixes for distribution packages +deb http://security.ubuntu.com/ubuntu {{ ansible_distribution_release }}-security main From 331d01d59b7f76864645f0c0426ceb3f7ad578d7 Mon Sep 17 00:00:00 2001 From: mickael e Date: Mon, 4 Jan 2021 20:29:46 -0500 Subject: [PATCH 2/8] Rename test file --- .../common/{test_cron_apt.py => test_automatic_updates.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename molecule/testinfra/common/{test_cron_apt.py => test_automatic_updates.py} (100%) diff --git a/molecule/testinfra/common/test_cron_apt.py b/molecule/testinfra/common/test_automatic_updates.py similarity index 100% rename from molecule/testinfra/common/test_cron_apt.py rename to molecule/testinfra/common/test_automatic_updates.py From 475cf14743a99d34b76187cad46e6069bef132c2 Mon Sep 17 00:00:00 2001 From: mickael e Date: Mon, 4 Jan 2021 20:30:21 -0500 Subject: [PATCH 3/8] Use unattended-upgrades instead of cron-apt under Ubuntu Focal --- .../roles/common/defaults/main.yml | 9 - .../roles/common/files/20auto-upgrades | 3 + .../common/tasks/unattended_upgrades.yml | 31 +++ .../common/templates/50unattended-upgrades.j2 | 56 +++++ .../roles/common/templates/sources.list.j2 | 1 + .../roles/common/vars/Ubuntu_focal.yml | 9 + .../roles/common/vars/Ubuntu_xenial.yml | 9 + .../common/test_automatic_updates.py | 192 +++++++++++++----- 8 files changed, 253 insertions(+), 57 deletions(-) create mode 100644 install_files/ansible-base/roles/common/files/20auto-upgrades create mode 100644 install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml create mode 100644 install_files/ansible-base/roles/common/templates/50unattended-upgrades.j2 diff --git a/install_files/ansible-base/roles/common/defaults/main.yml b/install_files/ansible-base/roles/common/defaults/main.yml index 1c2dda22b8..a9de7da68a 100644 --- a/install_files/ansible-base/roles/common/defaults/main.yml +++ b/install_files/ansible-base/roles/common/defaults/main.yml @@ -3,15 +3,6 @@ # and aid in clearing memory. Only the hour is configurable. daily_reboot_time: 4 # An integer between 0 and 23 -securedrop_common_packages: - - apt-transport-https - - aptitude - - cron-apt - - ntp - - ntpdate - - resolvconf - - tmux - disabled_kernel_modules: - btusb - bluetooth diff --git a/install_files/ansible-base/roles/common/files/20auto-upgrades b/install_files/ansible-base/roles/common/files/20auto-upgrades new file mode 100644 index 0000000000..ee4748cd54 --- /dev/null +++ b/install_files/ansible-base/roles/common/files/20auto-upgrades @@ -0,0 +1,3 @@ +APT::Periodic::Update-Package-Lists "1"; +APT::Periodic::Unattended-Upgrade "1"; +APT::Periodic::AutocleanInterval "1"; diff --git a/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml b/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml new file mode 100644 index 0000000000..63b2d6a40b --- /dev/null +++ b/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml @@ -0,0 +1,31 @@ +--- +- name: Install unattended-upgrades package. + apt: + name: unattended-upgrades + state: latest + update_cache: yes + tags: + - apt + - unattended-upgrades + +- name: Configure unattended-upgrades to update the packages from sources.list. + copy: + src: 20auto-upgrades + dest: /etc/apt/apt.conf.d/20auto-upgrades + mode: 0644 + owner: root + group: root + tags: + - apt + - unattended-upgrades + +- name: Configure unattended-upgrades to update the packages from sources.list. + template: + src: 50unattended-upgrades.j2 + dest: /etc/apt/apt.conf.d/50unattended-upgrades + mode: 0644 + owner: root + group: root + tags: + - apt + - unattended-upgrades diff --git a/install_files/ansible-base/roles/common/templates/50unattended-upgrades.j2 b/install_files/ansible-base/roles/common/templates/50unattended-upgrades.j2 new file mode 100644 index 0000000000..c49ee89e14 --- /dev/null +++ b/install_files/ansible-base/roles/common/templates/50unattended-upgrades.j2 @@ -0,0 +1,56 @@ +// Automatically upgrade packages from these (origin:archive) pairs +Unattended-Upgrade::Allowed-Origins { + "${distro_id}:${distro_codename}"; + "${distro_id}:${distro_codename}-security"; + "${distro_id}:${distro_codename}-updates"; + "SecureDrop:${distro_codename}"; +}; + +// List of packages to not update (regexp are supported) +Unattended-Upgrade::Package-Blacklist { +}; + +// This option allows you to control if on a unclean dpkg exit +// unattended-upgrades will automatically run +// dpkg --force-confold --configure -a +// The default is true, to ensure updates keep getting installed +// This mirrors the previous cron=apt config +Unattended-Upgrade::AutoFixInterruptedDpkg "true"; + +// Split the upgrade into the smallest possible chunks so that +// they can be interrupted with SIGUSR1. This makes the upgrade +// a bit slower but it has the benefit that shutdown while a upgrade +// is running is possible (with a small delay) +//Unattended-Upgrade::MinimalSteps "true"; + +// Install all unattended-upgrades when the machine is shuting down +// instead of doing it in the background while the machine is running +// This will (obviously) make shutdown slower +//Unattended-Upgrade::InstallOnShutdown "true"; + +// Send email to this address for problems or packages upgrades +// If empty or unset then no email is sent, make sure that you +// have a working mail setup on your system. A package that provides +// 'mailx' must be installed. E.g. "user@example.com" +//Unattended-Upgrade::Mail "root"; + +// Set this value to "true" to get emails only on errors. Default +// is to always send a mail if Unattended-Upgrade::Mail is set +//Unattended-Upgrade::MailOnlyOnError "true"; + +// Do automatic removal of new unused dependencies after the upgrade +// (equivalent to apt-get autoremove) +//Unattended-Upgrade::Remove-Unused-Dependencies "false"; + +// Automatically reboot *WITHOUT CONFIRMATION* +// if the file /var/run/reboot-required is found after the upgrade +Unattended-Upgrade::Automatic-Reboot "true"; + +// If automatic reboot is enabled and needed, reboot at the specific +// time instead of immediately +// Default: "now" +Unattended-Upgrade::Automatic-Reboot-Time "{{ daily_reboot_time }}:00"; + +// Use apt bandwidth limit feature, this example limits the download +// speed to 70kb/sec +//Acquire::http::Dl-Limit "70"; diff --git a/install_files/ansible-base/roles/common/templates/sources.list.j2 b/install_files/ansible-base/roles/common/templates/sources.list.j2 index a0a09fd05d..b622a6cc83 100644 --- a/install_files/ansible-base/roles/common/templates/sources.list.j2 +++ b/install_files/ansible-base/roles/common/templates/sources.list.j2 @@ -10,3 +10,4 @@ deb http://archive.ubuntu.com/ubuntu/ {{ ansible_distribution_release }}-updates ### Security fixes for distribution packages deb http://security.ubuntu.com/ubuntu {{ ansible_distribution_release }}-security main +deb http://security.ubuntu.com/ubuntu {{ ansible_distribution_release }}-security universe diff --git a/install_files/ansible-base/roles/common/vars/Ubuntu_focal.yml b/install_files/ansible-base/roles/common/vars/Ubuntu_focal.yml index 77c1ae66d1..2b06f4d6be 100644 --- a/install_files/ansible-base/roles/common/vars/Ubuntu_focal.yml +++ b/install_files/ansible-base/roles/common/vars/Ubuntu_focal.yml @@ -5,3 +5,12 @@ securedrop_kernel_packages_to_remove: - 'linux-image-.*generic' resolvconf_target_filepath: /etc/resolv.conf + +securedrop_common_packages: + - apt-transport-https + - aptitude + - unattended-upgrades + - ntp + - ntpdate + - resolvconf + - tmux diff --git a/install_files/ansible-base/roles/common/vars/Ubuntu_xenial.yml b/install_files/ansible-base/roles/common/vars/Ubuntu_xenial.yml index 5d99e06c75..55d9453bed 100644 --- a/install_files/ansible-base/roles/common/vars/Ubuntu_xenial.yml +++ b/install_files/ansible-base/roles/common/vars/Ubuntu_xenial.yml @@ -9,3 +9,12 @@ securedrop_kernel_packages_to_remove: - 'linux-headers-.*' resolvconf_target_filepath: /etc/resolvconf/resolv.conf.d/base + +securedrop_common_packages: + - apt-transport-https + - aptitude + - cron-apt + - ntp + - ntpdate + - resolvconf + - tmux diff --git a/molecule/testinfra/common/test_automatic_updates.py b/molecule/testinfra/common/test_automatic_updates.py index 1078968ed4..29db45a21a 100644 --- a/molecule/testinfra/common/test_automatic_updates.py +++ b/molecule/testinfra/common/test_automatic_updates.py @@ -7,34 +7,38 @@ testinfra_hosts = [test_vars.app_hostname, test_vars.monitor_hostname] -@pytest.mark.parametrize('dependency', [ - 'cron-apt', - 'ntp' -]) -def test_cron_apt_dependencies(host, dependency): +def test_automatic_updates_dependencies(host): """ Ensure critical packages are installed. If any of these are missing, the system will fail to receive automatic updates. - The current apt config uses cron-apt, rather than unattended-upgrades, - but this may change in the future. Previously the apt.freedom.press repo - was not reporting any "Origin" field, making use of unattended-upgrades - problematic. With better procedures in place regarding apt repo - maintenance, we can ensure the field is populated going forward. + In Xenial, the apt config uses cron-apt, rather than unattended-upgrades. + Previously the apt.freedom.press repo was not reporting any "Origin" field, + making use of unattended-upgrades problematic. + In Focal, the apt config uses unattended-upgrades. """ - assert host.package(dependency).is_installed + apt_dependencies = { + 'xenial': ['cron-apt', 'ntp'], + 'focal': ['unattended-upgrades', 'ntp'] + } + + for package in apt_dependencies[host.system_info.codename]: + assert host.package(package).is_installed def test_cron_apt_config(host): """ - Ensure custom cron-apt config file is present. + Ensure custom cron-apt config file is present in Xenial, and absent in Focal """ f = host.file('/etc/cron-apt/config') - assert f.is_file - assert f.user == "root" - assert f.mode == 0o644 - assert f.contains('^SYSLOGON="always"$') - assert f.contains('^EXITON=error$') + if host.system_info.codename == "xenial": + assert f.is_file + assert f.user == "root" + assert f.mode == 0o644 + assert f.contains('^SYSLOGON="always"$') + assert f.contains('^EXITON=error$') + else: + assert not f.exists @pytest.mark.parametrize('repo', [ @@ -54,31 +58,63 @@ def test_cron_apt_repo_list(host, repo): securedrop_target_distribution=host.system_info.codename ) f = host.file('/etc/apt/security.list') - assert f.is_file - assert f.user == "root" - assert f.mode == 0o644 - repo_regex = '^{}$'.format(re.escape(repo_config)) - assert f.contains(repo_regex) + if host.system_info.codename == "xenial": + assert f.is_file + assert f.user == "root" + assert f.mode == 0o644 + repo_regex = '^{}$'.format(re.escape(repo_config)) + assert f.contains(repo_regex) + else: + assert not f.exists + + +@pytest.mark.parametrize('repo', [ + 'deb http://security.ubuntu.com/ubuntu {securedrop_target_platform}-security main', + 'deb http://security.ubuntu.com/ubuntu {securedrop_target_platform}-security universe', + 'deb http://archive.ubuntu.com/ubuntu/ {securedrop_target_platform}-updates main', + 'deb http://archive.ubuntu.com/ubuntu/ {securedrop_target_platform} main' +]) +def test_sources_list(host, repo): + """ + Ensure the correct apt repositories are specified + in the sources.list for apt. + """ + repo_config = repo.format( + securedrop_target_platform=host.system_info.codename + ) + f = host.file('/etc/apt/sources.list') + if host.system_info.codename == "focal": + assert f.is_file + assert f.user == "root" + assert f.mode == 0o644 + repo_regex = '^{}$'.format(re.escape(repo_config)) + assert f.contains(repo_regex) def test_cron_apt_repo_config_update(host): """ - Ensure cron-apt updates repos from the security.list config. + Ensure cron-apt updates repos from the security.list config for Xenial. """ f = host.file('/etc/cron-apt/action.d/0-update') - assert f.is_file - assert f.user == "root" - assert f.mode == 0o644 - repo_config = str('update -o quiet=2' - ' -o Dir::Etc::SourceList=/etc/apt/security.list' - ' -o Dir::Etc::SourceParts=""') - assert f.contains('^{}$'.format(repo_config)) + if host.system_info.codename == "xenial": + assert f.is_file + assert f.user == "root" + assert f.mode == 0o644 + repo_config = str('update -o quiet=2' + ' -o Dir::Etc::SourceList=/etc/apt/security.list' + ' -o Dir::Etc::SourceParts=""') + assert f.contains('^{}$'.format(repo_config)) + else: + assert not f.exists def test_cron_apt_delete_vanilla_kernels(host): """ - Ensure cron-apt removes generic linux image packages when installed. + Ensure cron-apt removes generic linux image packages when installed. This + file is provisioned via ansible and via the securedrop-config package. We + should remove it once Xenial is fully deprecated. In the meantime, it will + not impact Focal systems running unattended-upgrades """ f = host.file('/etc/cron-apt/action.d/9-remove') @@ -96,15 +132,18 @@ def test_cron_apt_repo_config_upgrade(host): Ensure cron-apt upgrades packages from the security.list config. """ f = host.file('/etc/cron-apt/action.d/5-security') - assert f.is_file - assert f.user == "root" - assert f.mode == 0o644 - assert f.contains('^autoclean -y$') - repo_config = str('dist-upgrade -y -o APT::Get::Show-Upgraded=true' - ' -o Dir::Etc::SourceList=/etc/apt/security.list' - ' -o Dpkg::Options::=--force-confdef' - ' -o Dpkg::Options::=--force-confold') - assert f.contains(re.escape(repo_config)) + if host.system_info.codename == "xenial": + assert f.is_file + assert f.user == "root" + assert f.mode == 0o644 + assert f.contains('^autoclean -y$') + repo_config = str('dist-upgrade -y -o APT::Get::Show-Upgraded=true' + ' -o Dir::Etc::SourceList=/etc/apt/security.list' + ' -o Dpkg::Options::=--force-confdef' + ' -o Dpkg::Options::=--force-confold') + assert f.contains(re.escape(repo_config)) + else: + assert not f.exists def test_cron_apt_config_deprecated(host): @@ -130,18 +169,75 @@ def test_cron_apt_cron_jobs(host, cron_job): to make sure those have been cleaned up via the playbooks. """ f = host.file('/etc/cron.d/cron-apt') - assert f.is_file - assert f.user == "root" - assert f.mode == 0o644 - regex_job = '^{}$'.format(re.escape(cron_job['job'])) - if cron_job['state'] == 'present': - assert f.contains(regex_job) + if host.system_info.codename == "xenial": + assert f.is_file + assert f.user == "root" + assert f.mode == 0o644 + + regex_job = '^{}$'.format(re.escape(cron_job['job'])) + if cron_job['state'] == 'present': + assert f.contains(regex_job) + else: + assert not f.contains(regex_job) else: - assert not f.contains(regex_job) + assert not f.exists + + +def test_unattended_upgrades_config(host): + """ + Ensures the 50unattended-upgrades config is correct only under Ubuntu Focal + """ + f = host.file('/etc/apt/apt.conf.d/50unattended-upgrades') + if host.system_info.codename == "xenial": + assert not f.exists + else: + assert f.is_file + assert f.user == "root" + assert f.mode == 0o644 + assert f.contains("SecureDrop:${distro_codename}") + + +@pytest.mark.parametrize('option', [ + 'APT::Periodic::Update-Package-Lists "1";', + 'APT::Periodic::Unattended-Upgrade "1";', + 'APT::Periodic::AutocleanInterval "1";', + ]) +def test_auto_upgrades_config(host, option): + """ + Ensures the 20auto-upgrades config is correct only under Ubuntu Focal + """ + f = host.file('/etc/apt/apt.conf.d/20auto-upgrades') + if host.system_info.codename == "xenial": + assert not f.exists + else: + assert f.is_file + assert f.user == "root" + assert f.mode == 0o644 + assert f.contains('^{}$'.format(option)) + + +def test_unattended_upgrades_functional(host): + """ + Ensure unatteded-upgrades completes successfully and ensures all packages + are up-to-date. + """ + if host.system_info.codename != "xenial": + c = host.run('sudo unattended-upgrades -d') + assert c.rc == 0 + expected_origins = ( + "Allowed origins are: o=Ubuntu,a=focal, o=Ubuntu,a=focal-security" + ", o=Ubuntu,a=focal-updates, o=SecureDrop,a=focal" + ) + expected_result = ( + "No packages found that can be upgraded unattended and no pending auto-removals" + ) + + assert expected_origins in c.stdout + assert expected_result in c.stdout -def test_cron_apt_all_packages_updated(host): +def test_all_packages_updated(host): """ Ensure a safe-upgrade has already been run, by checking that no packages are eligible for upgrade currently. From 0e790281344e3f8bbe392b54f77bfefaa65751a1 Mon Sep 17 00:00:00 2001 From: mickael e Date: Wed, 6 Jan 2021 16:18:05 -0500 Subject: [PATCH 4/8] Use Origin-Patterns instead of Allowed-Origins The current configuration of SecureDrop apt servers does not contain the `Suite` field which, with `Origin`, is the default combination used by unattended-upgrades to allow certain package lists to be updated. In order to avoid using a wildcard (e.g. SecureDrop:*, which would allow any (configured in apt sources) Suite for the SecureDrop Origin to be updated through unatteded upgrades, we use the newer Allowed-Origins syntax to specify the `Codename` field in lieu of the `Suite` field for SecureDrop. --- .../roles/common/templates/50unattended-upgrades.j2 | 12 ++++++------ molecule/testinfra/common/test_automatic_updates.py | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/install_files/ansible-base/roles/common/templates/50unattended-upgrades.j2 b/install_files/ansible-base/roles/common/templates/50unattended-upgrades.j2 index c49ee89e14..2acf30d122 100644 --- a/install_files/ansible-base/roles/common/templates/50unattended-upgrades.j2 +++ b/install_files/ansible-base/roles/common/templates/50unattended-upgrades.j2 @@ -1,9 +1,9 @@ -// Automatically upgrade packages from these (origin:archive) pairs -Unattended-Upgrade::Allowed-Origins { - "${distro_id}:${distro_codename}"; - "${distro_id}:${distro_codename}-security"; - "${distro_id}:${distro_codename}-updates"; - "SecureDrop:${distro_codename}"; +// Automatically upgrade packages from these (origin:archive/codename) pairs +Unattended-Upgrade::Origins-Pattern { + "origin=${distro_id},archive=${distro_codename}"; + "origin=${distro_id},archive=${distro_codename}-security"; + "origin=${distro_id},archive=${distro_codename}-updates"; + "origin=SecureDrop,codename=${distro_codename}"; }; // List of packages to not update (regexp are supported) diff --git a/molecule/testinfra/common/test_automatic_updates.py b/molecule/testinfra/common/test_automatic_updates.py index 29db45a21a..7f33413fd4 100644 --- a/molecule/testinfra/common/test_automatic_updates.py +++ b/molecule/testinfra/common/test_automatic_updates.py @@ -195,7 +195,7 @@ def test_unattended_upgrades_config(host): assert f.is_file assert f.user == "root" assert f.mode == 0o644 - assert f.contains("SecureDrop:${distro_codename}") + assert f.contains("origin=SecureDrop,codename=${distro_codename}") @pytest.mark.parametrize('option', [ @@ -226,8 +226,8 @@ def test_unattended_upgrades_functional(host): c = host.run('sudo unattended-upgrades -d') assert c.rc == 0 expected_origins = ( - "Allowed origins are: o=Ubuntu,a=focal, o=Ubuntu,a=focal-security" - ", o=Ubuntu,a=focal-updates, o=SecureDrop,a=focal" + "Allowed origins are: origin=Ubuntu,archive=focal, origin=Ubuntu,archive=focal-security" + ", origin=Ubuntu,archive=focal-updates, origin=SecureDrop,codename=focal" ) expected_result = ( "No packages found that can be upgraded unattended and no pending auto-removals" From 50b9365a95905fc4966e3bdbe76fc8e0df5e5054 Mon Sep 17 00:00:00 2001 From: mickael e Date: Wed, 6 Jan 2021 16:58:18 -0500 Subject: [PATCH 5/8] Add SecureDrop to Origin field in apt-local's reprepro config This is inline with the configuration provided by apt(-test).freedom.press. --- devops/apt-local.yml | 2 +- molecule/upgrade/templates/distributions.j2 | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/devops/apt-local.yml b/devops/apt-local.yml index e15acc8318..62adab0813 100644 --- a/devops/apt-local.yml +++ b/devops/apt-local.yml @@ -29,6 +29,7 @@ rep_dist: "focal" molecule_dir: "../molecule/upgrade" dpkg_dir: /var/repos/debs + rep_origin: SecureDrop rep_component: main rep_arch: i386 amd64 release_file: "/var/repos/base/dists/{{ rep_dist }}/Release" @@ -45,4 +46,3 @@ - ssl_certificate_key /etc/ssl/private/apt_freedom_press.priv - root "/var/repos/base" - location / { autoindex on; } - diff --git a/molecule/upgrade/templates/distributions.j2 b/molecule/upgrade/templates/distributions.j2 index 4484011bc1..9b2096851d 100644 --- a/molecule/upgrade/templates/distributions.j2 +++ b/molecule/upgrade/templates/distributions.j2 @@ -1,3 +1,4 @@ +Origin: {{ rep_origin }} Codename: {{ rep_dist }} Components: {{ rep_component }} Architectures: {{ rep_arch }} From aef3584b59ff1338b5f08ae33fea982c9ab631b6 Mon Sep 17 00:00:00 2001 From: mickael e Date: Wed, 6 Jan 2021 17:55:28 -0500 Subject: [PATCH 6/8] Ensure unattended-upgrades reboots the hosts nightly Unattended upgrades will reboot the system if and only if the dependency that is being updated requires a reboot. However, SecureDrop relies on cron-apt's reboot to ensure the memory is regularly cleared from the system. In order to reboot on a daily basis, we drop the updates-required flag in /var/run to tell unattended-upgrades that the system should be rebooted at the scheduled time. The absence of update-notifier-common package will make daily reboots silently fail, so adding explicitly to the apt package install step to ensure it's installed as it's not pulled in the Depends field of unattended-upgrades. --- .../roles/common/tasks/unattended_upgrades.yml | 18 +++++++++++++++--- .../testinfra/common/test_automatic_updates.py | 14 ++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml b/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml index 63b2d6a40b..94c026d783 100644 --- a/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml +++ b/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml @@ -1,9 +1,12 @@ --- -- name: Install unattended-upgrades package. +- name: Install unattended-upgrades package and update-notifier-common apt: - name: unattended-upgrades - state: latest + name: "{{ item }}" + state: present update_cache: yes + with_items: + - unattended-upgrades + - update-notifier-common tags: - apt - unattended-upgrades @@ -29,3 +32,12 @@ tags: - apt - unattended-upgrades + +- name: Add cron job to indicate to unattended-upgrades that a reboot is required. + cron: + name: Indicate that a reboot is required at the scheduled time. + job: "touch /var/run/reboot-required" + hour: "*/12" + tags: + - cron + - unatted-upgrades diff --git a/molecule/testinfra/common/test_automatic_updates.py b/molecule/testinfra/common/test_automatic_updates.py index 7f33413fd4..7248460257 100644 --- a/molecule/testinfra/common/test_automatic_updates.py +++ b/molecule/testinfra/common/test_automatic_updates.py @@ -237,6 +237,20 @@ def test_unattended_upgrades_functional(host): assert expected_result in c.stdout +def test_reboot_required_cron(host): + """ + Unatteded-upgrades does not reboot the system if the updates don't require it. + However, we use daily reboots for SecureDrop to ensure memory is cleared periodically. + Here, we ensure that reboot-required flag is dropped twice daily to ensure the system + is rebooted every day at the scheduled time. + """ + if host.system_info.codename != "xenial": + with host.sudo(): + cronlist = host.run("crontab -l").stdout + cronjob = "* */12 * * * touch /var/run/reboot-required" + assert cronjob in cronlist + + def test_all_packages_updated(host): """ Ensure a safe-upgrade has already been run, by checking that no From bddc18453b489384fab097e021803907cd2cb53d Mon Sep 17 00:00:00 2001 From: mickael e Date: Thu, 21 Jan 2021 14:01:39 -0500 Subject: [PATCH 7/8] Provide unattended-upgrades configuration via apt package securedrop-config package will provide the configuration for focal only --- .../common/tasks/unattended_upgrades.yml | 39 ++------------- .../roles/common/templates/80securedrop.j2 | 4 ++ .../securedrop-config-focal/DEBIAN/control.j2 | 11 +++++ .../securedrop-config-focal/DEBIAN/postinst | 24 ++++++++++ .../etc/profile.d/securedrop_additions.sh | 22 +++++++++ .../opt/securedrop}/20auto-upgrades | 0 .../opt/securedrop/50unattended-upgrades} | 6 ++- .../opt/securedrop/reboot-flag | 4 ++ molecule/builder-focal/playbook.yml | 1 + .../tests/test_securedrop_deb_package.py | 16 +++++-- .../common/test_automatic_updates.py | 47 ++++++++++++++----- 11 files changed, 123 insertions(+), 51 deletions(-) create mode 100644 install_files/ansible-base/roles/common/templates/80securedrop.j2 create mode 100644 install_files/securedrop-config-focal/DEBIAN/control.j2 create mode 100755 install_files/securedrop-config-focal/DEBIAN/postinst create mode 100644 install_files/securedrop-config-focal/etc/profile.d/securedrop_additions.sh rename install_files/{ansible-base/roles/common/files => securedrop-config-focal/opt/securedrop}/20auto-upgrades (100%) rename install_files/{ansible-base/roles/common/templates/50unattended-upgrades.j2 => securedrop-config-focal/opt/securedrop/50unattended-upgrades} (89%) create mode 100644 install_files/securedrop-config-focal/opt/securedrop/reboot-flag diff --git a/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml b/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml index 94c026d783..86f4fc1fdf 100644 --- a/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml +++ b/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml @@ -1,43 +1,14 @@ --- -- name: Install unattended-upgrades package and update-notifier-common - apt: - name: "{{ item }}" - state: present - update_cache: yes - with_items: - - unattended-upgrades - - update-notifier-common - tags: - - apt - - unattended-upgrades +# Configuration for unattended upgrades is almost exclusively managed by the +# securedrop-config package under Focal. -- name: Configure unattended-upgrades to update the packages from sources.list. - copy: - src: 20auto-upgrades - dest: /etc/apt/apt.conf.d/20auto-upgrades - mode: 0644 - owner: root - group: root - tags: - - apt - - unattended-upgrades - -- name: Configure unattended-upgrades to update the packages from sources.list. +- name: Configure unattended-upgrades to reboot daily at the scheduled time. template: - src: 50unattended-upgrades.j2 - dest: /etc/apt/apt.conf.d/50unattended-upgrades + src: 80securedrop.j2 + dest: /etc/apt/apt.conf.d/80securedrop mode: 0644 owner: root group: root tags: - apt - unattended-upgrades - -- name: Add cron job to indicate to unattended-upgrades that a reboot is required. - cron: - name: Indicate that a reboot is required at the scheduled time. - job: "touch /var/run/reboot-required" - hour: "*/12" - tags: - - cron - - unatted-upgrades diff --git a/install_files/ansible-base/roles/common/templates/80securedrop.j2 b/install_files/ansible-base/roles/common/templates/80securedrop.j2 new file mode 100644 index 0000000000..8453dd7943 --- /dev/null +++ b/install_files/ansible-base/roles/common/templates/80securedrop.j2 @@ -0,0 +1,4 @@ +// If automatic reboot is enabled and needed, reboot at the specific +// time instead of immediately +// Default: "now" +Unattended-Upgrade::Automatic-Reboot-Time "{{ daily_reboot_time }}:00"; diff --git a/install_files/securedrop-config-focal/DEBIAN/control.j2 b/install_files/securedrop-config-focal/DEBIAN/control.j2 new file mode 100644 index 0000000000..e73924cad3 --- /dev/null +++ b/install_files/securedrop-config-focal/DEBIAN/control.j2 @@ -0,0 +1,11 @@ +Source: securedrop +Section: web +Priority: optional +Maintainer: SecureDrop Team +Homepage: https://securedrop.org +Package: securedrop-config +Version: 0.1.3+{{ securedrop_version }}+{{ securedrop_target_distribution }} +Depends: unattended-upgrades,update-notifier-common +Architecture: all +Description: Establishes baseline system state for running SecureDrop. + Configures apt repositories. diff --git a/install_files/securedrop-config-focal/DEBIAN/postinst b/install_files/securedrop-config-focal/DEBIAN/postinst new file mode 100755 index 0000000000..e71a1ada9a --- /dev/null +++ b/install_files/securedrop-config-focal/DEBIAN/postinst @@ -0,0 +1,24 @@ +#!/bin/sh +# postinst script for securedrop-config-focal + +set -e +set -x + +case "$1" in + configure) + # Configuration required for unattended-upgrades + cp /opt/securedrop/20auto-upgrades /etc/apt/apt.conf.d/ + cp /opt/securedrop/50unattended-upgrades /etc/apt/apt.conf.d/ + cp /opt/securedrop/reboot-flag /etc/cron.d/ + + ;; + abort-upgrade|abort-remove|abort-deconfigure) + ;; + + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/install_files/securedrop-config-focal/etc/profile.d/securedrop_additions.sh b/install_files/securedrop-config-focal/etc/profile.d/securedrop_additions.sh new file mode 100644 index 0000000000..4bab3c8ea3 --- /dev/null +++ b/install_files/securedrop-config-focal/etc/profile.d/securedrop_additions.sh @@ -0,0 +1,22 @@ +[[ $- != *i* ]] && return + +which tmux >/dev/null 2>&1 || return + +tmux_attach_via_proc() { + # If the tmux package is upgraded during the lifetime of a + # session, attaching with the new binary can fail due to different + # protocol versions. This function attaches using the reference to + # the old executable found in the /proc tree of an existing + # session. + pid=$(pgrep --newest tmux) + if test -n "$pid" + then + /proc/$pid/exe attach + fi + return 1 +} + +if test -z "$TMUX" +then + (tmux attach || tmux_attach_via_proc || tmux new-session) +fi diff --git a/install_files/ansible-base/roles/common/files/20auto-upgrades b/install_files/securedrop-config-focal/opt/securedrop/20auto-upgrades similarity index 100% rename from install_files/ansible-base/roles/common/files/20auto-upgrades rename to install_files/securedrop-config-focal/opt/securedrop/20auto-upgrades diff --git a/install_files/ansible-base/roles/common/templates/50unattended-upgrades.j2 b/install_files/securedrop-config-focal/opt/securedrop/50unattended-upgrades similarity index 89% rename from install_files/ansible-base/roles/common/templates/50unattended-upgrades.j2 rename to install_files/securedrop-config-focal/opt/securedrop/50unattended-upgrades index 2acf30d122..2b9360d088 100644 --- a/install_files/ansible-base/roles/common/templates/50unattended-upgrades.j2 +++ b/install_files/securedrop-config-focal/opt/securedrop/50unattended-upgrades @@ -49,7 +49,11 @@ Unattended-Upgrade::Automatic-Reboot "true"; // If automatic reboot is enabled and needed, reboot at the specific // time instead of immediately // Default: "now" -Unattended-Upgrade::Automatic-Reboot-Time "{{ daily_reboot_time }}:00"; +// This is set in a template in the common role under the file 80securedrop + +// Automatically reboot even if there are users currently logged in +// when Unattended-Upgrade::Automatic-Reboot is set to true +Unattended-Upgrade::Automatic-Reboot-WithUsers "true"; // Use apt bandwidth limit feature, this example limits the download // speed to 70kb/sec diff --git a/install_files/securedrop-config-focal/opt/securedrop/reboot-flag b/install_files/securedrop-config-focal/opt/securedrop/reboot-flag new file mode 100644 index 0000000000..469c699ed5 --- /dev/null +++ b/install_files/securedrop-config-focal/opt/securedrop/reboot-flag @@ -0,0 +1,4 @@ +# The purpose of this cron is to drop the reboot-required flag every 12 hours +# to ensure the system is rebooted nightly, regardless of updates being installed +# or not. +* */12 * * * touch /var/run/reboot-required diff --git a/molecule/builder-focal/playbook.yml b/molecule/builder-focal/playbook.yml index 6ee557567a..c428ab3111 100644 --- a/molecule/builder-focal/playbook.yml +++ b/molecule/builder-focal/playbook.yml @@ -48,6 +48,7 @@ - role: build-generic-pkg tags: securedrop-config package_name: securedrop-config + package_dirname: securedrop-config-focal when: ansible_host.endswith("-sd-config") or ansible_host == "localhost" tags: rebuild diff --git a/molecule/builder-xenial/tests/test_securedrop_deb_package.py b/molecule/builder-xenial/tests/test_securedrop_deb_package.py index 7685f7c42b..7dc4b855a2 100644 --- a/molecule/builder-xenial/tests/test_securedrop_deb_package.py +++ b/molecule/builder-xenial/tests/test_securedrop_deb_package.py @@ -543,10 +543,18 @@ def test_config_package_contains_expected_files(host: Host) -> None: Inspect the package contents to ensure all config files are included in the package. """ - wanted_files = [ - "/etc/cron-apt/action.d/9-remove", - "/etc/profile.d/securedrop_additions.sh", - ] + if SECUREDROP_TARGET_DISTRIBUTION == "xenial": + wanted_files = [ + "/etc/cron-apt/action.d/9-remove", + "/etc/profile.d/securedrop_additions.sh", + ] + else: + wanted_files = [ + "/etc/profile.d/securedrop_additions.sh", + "/opt/securedrop/20auto-upgrades", + "/opt/securedrop/50unattended-upgrades", + "/opt/securedrop/reboot-flag", + ] c = host.run("dpkg-deb --contents {}".format(deb_paths["securedrop_config"])) for wanted_file in wanted_files: assert re.search( diff --git a/molecule/testinfra/common/test_automatic_updates.py b/molecule/testinfra/common/test_automatic_updates.py index 7248460257..3d0fa534a8 100644 --- a/molecule/testinfra/common/test_automatic_updates.py +++ b/molecule/testinfra/common/test_automatic_updates.py @@ -118,13 +118,16 @@ def test_cron_apt_delete_vanilla_kernels(host): """ f = host.file('/etc/cron-apt/action.d/9-remove') - assert f.is_file - assert f.user == "root" - assert f.mode == 0o644 - command = str('remove -y' - ' linux-image-generic-lts-xenial linux-image-.*generic' - ' -o quiet=2') - assert f.contains('^{}$'.format(command)) + if host.system_info.codename == "xenial": + assert f.is_file + assert f.user == "root" + assert f.mode == 0o644 + command = str('remove -y' + ' linux-image-generic-lts-xenial linux-image-.*generic' + ' -o quiet=2') + assert f.contains('^{}$'.format(command)) + else: + assert not f.exists def test_cron_apt_repo_config_upgrade(host): @@ -198,6 +201,20 @@ def test_unattended_upgrades_config(host): assert f.contains("origin=SecureDrop,codename=${distro_codename}") +def test_unattended_securedrop_specific(host): + """ + Ensures the 80securedrop config is correct only under Ubuntu Focal + """ + f = host.file('/etc/apt/apt.conf.d/80securedrop') + if host.system_info.codename == "xenial": + assert not f.exists + else: + assert f.is_file + assert f.user == "root" + assert f.mode == 0o644 + assert f.contains("Automatic-Reboot-Time") + + @pytest.mark.parametrize('option', [ 'APT::Periodic::Update-Package-Lists "1";', 'APT::Periodic::Unattended-Upgrade "1";', @@ -244,11 +261,17 @@ def test_reboot_required_cron(host): Here, we ensure that reboot-required flag is dropped twice daily to ensure the system is rebooted every day at the scheduled time. """ - if host.system_info.codename != "xenial": - with host.sudo(): - cronlist = host.run("crontab -l").stdout - cronjob = "* */12 * * * touch /var/run/reboot-required" - assert cronjob in cronlist + f = host.file('/etc/cron.d/reboot-flag') + + if host.system_info.codename == "xenial": + assert not f.exists + else: + assert f.is_file + assert f.user == "root" + assert f.mode == 0o644 + + line = '^{}$'.format(re.escape("* */12 * * * touch /var/run/reboot-required")) + assert f.contains(line) def test_all_packages_updated(host): From 733bd118e1a02ee4b5b9aa7f3d17501e5a497013 Mon Sep 17 00:00:00 2001 From: mickael e Date: Mon, 1 Feb 2021 17:08:42 -0500 Subject: [PATCH 8/8] Unmask services and timers, bump securedrop-config to 0.1.4 Based on feedback from @kushaldas, in some conditions the apt-daily and apt-daily upgrade would be masked and stopped after install. This should ensure the services and timers are properly enabled after install. They are automatically started by cron when the upgrade processes are invoked. --- .circleci/config.yml | 1 - .../securedrop_application_server.yml | 2 +- .../group_vars/securedrop_monitor_server.yml | 2 +- .../common/tasks/unattended_upgrades.yml | 25 +++++++++++++++++++ .../ansible-base/securedrop-apt-local.yml | 1 - .../securedrop-config-focal/DEBIAN/control.j2 | 2 +- .../securedrop-config/DEBIAN/control.j2 | 2 +- molecule/builder-xenial/tests/vars.yml | 2 +- .../common/test_automatic_updates.py | 18 +++++++++++++ 9 files changed, 48 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4923e1516f..8c9b3c890a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,7 +93,6 @@ common-steps: - /focalcaches/layers.tar - version: 2 jobs: lint: diff --git a/install_files/ansible-base/group_vars/securedrop_application_server.yml b/install_files/ansible-base/group_vars/securedrop_application_server.yml index 10ce72f36a..e81244a649 100644 --- a/install_files/ansible-base/group_vars/securedrop_application_server.yml +++ b/install_files/ansible-base/group_vars/securedrop_application_server.yml @@ -7,7 +7,7 @@ ip_info: ### Used by the install_local_deb_pkgs role ### local_deb_packages: - "securedrop-keyring-0.1.4+{{ securedrop_version }}+{{ securedrop_target_distribution }}-amd64.deb" - - "securedrop-config-0.1.3+{{ securedrop_version }}+{{ securedrop_target_distribution }}-amd64.deb" + - "securedrop-config-0.1.4+{{ securedrop_version }}+{{ securedrop_target_distribution }}-amd64.deb" - "securedrop-ossec-agent-3.6.0+{{ securedrop_version }}+{{ securedrop_target_distribution }}-amd64.deb" - "{{ securedrop_app_code_deb }}.deb" - "ossec-agent-3.6.0+{{ securedrop_target_distribution }}-amd64.deb" diff --git a/install_files/ansible-base/group_vars/securedrop_monitor_server.yml b/install_files/ansible-base/group_vars/securedrop_monitor_server.yml index ad5ca58a3c..ed4a28eeb7 100644 --- a/install_files/ansible-base/group_vars/securedrop_monitor_server.yml +++ b/install_files/ansible-base/group_vars/securedrop_monitor_server.yml @@ -7,7 +7,7 @@ ip_info: ### Used by the install_local_deb_pkgs role ### local_deb_packages: - "securedrop-keyring-0.1.4+{{ securedrop_version }}+{{ securedrop_target_distribution }}-amd64.deb" - - "securedrop-config-0.1.3+{{ securedrop_version }}+{{ securedrop_target_distribution }}-amd64.deb" + - "securedrop-config-0.1.4+{{ securedrop_version }}+{{ securedrop_target_distribution }}-amd64.deb" - "securedrop-ossec-server-3.6.0+{{ securedrop_version }}+{{ securedrop_target_distribution }}-amd64.deb" - ossec-server-3.6.0+{{ securedrop_target_distribution }}-amd64.deb diff --git a/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml b/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml index 86f4fc1fdf..e82544daec 100644 --- a/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml +++ b/install_files/ansible-base/roles/common/tasks/unattended_upgrades.yml @@ -12,3 +12,28 @@ tags: - apt - unattended-upgrades + +- name: Ensure apt-daily and apt-daily-upgrade services are unmasked, started and enabled. + systemd: + name: "{{ item }}" + state: started + enabled: yes + masked: no + with_items: + - 'apt-daily' + - 'apt-daily-upgrade' + tags: + - apt + - unattended-upgrades + +- name: Ensure apt-daily and apt-daily-upgrade timers are started, and enabled. + systemd: + name: "{{ item }}" + state: started + enabled: yes + with_items: + - 'apt-daily.timer' + - 'apt-daily-upgrade.timer' + tags: + - apt + - unattended-upgrades diff --git a/install_files/ansible-base/securedrop-apt-local.yml b/install_files/ansible-base/securedrop-apt-local.yml index 28b081d0fa..20b1e0a5b5 100644 --- a/install_files/ansible-base/securedrop-apt-local.yml +++ b/install_files/ansible-base/securedrop-apt-local.yml @@ -30,4 +30,3 @@ state: present update_cache: yes become: yes - diff --git a/install_files/securedrop-config-focal/DEBIAN/control.j2 b/install_files/securedrop-config-focal/DEBIAN/control.j2 index e73924cad3..128b3c1181 100644 --- a/install_files/securedrop-config-focal/DEBIAN/control.j2 +++ b/install_files/securedrop-config-focal/DEBIAN/control.j2 @@ -4,7 +4,7 @@ Priority: optional Maintainer: SecureDrop Team Homepage: https://securedrop.org Package: securedrop-config -Version: 0.1.3+{{ securedrop_version }}+{{ securedrop_target_distribution }} +Version: 0.1.4+{{ securedrop_version }}+{{ securedrop_target_distribution }} Depends: unattended-upgrades,update-notifier-common Architecture: all Description: Establishes baseline system state for running SecureDrop. diff --git a/install_files/securedrop-config/DEBIAN/control.j2 b/install_files/securedrop-config/DEBIAN/control.j2 index 3cf3f3d924..5cf6e9b8e1 100644 --- a/install_files/securedrop-config/DEBIAN/control.j2 +++ b/install_files/securedrop-config/DEBIAN/control.j2 @@ -4,7 +4,7 @@ Priority: optional Maintainer: SecureDrop Team Homepage: https://securedrop.org Package: securedrop-config -Version: 0.1.3+{{ securedrop_version }}+{{ securedrop_target_distribution }} +Version: 0.1.4+{{ securedrop_version }}+{{ securedrop_target_distribution }} Architecture: all Description: Establishes baseline system state for running SecureDrop. Configures apt repositories. diff --git a/molecule/builder-xenial/tests/vars.yml b/molecule/builder-xenial/tests/vars.yml index 01f17decaf..3fb30a40f8 100644 --- a/molecule/builder-xenial/tests/vars.yml +++ b/molecule/builder-xenial/tests/vars.yml @@ -2,7 +2,7 @@ securedrop_version: "1.8.0~rc1" ossec_version: "3.6.0" keyring_version: "0.1.4" -config_version: "0.1.3" +config_version: "0.1.4" grsec_version: "4.14.188" # These values will be interpolated with values populated above diff --git a/molecule/testinfra/common/test_automatic_updates.py b/molecule/testinfra/common/test_automatic_updates.py index 3d0fa534a8..0b0b730125 100644 --- a/molecule/testinfra/common/test_automatic_updates.py +++ b/molecule/testinfra/common/test_automatic_updates.py @@ -254,6 +254,24 @@ def test_unattended_upgrades_functional(host): assert expected_result in c.stdout +@pytest.mark.parametrize('service', [ + 'apt-daily', + 'apt-daily.timer', + 'apt-daily-upgrade', + 'apt-daily-upgrade.timer', + ]) +def test_apt_daily_services_and_timers_enabled(host, service): + """ + Ensure the services and timers used for unattended upgrades are enabled + in Ubuntu 20.04 Focal. + """ + if host.system_info.codename != "xenial": + with host.sudo(): + # The services are started only when the upgrades are being performed. + s = host.service(service) + assert s.is_enabled + + def test_reboot_required_cron(host): """ Unatteded-upgrades does not reboot the system if the updates don't require it.