Merge pull request #619 from freedomofpress/608-import-export-securit…

…y-rebased

Template consolidation via GUI updater

VERSION

@@ -1 +1 @@
-0.4.0
+0.5.0-rc1

launcher/sdw_updater_gui/Updater.py

@@ -23,35 +23,71 @@
 LOCK_FILE = "sdw-launcher.lock"
 LOG_FILE = "launcher.log"
 
+
+# We use a hardcoded temporary directory path in dom0. As dom0 is not
+# a multi-user environment, we can safely assume that only the Updater is
+# managing that filepath. Later on, we should consider porting the check-migration
+# logic to leverage the Qubes Python API.
+MIGRATION_DIR = "/tmp/sdw-migrations"  # nosec
+
 sdlog = logging.getLogger(__name__)
 
 # The are the TemplateVMs that require full patch level at boot in order to start the client,
 # as well as their associated TemplateVMs.
 # In the future, we could use qvm-prefs to extract this information.
-current_templates = {
-    "dom0": "dom0",
+current_vms = {
     "fedora": "fedora-31",
-    "sd-viewer": "sd-viewer-buster-template",
-    "sd-app": "sd-app-buster-template",
-    "sd-log": "sd-log-buster-template",
-    "sd-devices": "sd-devices-buster-template",
-    "sd-proxy": "sd-proxy-buster-template",
+    "sd-viewer": "sd-large-buster-template",
+    "sd-app": "sd-small-buster-template",
+    "sd-log": "sd-small-buster-template",
+    "sd-devices": "sd-large-buster-template",
+    "sd-proxy": "sd-small-buster-template",
     "sd-whonix": "whonix-gw-15",
-    "sd-gpg": "securedrop-workstation-buster",
+    "sd-gpg": "sd-small-buster-template",
 }
 
+current_templates = set([val for key, val in current_vms.items() if key != "dom0"])
+
 
 def get_dom0_path(folder):
     return os.path.join(os.path.expanduser("~"), folder)
 
 
-def apply_updates(vms=current_templates.keys()):
+def run_full_install():
+    """
+    Re-apply the entire Salt config via sdw-admin. Required to enforce
+    VM state during major migrations, such as template consolidation.
+    """
+    sdlog.info("Running sdw-admin apply")
+    cmd = ["sdw-admin", "--apply"]
+    subprocess.check_call(cmd)
+
+    # Clean up flag requesting migration. Shell out since root created it.
+    subprocess.check_call(["sudo", "rm", "-rf", MIGRATION_DIR])
+
+
+def migration_is_required():
+    """
+    Check whether a full run of the Salt config via sdw-admin is required.
+    """
+    result = False
+    if os.path.exists(MIGRATION_DIR):
+        if len(os.listdir(MIGRATION_DIR)) > 0:
+            sdlog.info("Migration is required, will enforce full config during update")
+            result = True
+    return result
+
+
+def apply_updates(vms=current_templates):
     """
     Apply updates to all TemplateVMs
     """
+    # The updater thread sets 15% progress before the per-VM
+    # updates start, we'll base progress on that.
+    progress_start = 15
     sdlog.info("Applying all updates")
 
-    for progress_current, vm in enumerate(vms):
+    for progress_current, vm in enumerate(vms, 1):
         upgrade_results = UpdateStatus.UPDATES_FAILED
 
         if vm == "dom0":
@@ -63,7 +99,9 @@ def apply_updates(vms=current_templates.keys()):
         else:
             upgrade_results = _apply_updates_vm(vm)
 
-        progress_percentage = int(((progress_current + 1) / len(vms)) * 100 - 5)
+        progress_percentage = int(progress_start + ((progress_current) / len(vms)) * 100 - 25)
+        if progress_percentage < progress_start:
+            progress_percentage = progress_start
         yield vm, progress_percentage, upgrade_results
 
 
@@ -109,28 +147,18 @@ def _apply_updates_vm(vm):
     Apply updates to a given TemplateVM. Any update to the base fedora template
     will require a reboot after the upgrade.
     """
-    sdlog.info("Updating {}:{}".format(vm, current_templates[vm]))
+    sdlog.info("Updating {}".format(vm))
     try:
         subprocess.check_call(
-            [
-                "sudo",
-                "qubesctl",
-                "--skip-dom0",
-                "--targets",
-                current_templates[vm],
-                "state.sls",
-                "update.qubes-vm",
-            ]
+            ["sudo", "qubesctl", "--skip-dom0", "--targets", vm, "state.sls", "update.qubes-vm"]
         )
     except subprocess.CalledProcessError as e:
         sdlog.error(
-            "An error has occurred updating {}. Please contact your administrator.".format(
-                current_templates[vm]
-            )
+            "An error has occurred updating {}. Please contact your administrator.".format(vm)
         )
         sdlog.error(str(e))
         return UpdateStatus.UPDATES_FAILED
-    sdlog.info("{} update successful".format(current_templates[vm]))
+    sdlog.info("{} update successful".format(vm))
     return UpdateStatus.UPDATES_OK
 
 
@@ -338,11 +366,8 @@ def shutdown_and_start_vms():
         "sd-log",
     ]
 
-    # All TemplateVMs minus dom0
-    sdw_templates = [val for key, val in current_templates.items() if key != "dom0"]
-
     sdlog.info("Shutting down SDW TemplateVMs for updates")
-    for vm in sdw_templates:
+    for vm in sorted(current_templates):
         _safely_shutdown_vm(vm)
 
     sdlog.info("Shutting down SDW AppVMs for updates")

launcher/sdw_updater_gui/UpdaterApp.py

@@ -172,17 +172,34 @@ def __init__(self):
         QThread.__init__(self)
 
     def run(self):
-        upgrade_generator = Updater.apply_updates()
-        results = {}
 
-        for vm, progress, result in upgrade_generator:
-            results[vm] = result
-            self.progress_signal.emit(progress)
+        # Update dom0 first, then apply dom0 state. If full state run
+        # is required, the dom0 state will drop a flag.
+        self.progress_signal.emit(5)
+        upgrade_generator = Updater.apply_updates(["dom0"])
 
+        results = {}
         # apply dom0 state
+        self.progress_signal.emit(10)
         result = Updater.apply_dom0_state()
         # add to results dict, if it fails it will show error message
         results["apply_dom0"] = result.value
+
+        self.progress_signal.emit(15)
+        # rerun full config if dom0 checks determined it's required,
+        # otherwise proceed with per-VM package updates
+        if Updater.migration_is_required():
+            # Progress bar will freeze for ~15m during full state run
+            self.progress_signal.emit(35)
+            Updater.run_full_install()
+            self.progress_signal.emit(75)
+        else:
+            upgrade_generator = Updater.apply_updates()
+            results = {}
+            for vm, progress, result in upgrade_generator:
+                results[vm] = result
+                self.progress_signal.emit(progress)
+
         # reboot vms
         Updater.shutdown_and_start_vms()
 

launcher/tests/test_updater.py

@@ -13,6 +13,7 @@
 updater = SourceFileLoader("Updater", path_to_script).load_module()
 from Updater import UpdateStatus  # noqa: E402
 from Updater import current_templates  # noqa: E402
+from Updater import current_vms  # noqa: E402
 
 temp_dir = TemporaryDirectory().name
 
@@ -56,7 +57,11 @@
 
 
 def test_updater_vms_present():
-    assert len(updater.current_templates) == 9
+    assert len(updater.current_vms) == 8
+
+
+def test_updater_templatevms_present():
+    assert len(updater.current_templates) == 4
 
 
 @mock.patch("Updater._write_updates_status_flag_to_disk")
@@ -303,7 +308,7 @@ def test_apply_updates_dom0_failure(mocked_info, mocked_error, mocked_call):
     mocked_error.assert_has_calls(error_log)
 
 
-@pytest.mark.parametrize("vm", current_templates.keys())
+@pytest.mark.parametrize("vm", current_templates)
 @mock.patch("subprocess.check_call", side_effect="0")
 @mock.patch("Updater.sdlog.error")
 @mock.patch("Updater.sdlog.info")
@@ -313,30 +318,18 @@ def test_apply_updates_vms(mocked_info, mocked_error, mocked_call, vm):
         assert result == UpdateStatus.UPDATES_OK
 
         mocked_call.assert_called_once_with(
-            [
-                "sudo",
-                "qubesctl",
-                "--skip-dom0",
-                "--targets",
-                current_templates[vm],
-                "state.sls",
-                "update.qubes-vm",
-            ]
+            ["sudo", "qubesctl", "--skip-dom0", "--targets", vm, "state.sls", "update.qubes-vm"]
         )
         assert not mocked_error.called
 
 
-@pytest.mark.parametrize("vm", current_templates.keys())
+@pytest.mark.parametrize("vm", current_templates)
 @mock.patch("subprocess.check_call", side_effect=subprocess.CalledProcessError(1, "check_call"))
 @mock.patch("Updater.sdlog.error")
 @mock.patch("Updater.sdlog.info")
 def test_apply_updates_vms_fails(mocked_info, mocked_error, mocked_call, vm):
     error_calls = [
-        call(
-            "An error has occurred updating {}. Please contact your administrator.".format(
-                current_templates[vm]
-            )
-        ),
+        call("An error has occurred updating {}. Please contact your administrator.".format(vm)),
         call("Command 'check_call' returned non-zero exit status 1."),
     ]
     result = updater._apply_updates_vm(vm)
@@ -423,7 +416,7 @@ def test_overall_update_status_reboot_not_done_previously(
     assert not mocked_error.called
 
 
-@pytest.mark.parametrize("vm", current_templates.keys())
+@pytest.mark.parametrize("vm", current_vms.keys())
 @mock.patch("subprocess.check_output")
 @mock.patch("Updater.sdlog.error")
 @mock.patch("Updater.sdlog.info")
@@ -435,7 +428,7 @@ def test_safely_shutdown(mocked_info, mocked_error, mocked_output, vm):
     assert not mocked_error.called
 
 
-@pytest.mark.parametrize("vm", current_templates.keys())
+@pytest.mark.parametrize("vm", current_vms.keys())
 @mock.patch(
     "subprocess.check_output", side_effect=["0", "0", "0"],
 )
@@ -452,7 +445,7 @@ def test_safely_start(mocked_info, mocked_error, mocked_output, vm):
     assert not mocked_error.called
 
 
-@pytest.mark.parametrize("vm", current_templates.keys())
+@pytest.mark.parametrize("vm", current_vms.keys())
 @mock.patch(
     "subprocess.check_output", side_effect=subprocess.CalledProcessError(1, "check_output"),
 )
@@ -468,7 +461,7 @@ def test_safely_start_fails(mocked_info, mocked_error, mocked_output, vm):
     mocked_error.assert_has_calls(call_list)
 
 
-@pytest.mark.parametrize("vm", current_templates.keys())
+@pytest.mark.parametrize("vm", current_vms.keys())
 @mock.patch(
     "subprocess.check_output", side_effect=subprocess.CalledProcessError(1, "check_output"),
 )
@@ -508,13 +501,9 @@ def test_shutdown_and_start_vms(
     ]
     template_vm_calls = [
         call("fedora-31"),
-        call("sd-viewer-buster-template"),
-        call("sd-app-buster-template"),
-        call("sd-log-buster-template"),
-        call("sd-devices-buster-template"),
-        call("sd-proxy-buster-template"),
+        call("sd-large-buster-template"),
+        call("sd-small-buster-template"),
         call("whonix-gw-15"),
-        call("securedrop-workstation-buster"),
     ]
     app_vm_calls = [
         call("sd-app"),
@@ -560,13 +549,9 @@ def test_shutdown_and_start_vms_sysvm_fail(
     ]
     template_vm_calls = [
         call("fedora-31"),
-        call("sd-viewer-buster-template"),
-        call("sd-app-buster-template"),
-        call("sd-log-buster-template"),
-        call("sd-devices-buster-template"),
-        call("sd-proxy-buster-template"),
+        call("sd-large-buster-template"),
+        call("sd-small-buster-template"),
         call("whonix-gw-15"),
-        call("securedrop-workstation-buster"),
     ]
     error_calls = [
         call("Error while killing system VM: sys-firewall"),

rpm-build/SPECS/securedrop-workstation-dom0-config.spec

@@ -1,12 +1,12 @@
 Name:		securedrop-workstation-dom0-config
-Version:	0.4.0
-Release:	1%{?dist}
+Version:	0.5.0
+Release:	0.rc1.1%{?dist}
 Summary:	SecureDrop Workstation
 
 Group:		Library
 License:	GPLv3+
 URL:		https://github.com/freedomofpress/securedrop-workstation
-Source0:	securedrop-workstation-dom0-config-0.4.0.tar.gz
+Source0:	securedrop-workstation-dom0-config-0.5.0rc1.tar.gz
 
 BuildArch:      noarch
 BuildRequires:	python3-setuptools
@@ -28,7 +28,7 @@ configuration over time.
 %undefine py_auto_byte_compile
 
 %prep
-%setup -n securedrop-workstation-dom0-config-0.4.0
+%setup -n securedrop-workstation-dom0-config-0.5.0rc1
 
 %build
 %{__python3} setup.py build
@@ -62,6 +62,7 @@ install -m 644 dom0/*.conf %{buildroot}/srv/salt/
 install -m 755 dom0/remove-tags %{buildroot}/srv/salt/
 install -m 644 dom0/securedrop-login %{buildroot}/srv/salt/
 install -m 644 dom0/securedrop-launcher.desktop %{buildroot}/srv/salt/
+install -m 755 dom0/securedrop-check-migration %{buildroot}/srv/salt/
 install -m 755 dom0/securedrop-handle-upgrade %{buildroot}/srv/salt/
 install -m 755 dom0/update-xfce-settings %{buildroot}/srv/salt/
 install -m 755 scripts/sdw-admin.py %{buildroot}/%{_bindir}/sdw-admin
@@ -106,6 +107,10 @@ find /srv/salt -maxdepth 1 -type f -iname '*.top' \
     | xargs qubesctl top.enable > /dev/null
 
 %changelog
+* Tue Oct 27 2020 SecureDrop Team <[email protected]> - 0.5.0
+- Consolidates templates into small and large
+- Modifies updater UI to rerun full state if required
+
 * Tue Jul 07 2020 SecureDrop Team <[email protected]> - 0.4.0
 - Consolidates updates from two stages into one
 - Makes the updater UI more compact