Merge pull request #210 from freedomofpress/declare-salt-dependencies…

…-explicitly-in-provisioning

Declare salt dependencies explicitly in provisioning

Makefile

@@ -7,10 +7,8 @@ ifneq ($(HOST),dom0)
 endif
 
 ## Builds and provisions all VMs required for testing workstation
-all: assert-dom0 validate clean prep-dom0 \
-	sd-workstation-template \
-	sd-whonix sd-svs sd-gpg \
-	sd-proxy sd-svs-disp qubes-rpc
+all: assert-dom0 validate prep-salt
+	./scripts/provision-all
 
 clone: assert-dom0 ## Pulls the latest repo from work VM to dom0
 	@./scripts/clone-to-dom0
@@ -68,6 +66,11 @@ prep-salt: assert-dom0 ## Configures Salt layout for SD workstation VMs
 	@sudo cp -r sd-svs /srv/salt/sd
 	@sudo cp -r sd-workstation /srv/salt/sd
 	@sudo cp dom0/* /srv/salt/
+	sudo find /srv/salt -maxdepth 1 -type f -iname '*.top' \
+		| xargs -n1 basename \
+		| sed -e 's/\.top$$//g' \
+		| xargs sudo qubesctl top.enable
+
 #sudo cp -r sd-svs-disp /srv/salt/sd  # nothing there yet...
 
 remove-sd-whonix: assert-dom0 ## Destroys SD Whonix VM

README.md

@@ -40,7 +40,18 @@ Installing this project is involved. It requires an up-to-date Qubes 4.0 install
 
 Before trying to use this project, install [Qubes 4.0](https://www.qubes-os.org/downloads/) on your development machine. Accept the default VM configuration during the install process.
 
-After installing Qubes, update your debian-9 template VM to include the latest version of the `qubes-kernel-vm-support` package. Open a terminal in the debian-9 VM and run:
+After installing Qubes, you must update both dom0 and debian-9 template VM to include the latest version of the `qubes-kernel-vm-support` package.
+
+##### dom0
+
+Open a terminal in dom0 by clicking on the Qubes menu top-right of the screen and left-clicking on Terminal Emulator and run:
+
+```
+sudo qubes-dom0-update
+```
+
+##### debian-9
+Open a terminal in the debian-9 TemplateVM and run:
 
 ```
 sudo apt-get update
@@ -70,7 +81,7 @@ Qubes provisioning is handled by Salt on `dom0`, so this project must be copied
 After that initial manual step, the code in your development VM may be copied into place on `dom0` by setting the `SECUREDROP_DEV_VM` and `SECUREDROP_DEV_DIR` environmental variables to reflect the VM and directory to which you've cloned this repo, and running `make clone` from the root of the project on `dom0`:
 
 ```
-export SECUREDROP_DEV_DIR=sd-dev    # set to your dev VM
+export SECUREDROP_DEV_DIR=work    # set to your dev VM
 export SECUREDROP_DEV_DIR=/home/user/projects/securedrop-workstation    # set to your working directory
 make clone
 ```
@@ -85,10 +96,59 @@ make all
 
 The build process takes quite a while. You will be presented with a dialog asking how to connect to Tor: you should be able to select the default option and continue. You may wish to increase the scrollback in the dom0 terminal from 1000 (the default) to 100000, to ensure you can review any errors in the verbose output.
 
-**NOTE:** Due to [issue #202](https://github.com/freedomofpress/securedrop-workstation/issues/202), the installation may fail with disk quota errors. If this happens, reboot the entire workstation and run `make all` again.
+**NOTE:** Due to [issue #202](https://github.com/freedomofpress/securedrop-workstation/issues/202), the installation may fail with disk quota errors. If this happens, reboot the entire workstation and run `make all` again. The error will contain the following informating in your dom0 terminal:
+
+```
+qfile-agent : Fatal error: File copy: Disk quota exceeded; Last file: <...> (error type: Disk quota exceeded) '/usr/lib/qubes/qrexec-client-vm dom0 qubes.Receiveupdates /usr/lib/qubes/qfile-agent /var/lib/qubes/dom0-updates/packages/*.rpm' failed with exit code 1!
+```
 
 When the installation process completes, a number of new VMs will be available on your machine, all prefixed with `sd-`.
 
+Proceed to the following steps to clean up templates on workstation, which are necessary due to the inclusion of end-of-life templates in Qubes 4.0.
+
+##### Upgrading `sys-net`, `sys-usb` and `sys-firewall` to fedora-28
+
+Qubes 4.0 ships with end-of-life fedora-26 templates which are used by default for `sys-net`, `sys-firewall` and `sys-usb`. One needs to manually upgrade their `sys-net`, `sys-firewall` and `sys-usb` VMs to fedora-28. The following commands should be run for all VMs that use the fedora-26 template:
+
+```
+qvm-kill sys-net
+qvm-prefs sys-net template fedora-28
+qvm-start sys-net
+```
+
+Any other VM you may have created (e.g., `work`) should also be updated.
+
+You will also need to update the default dispvm template to fedora-28:
+
+```
+qvm-create --template fedora-28 --label red fedora-28-dvm
+qvm-prefs fedora-28-dvm template_for_dispvms True
+qubes-prefs default_dispvm fedora-28-dvm
+```
+
+You can then delete the end-of-life fedora-26 template in dom0 by running:
+
+```
+sudo dnf remove qubes-template-fedora-26
+```
+
+#### Upgrading `sys-whonix` and `whonix-ws` AppVMs to Whonix 14
+
+Qubes 4.0 also ships with end-of-life Whonix templates (`whonix-gw` and `whonix-ws`).`sys-whonix` is used by `sd-whonix` to fetch updates, and should be upgraded. You should destroy `whonix-gw` from the Qube Manager and re-provision a new `sys-whonix` AppVM with the potion **provides network** based on `whonix-gw-14`. You will need to delete the `whonix-ws-dvm` and `anon-whonix` VMs. You can then remove the end-of-life templates:
+
+```
+sudo dnf remove qubes-template-whonix-gw
+sudo dnf remove qubes-template-whonix-ws
+```
+
+Upon release, Qubes 4.0.1 will no longer ship fedora-26 or older Whonix templates, and the above steps will no longer be necessary.
+
+Finally, update all the templates and reboot the machine. Your workstation will then be ready for use. In dom0, run:
+
+```
+sudo securedrop-update
+```
+
 #### Using the SecureDrop Client
 
 

dom0/fpf-apt-test-repo.sls

@@ -14,5 +14,5 @@ configure apt-test apt repo:
     - name: "deb [arch=amd64] https://apt-test-qubes.freedom.press stretch main"
     - file: /etc/apt/sources.list.d/fpf-apt-test.list
     - key_url: "salt://sd/sd-workstation/apt-test-pubkey.asc"
-  require:
-    - pkg: install-python-apt-for-repo-config
+    - require:
+      - pkg: install-python-apt-for-repo-config

dom0/sd-dom0-files.sls

@@ -41,8 +41,8 @@ dom0-securedrop-icon:
     - user: root
     - group: root
     - mode: 644
-  require:
-    - file: dom0-securedrop-icons-directory
+    - require:
+      - file: dom0-securedrop-icons-directory
 
 # Install latest templates required for SDW VMs.
 dom0-install-fedora-28-template:
@@ -72,3 +72,32 @@ dom0-enabled-apparmor-on-whonix-ws-14-template:
       - kernelopts: "nopat apparmor=1 security=apparmor"
     - require:
       - pkg: dom0-install-whonix-14-templates
+
+dom0-create-opt-securedrop-directory:
+  file.directory:
+    - name: /opt/securedrop
+
+# Temporary workaround to bootstrap Salt support on target.
+dom0-whonix-gw-14-install-python-futures:
+  cmd.run:
+    - name: >
+        test -f /opt/securedrop/whonix-gw-14-python-futures ||
+        qvm-run -a whonix-gw-14
+        "python -c 'import concurrent.futures' ||
+        { sudo apt-get update && sudo apt-get install -qq python-futures ; }" &&
+        qvm-shutdown --wait whonix-gw-14 &&
+        touch /opt/securedrop/whonix-gw-14-python-futures
+    - require:
+      - file: dom0-create-opt-securedrop-directory
+
+dom0-whonix-ws-14-install-python-futures:
+  cmd.run:
+    - name: >
+        test -f /opt/securedrop/whonix-ws-14-python-futures ||
+        qvm-run -a whonix-ws-14
+        "python -c 'import concurrent.futures' ||
+        { sudo apt-get update && sudo apt-get install -qq python-futures ; }" &&
+        qvm-shutdown --wait whonix-ws-14 &&
+        touch /opt/securedrop/whonix-ws-14-python-futures
+    - require:
+      - file: dom0-create-opt-securedrop-directory

dom0/sd-gpg-files.sls

@@ -18,16 +18,27 @@ sd-gpg-increase-keyring-access-timeout:
     - content: |
         export QUBES_GPG_AUTOACCEPT=28800
 
+sd-gpg-create-keyring-directory:
+  file.directory:
+    - name: /home/user/.gnupg
+    - user: user
+    - group: user
+    - mode: 700
+
 sd-gpg-import-submission-key:
   file.managed:
-    - name: /tmp/sd-journalist.sec
+    - name: /home/user/.gnupg/sd-journalist.sec
     - source: salt://sd/sd-journalist.sec
     - user: user
     - group: user
-    - mode: 644
+    - mode: 600
     # Don't print private key to stdout
     - show_changes: False
+    - require:
+      - file: sd-gpg-create-keyring-directory
   cmd.run:
-    - name: sudo -u user gpg --import /tmp/sd-journalist.sec
+    - name: sudo -u user gpg --import /home/user/.gnupg/sd-journalist.sec
     - require:
       - file: sd-gpg-import-submission-key
+    - onchanges:
+      - file: sd-gpg-import-submission-key

dom0/sd-gpg.sls

@@ -9,6 +9,9 @@
 # This VM has no network configured.
 ##
 
+include:
+  - sd-workstation-template
+
 sd-gpg:
   qvm.vm:
     - name: sd-gpg
@@ -20,3 +23,5 @@ sd-gpg:
     - tags:
       - add:
         - sd-workstation
+    - require:
+      - sls: sd-workstation-template

dom0/sd-proxy-template-files.sls

@@ -34,6 +34,10 @@ sd-proxy-configure-mimetypes:
       - file: sd-proxy-configure-mimetypes
       - file: sd-proxy-do-not-open-here-desktop-file
       - file: sd-proxy-do-not-open-here-script
+    - onchanges:
+      - file: sd-proxy-do-not-open-here-script
+      - file: sd-proxy-do-not-open-here-desktop-file
+      - file: sd-proxy-configure-mimetypes
 
 # Depends on FPF-controlled apt repo, already present
 # in underlying "securedrop-workstation" base template.

dom0/sd-proxy.sls

@@ -39,20 +39,6 @@ sd-proxy:
       - pkg: qubes-template-whonix-ws-14
       - qvm: sd-whonix
       - qvm: sd-proxy-template
-      - cmd: sd-proxy-install-python-futures
-
-# Temporary workarounds for sd-proxy:
-#
-#   * python-futures required bootstrap Salt support
-#   * python-qt4 required for GUI window to inform people not to take actions in this VM
-#
-sd-proxy-install-python-futures:
-  cmd.run:
-    - name: >
-        qvm-run -a whonix-ws-14
-        "python -c 'import concurrent.futures' ||
-        { sudo apt-get update && sudo apt-get install -qq python-futures ; }" &&
-        qvm-shutdown --wait whonix-ws-14
 
 # Permit the SecureDrop Proxy to manage Client connections
 sd-proxy-dom0-securedrop.Proxy:

dom0/sd-svs-disp-files.sls

@@ -17,5 +17,5 @@ sd-svs-disp-install-mimetype-handler-package:
   pkg.installed:
     - pkgs:
       - securedrop-workstation-svs-disp
-  require:
-    - sls: fpf-apt-test-repo
+    - require:
+      - sls: fpf-apt-test-repo

dom0/sd-svs-disp.sls

@@ -11,12 +11,17 @@
 # This VM has no network configured.
 ##
 
+include:
+  - sd-workstation-template
+
 sd-svs-disp-template:
   qvm.vm:
     - name: sd-svs-disp-template
     - clone:
       - source: sd-workstation-template
       - label: green
+    - require:
+      - sls: sd-workstation-template
 
 sd-svs-disp:
   qvm.vm:
@@ -26,18 +31,10 @@ sd-svs-disp:
       - label: green
     - prefs:
       - netvm: ""
+      - template_for_dispvms: True
     - tags:
       - add:
         - sd-workstation
-
-# tell qubes this VM can be used as a disp VM template
-qvm-prefs sd-svs-disp template_for_dispvms True:
-  cmd.run
-
-# Allow dispvms based on this vm to open files in sd-svs.
-# (eg, "dispvms created from this VM can use the OpenInVM facility provided
-# by sd-svs"), but the "$dispvm:sd-svs" syntax can only be used as an
-# RPC policy *target*, not source. Tagged VMs can be used as a source.
-# This feels like a Qubes bug.
-qvm-tags sd-svs-disp add sd-svs-disp-vm:
-  cmd.run
+        - sd-svs-disp-vm
+    - require:
+      - qvm: sd-svs-disp-template

dom0/sd-svs-files.sls

@@ -8,35 +8,41 @@
 # Moves files into place on sd-svs
 #
 ##
+include:
+  - fpf-apt-test-repo
 
-/etc/profile.d/sd-svs-qubes-gpg-domain.sh:
+sd-svs-configure-gpg-domain:
   file.managed:
+    - name: /etc/profile.d/sd-svs-qubes-gpg-domain.sh
     - source: salt://sd/sd-svs/dot-profile
     - user: root
     - group: root
     - mode: 644
 
-/usr/share/applications/open-in-dvm.desktop:
+sd-svs-open-in-dvm-desktop-file:
   file.managed:
+    - name: /usr/share/applications/open-in-dvm.desktop
     - source: salt://sd/sd-svs/open-in-dvm.desktop
     - user: root
     - group: root
     - mode: 644
     - makedirs: True
 
-/usr/share/applications/mimeapps.list:
+sd-svs-configure-mimetypes:
   file.managed:
+    - name: /usr/share/applications/mimeapps.list
     - source: salt://sd/sd-svs/mimeapps.list
     - user: user
     - group: user
     - mode: 644
     - makedirs: True
-
-sudo update-mime-database /usr/share/mime:
-  cmd.run
-
-sudo update-desktop-database /usr/share/applications:
-  cmd.run
+  cmd.run:
+    - name: sudo update-desktop-database /usr/share/applications
+    - require:
+      - file: sd-svs-configure-mimetypes
+    - onchanges:
+      - file: sd-svs-configure-mimetypes
+      - file: sd-svs-open-in-dvm-desktop-file
 
 # FPF repo is setup in "securedrop-workstation" template
 install-securedrop-client-package:
@@ -45,5 +51,5 @@ install-securedrop-client-package:
       - python3-pyqt5
       - python3-pyqt5.qtsvg
       - securedrop-client
-  require:
-    - sls: fpf-apt-test-repo
+    - require:
+      - sls: fpf-apt-test-repo

dom0/sd-svs.sls

@@ -8,6 +8,8 @@
 # Installs 'sd-svs' AppVM, to persistently store SD data
 # This VM has no network configured.
 ##
+include:
+  - sd-workstation-template
 
 sd-svs-template:
   qvm.vm:
@@ -18,6 +20,8 @@ sd-svs-template:
     - tags:
       - add:
         - sd-workstation
+    - require:
+      - sls: sd-workstation-template
 
 sd-svs:
   qvm.vm:
@@ -30,14 +34,17 @@ sd-svs:
     - tags:
       - add:
         - sd-workstation
-  require:
-    - qvm: sd-svs-template
+    - require:
+      - qvm: sd-svs-template
 
 # Ensure the Qubes menu is populated with relevant app entries,
 # so that Nautilus/Files can be started via GUI interactions.
 sd-svs-template-sync-appmenus:
   cmd.run:
     - name: >
         qvm-start --skip-if-running sd-svs-template &&
-        qvm-sync-appmenus sd-svs-template &&
-        qvm-shutdown sd-svs-template
+        qvm-sync-appmenus sd-svs-template
+    - require:
+      - qvm: sd-svs-template
+    - onchanges:
+      - qvm: sd-svs-template