Add black formatter check

.circleci/config.yml

@@ -2,12 +2,19 @@
 version: 2
 jobs:
   lint:
-    machine: true
+    docker:
+      - image: circleci/python:3.7-buster
     steps:
       - checkout
-      - run: sudo apt-get update
-      - run: sudo apt-get install -qq make
-      - run: make flake8
+      - run:
+          name: Install dependencies and run linters
+          command: |
+            virtualenv .venv
+            source .venv/bin/activate
+            pip install --require-hashes -r dev-requirements.txt
+            sudo apt install file
+            make flake8
+            make black
   buildrpm:
     machine: true
     steps:
@@ -27,7 +34,7 @@ jobs:
             source .venv/bin/activate
             pip install --require-hashes -r test-requirements.txt
             sudo apt install lsof
-            make test && make bandit && make black
+            make test && make bandit
 
 workflows:
   version: 2

.flake8

@@ -1,3 +1,3 @@
 [flake8]
 ignore: W605
-max-line-length = 99
+max-line-length = 100

.git-blame-ignore-revs

@@ -0,0 +1 @@
+782a04e59ae10d40085b5d8807e255143e5d05c5

Makefile

@@ -13,6 +13,12 @@ all: ## Builds and provisions all VMs required for testing workstation
 	$(MAKE) prep-salt
 	./scripts/provision-all
 
+.PHONY: black
+black: ## Lints all Python files with black
+# Not requiring dom0 since linting requires extra packages,
+# available only in the developer environment, i.e. Work VM.
+	@./scripts/lint-all "black --check"
+
 dev: all ## Builds and provisions all VMs required for testing workstation
 
 prod: ## Configures a PRODUCTION install for pilot use
@@ -138,9 +144,7 @@ validate: assert-dom0 ## Checks for local requirements in dev env
 flake8: ## Lints all Python files with flake8
 # Not requiring dom0 since linting requires extra packages,
 # available only in the developer environment, i.e. Work VM.
-	@docker run -v $(PWD):/code -w /code --name sdw_flake8 --rm \
-		--entrypoint /code/scripts/flake8-linting \
-		python:3.5.7-slim-stretch
+	@./scripts/lint-all "flake8"
 
 prep-dom0: prep-salt # Copies dom0 config files
 	sudo qubesctl --show-output --targets dom0 state.highstate
@@ -152,6 +156,11 @@ destroy-all: ## Destroys all VMs managed by Workstation salt config
 update-pip-requirements: ## Updates all Python requirements files via pip-compile.
 	pip-compile --generate-hashes --output-file requirements.txt requirements.in
 
+.PHONY: venv
+venv:  ## Provision and activate a Python 3 virtualenv for development.
+	python3 -m venv .venv
+	.venv/bin/pip install --require-hashes -r dev-requirements.txt
+
 # Explanation of the below shell command should it ever break.
 # 1. Set the field separator to ": ##" to parse lines for make targets.
 # 2. Check for second field matching, skip otherwise.

dev-requirements.in

@@ -0,0 +1,2 @@
+black
+flake8

dev-requirements.txt

@@ -0,0 +1,100 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=dev-requirements.txt dev-requirements.in
+#
+appdirs==1.4.4 \
+    --hash=sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41 \
+    --hash=sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128 \
+    # via black
+attrs==19.3.0 \
+    --hash=sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c \
+    --hash=sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72 \
+    # via black
+black==19.10b0 \
+    --hash=sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b \
+    --hash=sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539 \
+    # via -r dev-requirements.in
+click==7.1.2 \
+    --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \
+    --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc \
+    # via black
+flake8==3.8.3 \
+    --hash=sha256:15e351d19611c887e482fb960eae4d44845013cc142d42896e9862f775d8cf5c \
+    --hash=sha256:f04b9fcbac03b0a3e58c0ab3a0ecc462e023a9faf046d57794184028123aa208 \
+    # via -r dev-requirements.in
+importlib-metadata==1.6.1 \
+    --hash=sha256:0505dd08068cfec00f53a74a0ad927676d7757da81b7436a6eefe4c7cf75c545 \
+    --hash=sha256:15ec6c0fd909e893e3a08b3a7c76ecb149122fb14b7efe1199ddd4c7c57ea958 \
+    # via flake8
+mccabe==0.6.1 \
+    --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \
+    --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f \
+    # via flake8
+pathspec==0.8.0 \
+    --hash=sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0 \
+    --hash=sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061 \
+    # via black
+pycodestyle==2.6.0 \
+    --hash=sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367 \
+    --hash=sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e \
+    # via flake8
+pyflakes==2.2.0 \
+    --hash=sha256:0d94e0e05a19e57a99444b6ddcf9a6eb2e5c68d3ca1e98e90707af8152c90a92 \
+    --hash=sha256:35b2d75ee967ea93b55750aa9edbbf72813e06a66ba54438df2cfac9e3c27fc8 \
+    # via flake8
+regex==2020.5.14 \
+    --hash=sha256:1386e75c9d1574f6aa2e4eb5355374c8e55f9aac97e224a8a5a6abded0f9c927 \
+    --hash=sha256:27ff7325b297fb6e5ebb70d10437592433601c423f5acf86e5bc1ee2919b9561 \
+    --hash=sha256:329ba35d711e3428db6b45a53b1b13a0a8ba07cbbcf10bbed291a7da45f106c3 \
+    --hash=sha256:3a9394197664e35566242686d84dfd264c07b20f93514e2e09d3c2b3ffdf78fe \
+    --hash=sha256:51f17abbe973c7673a61863516bdc9c0ef467407a940f39501e786a07406699c \
+    --hash=sha256:579ea215c81d18da550b62ff97ee187b99f1b135fd894a13451e00986a080cad \
+    --hash=sha256:70c14743320a68c5dac7fc5a0f685be63bc2024b062fe2aaccc4acc3d01b14a1 \
+    --hash=sha256:7e61be8a2900897803c293247ef87366d5df86bf701083b6c43119c7c6c99108 \
+    --hash=sha256:8044d1c085d49673aadb3d7dc20ef5cb5b030c7a4fa253a593dda2eab3059929 \
+    --hash=sha256:89d76ce33d3266173f5be80bd4efcbd5196cafc34100fdab814f9b228dee0fa4 \
+    --hash=sha256:99568f00f7bf820c620f01721485cad230f3fb28f57d8fbf4a7967ec2e446994 \
+    --hash=sha256:a7c37f048ec3920783abab99f8f4036561a174f1314302ccfa4e9ad31cb00eb4 \
+    --hash=sha256:c2062c7d470751b648f1cacc3f54460aebfc261285f14bc6da49c6943bd48bdd \
+    --hash=sha256:c9bce6e006fbe771a02bda468ec40ffccbf954803b470a0345ad39c603402577 \
+    --hash=sha256:ce367d21f33e23a84fb83a641b3834dd7dd8e9318ad8ff677fbfae5915a239f7 \
+    --hash=sha256:ce450ffbfec93821ab1fea94779a8440e10cf63819be6e176eb1973a6017aff5 \
+    --hash=sha256:ce5cc53aa9fbbf6712e92c7cf268274eaff30f6bd12a0754e8133d85a8fb0f5f \
+    --hash=sha256:d466967ac8e45244b9dfe302bbe5e3337f8dc4dec8d7d10f5e950d83b140d33a \
+    --hash=sha256:d881c2e657c51d89f02ae4c21d9adbef76b8325fe4d5cf0e9ad62f850f3a98fd \
+    --hash=sha256:e565569fc28e3ba3e475ec344d87ed3cd8ba2d575335359749298a0899fe122e \
+    --hash=sha256:ea55b80eb0d1c3f1d8d784264a6764f931e172480a2f1868f2536444c5f01e01 \
+    # via black
+toml==0.10.1 \
+    --hash=sha256:926b612be1e5ce0634a2ca03470f95169cf16f939018233a670519cb4ac58b0f \
+    --hash=sha256:bda89d5935c2eac546d648028b9901107a595863cb36bae0c73ac804a9b4ce88 \
+    # via black
+typed-ast==1.4.1 \
+    --hash=sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355 \
+    --hash=sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919 \
+    --hash=sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa \
+    --hash=sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652 \
+    --hash=sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75 \
+    --hash=sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01 \
+    --hash=sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d \
+    --hash=sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1 \
+    --hash=sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907 \
+    --hash=sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c \
+    --hash=sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3 \
+    --hash=sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b \
+    --hash=sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614 \
+    --hash=sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb \
+    --hash=sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b \
+    --hash=sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41 \
+    --hash=sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6 \
+    --hash=sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34 \
+    --hash=sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe \
+    --hash=sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4 \
+    --hash=sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7 \
+    # via black
+zipp==3.1.0 \
+    --hash=sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b \
+    --hash=sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96 \
+    # via importlib-metadata

launcher/Makefile

@@ -11,7 +11,7 @@ test:
 	pytest --cov-report term-missing --cov=sdw_notify --cov=sdw_updater_gui/ --cov=sdw_util -v tests/
 
 black: ## Runs the black code formatter on the launcher code
-	black --check .
+	black --check --line-length=100 .
 
 # Explanation of the below shell command should it ever break.
 # 1. Set the field separator to ": ##" to parse lines for make targets.

launcher/pyproject.toml

@@ -0,0 +1,2 @@
+[tool.black]
+line-length = 100

launcher/sdw_notify/Notify.py

@@ -60,9 +60,7 @@ def is_update_check_necessary():
             sdlog.error(
                 "Data in {} not in the expected format. "
                 "Expecting a timestamp in format '{}'. "
-                "Showing security warning.".format(
-                    LAST_UPDATED_FILE, LAST_UPDATED_FORMAT
-                )
+                "Showing security warning.".format(LAST_UPDATED_FILE, LAST_UPDATED_FORMAT)
             )
             return True
 
@@ -76,9 +74,7 @@ def is_update_check_necessary():
     if not last_updated_file_exists:
         sdlog.warning(
             "Timestamp file '{}' does not exist. "
-            "Updater may never have run. Showing security warning.".format(
-                LAST_UPDATED_FILE
-            )
+            "Updater may never have run. Showing security warning.".format(LAST_UPDATED_FILE)
         )
         return True
     else:
@@ -113,9 +109,7 @@ def is_update_check_necessary():
             sdlog.info(
                 "Last successful update ({0:.1f} hours ago) "
                 "is below the warning threshold ({1:.1f} hours). "
-                "Exiting without warning.".format(
-                    updated_hours_ago, warning_threshold_hours
-                )
+                "Exiting without warning.".format(updated_hours_ago, warning_threshold_hours)
             )
             return False
 

launcher/sdw_updater_gui/Updater.py

@@ -54,9 +54,7 @@ def check_all_updates():
 
     for progress_current, vm in enumerate(current_templates.keys()):
         # yield the progress percentage for UI updates
-        progress_percentage = int(
-            ((progress_current + 1) / len(current_templates.keys())) * 100
-        )
+        progress_percentage = int(((progress_current + 1) / len(current_templates.keys())) * 100)
         update_results = check_updates(vm)
         yield vm, progress_percentage, update_results
 
@@ -129,17 +127,11 @@ def _check_updates_debian(vm):
         sdlog.info("Checking for updates {}:{}".format(vm, current_templates[vm]))
         subprocess.check_call(["qvm-run", current_templates[vm], "sudo apt update"])
         subprocess.check_call(
-            [
-                "qvm-run",
-                current_templates[vm],
-                "[[ $(apt list --upgradable | wc -l) -eq 1 ]]",
-            ]
+            ["qvm-run", current_templates[vm], "[[ $(apt list --upgradable | wc -l) -eq 1 ]]"]
         )
     except subprocess.CalledProcessError as e:
         sdlog.error(
-            "Updates required for {} or cannot check for updates".format(
-                current_templates[vm]
-            )
+            "Updates required for {} or cannot check for updates".format(current_templates[vm])
         )
         sdlog.error(str(e))
         updates_required = True
@@ -164,9 +156,7 @@ def _apply_updates_dom0():
     try:
         subprocess.check_call(["sudo", "qubes-dom0-update", "-y"])
     except subprocess.CalledProcessError as e:
-        sdlog.error(
-            "An error has occurred updating dom0. Please contact your administrator."
-        )
+        sdlog.error("An error has occurred updating dom0. Please contact your administrator.")
         sdlog.error(str(e))
         return UpdateStatus.UPDATES_FAILED
     sdlog.info("dom0 update successful")
@@ -247,11 +237,7 @@ def _write_updates_status_flag_to_disk(status):
     try:
         sdlog.info("Setting update flag to {} in sd-app".format(status.value))
         subprocess.check_call(
-            [
-                "qvm-run",
-                "sd-app",
-                "echo '{}' > {}".format(status.value, flag_file_path_sd_app),
-            ]
+            ["qvm-run", "sd-app", "echo '{}' > {}".format(status.value, flag_file_path_sd_app)]
         )
     except subprocess.CalledProcessError as e:
         sdlog.error("Error writing update status flag to sd-app")
@@ -286,9 +272,7 @@ def last_required_reboot_performed():
         return True
 
     if int(flag_contents["status"]) == int(UpdateStatus.REBOOT_REQUIRED.value):
-        reboot_time = datetime.strptime(
-            flag_contents["last_status_update"], DATE_FORMAT
-        )
+        reboot_time = datetime.strptime(flag_contents["last_status_update"], DATE_FORMAT)
         boot_time = datetime.now() - _get_uptime()
 
         # The session was started *before* the reboot was requested by
@@ -315,9 +299,7 @@ def _get_uptime():
     uptime_minutes = (uptime % 3600) // 60
     uptime_seconds = uptime % 60
 
-    delta = timedelta(
-        hours=uptime_hours, minutes=uptime_minutes, seconds=uptime_seconds
-    )
+    delta = timedelta(hours=uptime_hours, minutes=uptime_minutes, seconds=uptime_seconds)
 
     return delta
 
@@ -471,9 +453,7 @@ def _safely_start_vm(vm):
             ["qvm-ls", "--running", "--raw-list"], stderr=subprocess.PIPE
         )
         sdlog.info("VMs running before start of {}: {}".format(vm, running_vms))
-        subprocess.check_output(
-            ["qvm-start", "--skip-if-running", vm], stderr=subprocess.PIPE
-        )
+        subprocess.check_output(["qvm-start", "--skip-if-running", vm], stderr=subprocess.PIPE)
     except subprocess.CalledProcessError as e:
         sdlog.error("Error while starting {}".format(vm))
         sdlog.error(str(e))
@@ -493,9 +473,7 @@ def should_launch_updater(interval):
                 return False
             elif status["status"] == UpdateStatus.REBOOT_REQUIRED.value:
                 if last_required_reboot_performed():
-                    sdlog.info(
-                        "Required reboot performed, updating status and launching client."
-                    )
+                    sdlog.info("Required reboot performed, updating status and launching client.")
                     _write_updates_status_flag_to_disk(UpdateStatus.UPDATES_OK)
                     return False
                 else: