Parallelize Py monorepo scripts (test, lint, etc)

.circleci/config.yml

@@ -213,7 +213,7 @@ jobs:
             - run:
                   command: |
                       cd python-packages
-                      ./cmd_pkgs_in_dep_order.py coverage run setup.py test
+                      ./parallel coverage run setup.py test
                       ./build_docs
             - save_cache:
                   key: coverage-python-contract-addresses-{{ .Environment.CIRCLE_SHA1 }}

python-packages/build_docs

@@ -1,13 +1,3 @@
-#!/usr/bin/env python
+#!/usr/bin/env bash
 
-"""Script to run linters against local copy of all components."""
-
-from os import path
-import subprocess
-
-subprocess.check_call(
-    (
-        f"{path.join('.', 'cmd_pkgs_in_dep_order.py')}"
-        + f" {path.join('.', 'setup.py')} build_sphinx"
-    ).split()
-)
+./parallel ./setup.py build_sphinx

python-packages/install_editable

@@ -1,13 +1,3 @@
-#!/usr/bin/env python
+#!/usr/bin/env bash
 
-"""Script to install all packages in editable mode (pip install -e .)."""
-
-from os import path
-import subprocess
-
-# install all packages
-subprocess.check_call(
-    (
-        path.join(".", "cmd_pkgs_in_dep_order.py") + " pip install -e .[dev]"
-    ).split()
-)
+./parallel pip install -e .[dev]

python-packages/lint

@@ -1,13 +1,3 @@
-#!/usr/bin/env python
+#!/usr/bin/env bash
 
-"""Script to run linters against local copy of all components."""
-
-from os import path
-import subprocess
-
-subprocess.check_call(
-    (
-        f"{path.join('.', 'cmd_pkgs_in_dep_order.py')}"
-        + f" {path.join('.', 'setup.py')} lint"
-    ).split()
-)
+./parallel ./setup.py lint

python-packages/parallel

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+
+"""Run the given command in all packages in parallel.
+
+Handy for quick verification test runs, but annoying in that all of the output
+is interleaved.
+
+$ ./parallel ./setup.py lint
+
+This will `cd` into each package, run `./setup.py lint`, then `cd ..`, all in
+parallel, in a separate process for each package.  The number of processes is
+decided by ProcessPoolExecutor.  Replace "lint" with any of "test", "clean",
+"build_sphinx" (for docs), etc.
+
+Also consider:
+
+$ ./parallel pip install -e .[dev] # install all the packages in editable mode
+
+$ ./parallel pip uninstall $(basename $(pwd))
+
+>>>"""
+
+from concurrent.futures import ProcessPoolExecutor
+from os import chdir
+from subprocess import check_call
+from sys import argv
+
+PACKAGES = [
+    "contract_addresses",
+    "contract_artifacts",
+    "contract_wrappers",
+    "json_schemas",
+    "sra_client",
+    "order_utils",
+    "middlewares",
+]
+
+def run_cmd_on_package(package: str):
+    """cd to the package dir, ./setup.py lint, cd .."""
+    chdir(package)
+    check_call(f"{' '.join(argv[1:])}".split())
+    chdir("..")
+
+ProcessPoolExecutor().map(run_cmd_on_package, PACKAGES)

python-packages/pre_install

@@ -5,18 +5,21 @@
 from os import chdir, path
 import subprocess
 
+from concurrent.futures import ProcessPoolExecutor
+from os import chdir
+from subprocess import check_call
+from sys import argv
+
 PACKAGES = [
     "contract_wrappers",
     "contract_artifacts",
     "json_schemas",
 ]
 
-for package in PACKAGES:
-    print(f"Running command `pre_install` in package {package}")
+def run_cmd_on_package(package: str):
+    """cd to the package dir, ./setup.py pre_install, cd .."""
     chdir(package)
-    subprocess.check_call(
-        (
-            path.join(".", "setup.py") + " pre_install"
-        ).split()
-    )
+    check_call(f"{path.join('.', 'setup.py')} pre_install".split())
     chdir("..")
+
+ProcessPoolExecutor().map(run_cmd_on_package, PACKAGES)

python-packages/test

@@ -1,13 +1,3 @@
-#!/usr/bin/env python
+#!/usr/bin/env bash
 
-"""Script to run tests against local copy of all components."""
-
-from os import path
-import subprocess
-
-subprocess.check_call(
-    (
-        f"{path.join('.', 'cmd_pkgs_in_dep_order.py')}"
-        + f" {path.join('.', 'setup.py')} test"
-    ).split()
-)
+./parallel ./setup.py test