From e2d932fb1118b9c05175b95d073522cd2ae04794 Mon Sep 17 00:00:00 2001
From: "F. Eugene Aumson" <feuGeneA@users.noreply.github.com>
Date: Fri, 19 Jul 2019 23:20:15 -0400
Subject: [PATCH] Parallelize Py monorepo scripts (test, lint, etc)

---
 .circleci/config.yml             |  2 +-
 python-packages/build_docs       | 14 ++--------
 python-packages/install_editable | 14 ++--------
 python-packages/lint             | 14 ++--------
 python-packages/parallel         | 44 ++++++++++++++++++++++++++++++++
 python-packages/pre_install      | 17 +++++++-----
 python-packages/test             | 14 ++--------
 7 files changed, 63 insertions(+), 56 deletions(-)
 create mode 100755 python-packages/parallel

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 2192b0928d..fd4c5fed92 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -213,7 +213,7 @@ jobs:
             - run:
                   command: |
                       cd python-packages
-                      ./cmd_pkgs_in_dep_order.py coverage run setup.py test
+                      ./parallel coverage run setup.py test
                       ./build_docs
             - save_cache:
                   key: coverage-python-contract-addresses-{{ .Environment.CIRCLE_SHA1 }}
diff --git a/python-packages/build_docs b/python-packages/build_docs
index e2de496636..18271a5270 100755
--- a/python-packages/build_docs
+++ b/python-packages/build_docs
@@ -1,13 +1,3 @@
-#!/usr/bin/env python
+#!/usr/bin/env bash
 
-"""Script to run linters against local copy of all components."""
-
-from os import path
-import subprocess
-
-subprocess.check_call(
-    (
-        f"{path.join('.', 'cmd_pkgs_in_dep_order.py')}"
-        + f" {path.join('.', 'setup.py')} build_sphinx"
-    ).split()
-)
+./parallel ./setup.py build_sphinx
diff --git a/python-packages/install_editable b/python-packages/install_editable
index 6ac68b289c..d044c0e7c3 100755
--- a/python-packages/install_editable
+++ b/python-packages/install_editable
@@ -1,13 +1,3 @@
-#!/usr/bin/env python
+#!/usr/bin/env bash
 
-"""Script to install all packages in editable mode (pip install -e .)."""
-
-from os import path
-import subprocess
-
-# install all packages
-subprocess.check_call(
-    (
-        path.join(".", "cmd_pkgs_in_dep_order.py") + " pip install -e .[dev]"
-    ).split()
-)
+./parallel pip install -e .[dev]
diff --git a/python-packages/lint b/python-packages/lint
index 0974f273c6..e9d12a296b 100755
--- a/python-packages/lint
+++ b/python-packages/lint
@@ -1,13 +1,3 @@
-#!/usr/bin/env python
+#!/usr/bin/env bash
 
-"""Script to run linters against local copy of all components."""
-
-from os import path
-import subprocess
-
-subprocess.check_call(
-    (
-        f"{path.join('.', 'cmd_pkgs_in_dep_order.py')}"
-        + f" {path.join('.', 'setup.py')} lint"
-    ).split()
-)
+./parallel ./setup.py lint
diff --git a/python-packages/parallel b/python-packages/parallel
new file mode 100755
index 0000000000..12f385baaf
--- /dev/null
+++ b/python-packages/parallel
@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+
+"""Run the given command in all packages in parallel.
+
+Handy for quick verification test runs, but annoying in that all of the output
+is interleaved.
+
+$ ./parallel ./setup.py lint
+
+This will `cd` into each package, run `./setup.py lint`, then `cd ..`, all in
+parallel, in a separate process for each package.  The number of processes is
+decided by ProcessPoolExecutor.  Replace "lint" with any of "test", "clean",
+"build_sphinx" (for docs), etc.
+
+Also consider:
+
+$ ./parallel pip install -e .[dev] # install all the packages in editable mode
+
+$ ./parallel pip uninstall $(basename $(pwd))
+
+>>>"""
+
+from concurrent.futures import ProcessPoolExecutor
+from os import chdir
+from subprocess import check_call
+from sys import argv
+
+PACKAGES = [
+    "contract_addresses",
+    "contract_artifacts",
+    "contract_wrappers",
+    "json_schemas",
+    "sra_client",
+    "order_utils",
+    "middlewares",
+]
+
+def run_cmd_on_package(package: str):
+    """cd to the package dir, ./setup.py lint, cd .."""
+    chdir(package)
+    check_call(f"{' '.join(argv[1:])}".split())
+    chdir("..")
+
+ProcessPoolExecutor().map(run_cmd_on_package, PACKAGES)
diff --git a/python-packages/pre_install b/python-packages/pre_install
index 259db0e1b8..42e561ca88 100755
--- a/python-packages/pre_install
+++ b/python-packages/pre_install
@@ -5,18 +5,21 @@
 from os import chdir, path
 import subprocess
 
+from concurrent.futures import ProcessPoolExecutor
+from os import chdir
+from subprocess import check_call
+from sys import argv
+
 PACKAGES = [
     "contract_wrappers",
     "contract_artifacts",
     "json_schemas",
 ]
 
-for package in PACKAGES:
-    print(f"Running command `pre_install` in package {package}")
+def run_cmd_on_package(package: str):
+    """cd to the package dir, ./setup.py pre_install, cd .."""
     chdir(package)
-    subprocess.check_call(
-        (
-            path.join(".", "setup.py") + " pre_install"
-        ).split()
-    )
+    check_call(f"{path.join('.', 'setup.py')} pre_install".split())
     chdir("..")
+
+ProcessPoolExecutor().map(run_cmd_on_package, PACKAGES)
diff --git a/python-packages/test b/python-packages/test
index 5bf37cca3f..62c25dc1ec 100755
--- a/python-packages/test
+++ b/python-packages/test
@@ -1,13 +1,3 @@
-#!/usr/bin/env python
+#!/usr/bin/env bash
 
-"""Script to run tests against local copy of all components."""
-
-from os import path
-import subprocess
-
-subprocess.check_call(
-    (
-        f"{path.join('.', 'cmd_pkgs_in_dep_order.py')}"
-        + f" {path.join('.', 'setup.py')} test"
-    ).split()
-)
+./parallel ./setup.py test