Parallelize tests run on GitHub Actions

In addition to being broken down by OS, tests are now also broken down by
major type (ide, api, unit) and browser $(chrome, firefox, edge, safari)
for ide and api tests.

Observations:
- can now more readily see what we are testing by the job breakdown in
GitHub Actions UI
- nice to have an overall shorter test run. I'm seeing ~8m to completion instead
of the former ~15m
- some windows tests might be a bit flaky, but good to know, so that we can
address, right?
- very nice to be able to rerun a single failed job, for example, if
`unit windows` has failed, I can rerun it alone via the GitHub UI.

The current tests only invoke chrome and firefox for ide testing, so
I've stuck with that. Api testing matches current browser defaults.

I'm liking keeping the brunt of the work out of yaml and in babashka
tasks. The yaml has a setup job which:
- brings down Clojure deps to be cached
- learns what tests should be run via
`bb script/test.clj matrix-for-ci --format=json`

The build task then:
- runs the tests found by the setup task. Test runs on ubuntu include
launching a virtual display (necessary for GitHub Actions)

Of note:
- New custom test reporting spits out the browser (for api and ide
tests) and the name of the test being run. This feedback is nice
especially for api tests which are long-running. It also helps to
validate we are testing what we think we are testing.
- Added `^:unit` metadata to unit test namespaces
- I noticed GitHub Actions was passing failing Windows tests and learned
how powershell needs a little help in propagating return codes.
- Because browser sets are different for api and ide tests, I left
the existing `ETAOIN_TEST_DRIVERS` env var for api tests and introduced
`ETAOIN_IDE_TEST_DRIVERS` for ide tests.
- Brought in some more scripting helpers from other projects, refactored
existing bb tools-versions script accordingly

Closes clj-commons#420

.clj-kondo/babashka/fs/config.edn

@@ -0,0 +1 @@
+{:lint-as {babashka.fs/with-temp-dir clojure.core/let}}

.clj-kondo/lread/status-line/config.edn

@@ -0,0 +1,2 @@
+{:lint-as {lread.status-line/line clojure.tools.logging/infof
+           lread.status-line/die clojure.tools.logging/infof}}

.clj-kondo/rewrite-clj/rewrite-clj/config.edn

@@ -0,0 +1,5 @@
+{:lint-as
+ {rewrite-clj.zip/subedit-> clojure.core/->
+  rewrite-clj.zip/subedit->> clojure.core/->>
+  rewrite-clj.zip/edit-> clojure.core/->
+  rewrite-clj.zip/edit->> clojure.core/->>}}

.github/workflows/test.yml

@@ -6,14 +6,57 @@ on:
   pull_request:
 
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+
+    outputs:
+      tests: ${{ steps.set-tests.outputs.tests }}
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Clojure deps cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.m2/repository
+          ~/.deps.clj
+          ~/.gitlibs
+        key: cljdeps-${{ hashFiles('project.clj, bb.edn') }}
+        restore-keys: ${{ runner.os }}-cljdeps-
+
+    - name: "Setup Java"
+      uses: actions/setup-java@v3
+      with:
+        distribution: 'temurin'
+        java-version: '11'
+
+    - name: Install Clojure Tools
+      uses: DeLaGuardo/[email protected]
+      with:
+        bb: 'latest'
+        lein: 'latest'
+
+    # This assumes downloaded deps are same for all OSes
+    - name: Bring down deps
+      run: |
+        lein deps
+        bb --version
+
+    - id: set-tests
+      name: Set test var for matrix
+      # run test.clj directly instead of via bb task to avoid generic task output
+      run: echo "::set-output name=tests::$(bb script/test.clj matrix-for-ci --format=json)"
+
   build:
+    needs: setup
     runs-on: ${{ matrix.os }}-latest
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu, macos, windows ]
+        include: ${{fromJSON(needs.setup.outputs.tests)}}
 
-    name: ${{ matrix.os }}
+    name: ${{ matrix.desc }}
 
     steps:
 
@@ -24,7 +67,9 @@ jobs:
       with:
         path: |
           ~/.m2/repository
-        key: ${{ runner.os }}-cljdeps-${{ hashFiles('project.clj') }}
+          ~/.deps.clj
+          ~/.gitlibs
+        key: cljdeps-${{ hashFiles('project.clj, bb.edn') }}
         restore-keys: ${{ runner.os }}-cljdeps-
 
     - name: "Setup Java"
@@ -42,19 +87,6 @@ jobs:
     - name: Tools versions
       run: bb tools-versions
 
-    - name: Bring down deps
-      run: lein deps
-
-    - name: Launch Virtual Display (ubuntu)
-      if: matrix.os == 'ubuntu'
-      run: Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
-
-    - name: Run Tests (unbuntu)
-      if: matrix.os == 'ubuntu'
-      run: lein test
-      env:
-        DISPLAY: :99.0
-
-    - name: Run Tests (macos, windows)
-      if: matrix.os != 'ubuntu'
-      run: lein test
+    - name: Run Tests
+      # To see all commands: bb test matrix-for-ci
+      run: ${{ matrix.cmd }}

bb.edn

@@ -1,3 +1,15 @@
 {:paths ["script"]
- :deps {doric/doric {:mvn/version "0.9.0"}}
- :tasks {tools-versions {:task tools-versions/report :doc "report on tools versions"}}}
+ :deps {doric/doric {:mvn/version "0.9.0"}
+        lread/status-line {:git/url "https://github.com/lread/status-line.git"
+                           :sha "35ed39645038e81b42cb15ed6753b8462e60a06d"}
+        dev.nubank/docopt {:mvn/version "0.6.1-fix7"}}
+ :tasks
+ {;; setup
+  :requires ([clojure.string :as string]
+             [lread.status-line :as status])
+  :enter (let [{:keys [name]} (current-task)] (status/line :head "TASK %s %s" name (string/join " " *command-line-args*)))
+  :leave (let [{:keys [name]} (current-task)] (status/line :detail "\nTASK %s done." name))
+
+  ;; commands
+  tools-versions {:task tools-versions/-main :doc "report on tools versions"}
+  test           {:task test/-main           :doc "run all or a subset of tests, use --help for args"}}}

project.clj

@@ -41,6 +41,8 @@
                  [org.clojure/tools.logging "0.3.1"]
                  [org.clojure/data.codec "0.1.0"]]
 
+  :test-selectors {:unit :unit}
+
   ;;
   ;; When running the tests as `lein test2junit`,
   ;; emit XUNIT test reports to enable CircleCI

script/helper/main.clj

@@ -0,0 +1,67 @@
+(ns helper.main
+  (:require  [clojure.string :as string]
+             [docopt.core :as docopt]
+             [lread.status-line :as status]))
+
+(defmacro when-invoked-as-script
+  "Runs `body` when clj was invoked from command line as a script."
+  [& body]
+  `(when (= *file* (System/getProperty "babashka.file"))
+     ~@body))
+
+(defn- args-usage-to-docopt-usage
+  "We specify
+
+     Valid args:
+       cmd1
+       cmd2
+       --opt1
+
+   Or:
+
+     Valid args: [options]
+
+   But docopt expects:
+
+     Usage:
+       foo cmd1
+       foo cmd2
+       foo -opt1
+
+   Or:
+
+     Usage: [options]
+
+   This little fn converts from our args usage to something docopt can understand"
+  [usage]
+  (let [re-arg-usage #"(?msi)^Valid args:.*?(?=\n\n)"]
+    (if-let [args-usage (re-find re-arg-usage usage)]
+      (let [[label-line & variant-lines] (-> args-usage string/split-lines)
+            docopt-usage-block (string/join "\n" (concat [(if (re-find #"(?i)Valid args:( +\S.*)" label-line)
+                                                            (string/replace label-line #"(?i)Valid args:( +\S.*)" "Usage: foo $1")
+                                                            "Usage:")]
+                                                         (map #(string/replace % #"^  " "  foo ") variant-lines)))
+            docopt-usage-block (str docopt-usage-block "\n")]
+        (string/replace usage re-arg-usage docopt-usage-block))
+      (throw (ex-info "Did not find expected 'Valid args:' in usage" {})))))
+
+(def default-arg-usage "Valid args: [--help]
+
+This command accepts no arguments.")
+
+(defn doc-arg-opt
+  "Args usage wrapper for docopt.
+
+   You'll need to specify --help in your arg-usage, but code to handle --help is provided here."
+  ([args]
+   (doc-arg-opt default-arg-usage args))
+  ([arg-usage args]
+   (let [opts (docopt/docopt (args-usage-to-docopt-usage arg-usage)
+                             args
+                             identity
+                             (fn usage-error [_docopt-usage] (status/die 1 arg-usage)))]
+     (if (get opts "--help")
+       (do
+         (status/line :detail arg-usage)
+         nil)
+       opts))))

script/helper/os.clj

@@ -0,0 +1,11 @@
+(ns helper.os
+  (:require [clojure.string :as string]))
+
+(defn get-os []
+  (let [os-name (string/lower-case (System/getProperty "os.name"))]
+    (condp re-find os-name
+      #"win" :win
+      #"mac" :mac
+      #"(nix|nux|aix)" :unix
+      #"sunos" :solaris
+      :unknown)))

script/helper/shell.clj

@@ -0,0 +1,33 @@
+(ns helper.shell
+  (:require [babashka.tasks :as tasks]
+            [clojure.pprint :as pprint]
+            [clojure.string :as string]
+            [helper.os :as os]
+            [lread.status-line :as status]))
+
+(def default-opts {:error-fn
+                   (fn die-on-error [{{:keys [exit cmd]} :proc}]
+                               (status/die exit
+                                           "shell exited with %d for: %s"
+                                           exit
+                                           (with-out-str (pprint/pprint cmd))))})
+
+(defn command
+  "Thin wrapper on babashka.tasks/shell that on error, prints status error message and exits.
+  Launches everything through powershell if on windows (maybe not a good general solution (?) but
+  ok for this project)."
+  [cmd & args]
+  (let [[opts cmd args] (if (map? cmd)
+                          [cmd (first args) (rest args)]
+                          [nil cmd args])
+        opts (merge opts default-opts)]
+    (if (= :win (os/get-os))
+      (let [full-cmd (if (seq args)
+                       ;; naive, but fine for our uses for now, adjust as necessary
+                       (str cmd " " (string/join " " args))
+                       cmd)]
+        (tasks/shell opts "powershell" "-command"
+                     ;; powershell -command does not automatically propagate exit code,
+                     ;; hence the secret exit sauce here
+                     (str full-cmd ";exit $LASTEXITCODE") ))
+      (apply tasks/shell opts cmd args))))

script/test.clj

@@ -0,0 +1,119 @@
+(ns test
+  (:require [babashka.fs :as fs]
+            [babashka.process :as process]
+            [cheshire.core :as json]
+            [clojure.string :as string]
+            [doric.core :as doric]
+            [helper.main :as main]
+            [helper.shell :as shell]
+            [lread.status-line :as status]))
+
+(defn- test-def [os id browser]
+  {:os os
+   :cmd (->> ["bb test" id
+              (when browser (str "--browser=" browser))
+              (when (= "ubuntu" os) "--launch-virtual-display")]
+             (remove nil?)
+             (string/join " "))
+   :desc (->> [id os browser]
+              (remove nil?)
+              (string/join " "))})
+
+(defn- github-actions-matrix []
+  (let [oses ["macos" "ubuntu" "windows"]
+        ide-browsers ["chrome" "firefox"]
+        api-browsers ["chrome" "firefox" "edge" "safari"]]
+    (->> (concat
+          (for [os oses]
+            (test-def os "unit" nil))
+          (for [os oses
+                browser ide-browsers]
+            (test-def os "ide" browser))
+          (for [os oses
+                browser api-browsers
+                :when (not (or (and (= "ubuntu" os) (some #{browser} ["edge" "safari"]))
+                               (and (= "windows" os) (= "safari" browser))))]
+            (test-def os "api" browser)))
+         (sort-by :desc)
+         (into []))))
+
+(defn- launch-xvfb []
+  (if (fs/which "Xvfb")
+    (process/process "Xvfb :99 -screen 0 1024x768x24" {:out (fs/file "/dev/null")
+                                                       :err (fs/file "/dev/null")})
+    (status/die 1 "Xvfb not found"))
+  (let [deadline (+ (System/currentTimeMillis) 10000)]
+    (loop []
+      (let [{:keys [exit]} (shell/command {:out (fs/file "/dev/null")
+                                           :err (fs/file "/dev/null")
+                                           :continue true}
+                                          "xdpyinfo -display :99")]
+        (if (zero? exit)
+          (status/line :detail "Xvfb process looks good.")
+          (if (> (System/currentTimeMillis) deadline)
+            (status/die 1 "Failed to get status from Xvfb process")
+            (do
+              (status/line :detail "Waiting for Xvfb process.")
+              (Thread/sleep 500)
+              (recur))))))))
+
+(def args-usage "Valid args:
+  (api|ide) [--browser=<edge|safari|firefox|chrome>]... [--launch-virtual-display]
+  (unit|all) [--launch-virtual-display]
+  matrix-for-ci [--format=json]
+  --help
+
+Commands:
+  unit           Run only unit tests
+  api            Run only api tests, optionally specifying browsers to override defaults
+  ide            Run only ide tests, optionally specifying browsers to override defaults
+  all            Run all tests using browser defaults
+  matrix-for-ci  Return text matrix for GitHub Actions
+
+Options:
+  --launch-virtual-display   Launch a virtual display for browsers (use on linux only)
+  --help                     Show this help
+
+Notes:
+- ide tests default to firefox and chrome only.
+- api tests default browsers based on OS on which they are run.
+- launching a virtual display is necessary for GitHub Actions but not so for CircleCI")
+
+(defn -main [& args]
+  (when-let [opts (main/doc-arg-opt args-usage args)]
+    (cond
+      (get opts "matrix-for-ci")
+      (if (= "json" (get opts "--format"))
+        (status/line :detail (-> (github-actions-matrix)
+                                 (json/generate-string #_{:pretty true})))
+        (status/line :detail (->> (github-actions-matrix)
+                                  (doric/table [:os :cmd :desc]))))
+
+      :else
+      (let [lein-args (cond
+                        (get opts "api") "test :only etaoin.api-test"
+                        (get opts "ide") "test :only etaoin.ide-test"
+                        (get opts "unit") "test :unit"
+                        :else "test")
+            browsers (->> (get opts "--browser") (keep identity))
+            env (cond-> {}
+                  (seq browsers)
+                  (assoc (if (get opts "api")
+                           "ETAOIN_TEST_DRIVERS"
+                           "ETAOIN_IDE_TEST_DRIVERS")
+                         (mapv keyword browsers))
+
+                  (get opts "--launch-virtual-display")
+                  (assoc "DISPLAY" ":99.0"))
+            shell-opts (if (seq env)
+                         {:extra-env env}
+                         {})]
+        (when (get opts "--launch-virtual-display")
+          (status/line :head "Launching virtual display")
+          (launch-xvfb))
+        (status/line :head "Running tests")
+        (shell/command shell-opts (str "lein " lein-args))))))
+
+(main/when-invoked-as-script
+ (apply -main *command-line-args*))
+