Merge pull request #425 from lread/lread-parallel-ci

Parallelize tests run on GitHub Actions

.clj-kondo/babashka/fs/config.edn

@@ -0,0 +1 @@
+{:lint-as {babashka.fs/with-temp-dir clojure.core/let}}

.clj-kondo/lread/status-line/config.edn

@@ -0,0 +1,2 @@
+{:lint-as {lread.status-line/line clojure.tools.logging/infof
+           lread.status-line/die clojure.tools.logging/infof}}

.clj-kondo/rewrite-clj/rewrite-clj/config.edn

@@ -0,0 +1,5 @@
+{:lint-as
+ {rewrite-clj.zip/subedit-> clojure.core/->
+  rewrite-clj.zip/subedit->> clojure.core/->>
+  rewrite-clj.zip/edit-> clojure.core/->
+  rewrite-clj.zip/edit->> clojure.core/->>}}

.github/workflows/test.yml

@@ -6,14 +6,57 @@ on:
   pull_request:
 
 jobs:
+  setup:
+    runs-on: ubuntu-latest
+
+    outputs:
+      tests: ${{ steps.set-tests.outputs.tests }}
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Clojure deps cache
+      uses: actions/cache@v3
+      with:
+        path: |
+          ~/.m2/repository
+          ~/.deps.clj
+          ~/.gitlibs
+        key: cljdeps-${{ hashFiles('project.clj, bb.edn') }}
+        restore-keys: ${{ runner.os }}-cljdeps-
+
+    - name: "Setup Java"
+      uses: actions/setup-java@v3
+      with:
+        distribution: 'temurin'
+        java-version: '11'
+
+    - name: Install Clojure Tools
+      uses: DeLaGuardo/[email protected]
+      with:
+        bb: 'latest'
+        lein: 'latest'
+
+    # This assumes downloaded deps are same for all OSes
+    - name: Bring down deps
+      run: |
+        lein deps
+        bb --version
+
+    - id: set-tests
+      name: Set test var for matrix
+      # run test.clj directly instead of via bb task to avoid generic task output
+      run: echo "::set-output name=tests::$(bb script/test.clj matrix-for-ci --format=json)"
+
   build:
+    needs: setup
     runs-on: ${{ matrix.os }}-latest
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu, macos, windows ]
+        include: ${{fromJSON(needs.setup.outputs.tests)}}
 
-    name: ${{ matrix.os }}
+    name: ${{ matrix.desc }}
 
     steps:
 
@@ -24,7 +67,9 @@ jobs:
       with:
         path: |
           ~/.m2/repository
-        key: ${{ runner.os }}-cljdeps-${{ hashFiles('project.clj') }}
+          ~/.deps.clj
+          ~/.gitlibs
+        key: cljdeps-${{ hashFiles('project.clj, bb.edn') }}
         restore-keys: ${{ runner.os }}-cljdeps-
 
     - name: "Setup Java"
@@ -42,19 +87,6 @@ jobs:
     - name: Tools versions
       run: bb tools-versions
 
-    - name: Bring down deps
-      run: lein deps
-
-    - name: Launch Virtual Display (ubuntu)
-      if: matrix.os == 'ubuntu'
-      run: Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
-
-    - name: Run Tests (unbuntu)
-      if: matrix.os == 'ubuntu'
-      run: lein test
-      env:
-        DISPLAY: :99.0
-
-    - name: Run Tests (macos, windows)
-      if: matrix.os != 'ubuntu'
-      run: lein test
+    - name: Run Tests
+      # To see all commands: bb test matrix-for-ci
+      run: ${{ matrix.cmd }}

bb.edn

@@ -1,3 +1,15 @@
 {:paths ["script"]
- :deps {doric/doric {:mvn/version "0.9.0"}}
- :tasks {tools-versions {:task tools-versions/report :doc "report on tools versions"}}}
+ :deps {doric/doric {:mvn/version "0.9.0"}
+        lread/status-line {:git/url "https://github.com/lread/status-line.git"
+                           :sha "35ed39645038e81b42cb15ed6753b8462e60a06d"}
+        dev.nubank/docopt {:mvn/version "0.6.1-fix7"}}
+ :tasks
+ {;; setup
+  :requires ([clojure.string :as string]
+             [lread.status-line :as status])
+  :enter (let [{:keys [name]} (current-task)] (status/line :head "TASK %s %s" name (string/join " " *command-line-args*)))
+  :leave (let [{:keys [name]} (current-task)] (status/line :detail "\nTASK %s done." name))
+
+  ;; commands
+  tools-versions {:task tools-versions/-main :doc "report on tools versions"}
+  test           {:task test/-main           :doc "run all or a subset of tests, use --help for args"}}}

project.clj

@@ -41,6 +41,8 @@
                  [org.clojure/tools.logging "0.3.1"]
                  [org.clojure/data.codec "0.1.0"]]
 
+  :test-selectors {:unit :unit}
+
   ;;
   ;; When running the tests as `lein test2junit`,
   ;; emit XUNIT test reports to enable CircleCI

script/helper/main.clj

@@ -0,0 +1,67 @@
+(ns helper.main
+  (:require  [clojure.string :as string]
+             [docopt.core :as docopt]
+             [lread.status-line :as status]))
+
+(defmacro when-invoked-as-script
+  "Runs `body` when clj was invoked from command line as a script."
+  [& body]
+  `(when (= *file* (System/getProperty "babashka.file"))
+     ~@body))
+
+(defn- args-usage-to-docopt-usage
+  "We specify
+
+     Valid args:
+       cmd1
+       cmd2
+       --opt1
+
+   Or:
+
+     Valid args: [options]
+
+   But docopt expects:
+
+     Usage:
+       foo cmd1
+       foo cmd2
+       foo -opt1
+
+   Or:
+
+     Usage: [options]
+
+   This little fn converts from our args usage to something docopt can understand"
+  [usage]
+  (let [re-arg-usage #"(?msi)^Valid args:.*?(?=\n\n)"]
+    (if-let [args-usage (re-find re-arg-usage usage)]
+      (let [[label-line & variant-lines] (-> args-usage string/split-lines)
+            docopt-usage-block (string/join "\n" (concat [(if (re-find #"(?i)Valid args:( +\S.*)" label-line)
+                                                            (string/replace label-line #"(?i)Valid args:( +\S.*)" "Usage: foo $1")
+                                                            "Usage:")]
+                                                         (map #(string/replace % #"^  " "  foo ") variant-lines)))
+            docopt-usage-block (str docopt-usage-block "\n")]
+        (string/replace usage re-arg-usage docopt-usage-block))
+      (throw (ex-info "Did not find expected 'Valid args:' in usage" {})))))
+
+(def default-arg-usage "Valid args: [--help]
+
+This command accepts no arguments.")
+
+(defn doc-arg-opt
+  "Args usage wrapper for docopt.
+
+   You'll need to specify --help in your arg-usage, but code to handle --help is provided here."
+  ([args]
+   (doc-arg-opt default-arg-usage args))
+  ([arg-usage args]
+   (let [opts (docopt/docopt (args-usage-to-docopt-usage arg-usage)
+                             args
+                             identity
+                             (fn usage-error [_docopt-usage] (status/die 1 arg-usage)))]
+     (if (get opts "--help")
+       (do
+         (status/line :detail arg-usage)
+         nil)
+       opts))))

script/helper/os.clj

@@ -0,0 +1,11 @@
+(ns helper.os
+  (:require [clojure.string :as string]))
+
+(defn get-os []
+  (let [os-name (string/lower-case (System/getProperty "os.name"))]
+    (condp re-find os-name
+      #"win" :win
+      #"mac" :mac
+      #"(nix|nux|aix)" :unix
+      #"sunos" :solaris
+      :unknown)))

script/helper/shell.clj

@@ -0,0 +1,33 @@
+(ns helper.shell
+  (:require [babashka.tasks :as tasks]
+            [clojure.pprint :as pprint]
+            [clojure.string :as string]
+            [helper.os :as os]
+            [lread.status-line :as status]))
+
+(def default-opts {:error-fn
+                   (fn die-on-error [{{:keys [exit cmd]} :proc}]
+                               (status/die exit
+                                           "shell exited with %d for: %s"
+                                           exit
+                                           (with-out-str (pprint/pprint cmd))))})
+
+(defn command
+  "Thin wrapper on babashka.tasks/shell that on error, prints status error message and exits.
+  Launches everything through powershell if on windows (maybe not a good general solution (?) but
+  ok for this project)."
+  [cmd & args]
+  (let [[opts cmd args] (if (map? cmd)
+                          [cmd (first args) (rest args)]
+                          [nil cmd args])
+        opts (merge opts default-opts)]
+    (if (= :win (os/get-os))
+      (let [full-cmd (if (seq args)
+                       ;; naive, but fine for our uses for now, adjust as necessary
+                       (str cmd " " (string/join " " args))
+                       cmd)]
+        (tasks/shell opts "powershell" "-command"
+                     ;; powershell -command does not automatically propagate exit code,
+                     ;; hence the secret exit sauce here
+                     (str full-cmd ";exit $LASTEXITCODE") ))
+      (apply tasks/shell opts cmd args))))

script/test.clj

@@ -0,0 +1,119 @@
+(ns test
+  (:require [babashka.fs :as fs]
+            [babashka.process :as process]
+            [cheshire.core :as json]
+            [clojure.string :as string]
+            [doric.core :as doric]
+            [helper.main :as main]
+            [helper.shell :as shell]
+            [lread.status-line :as status]))
+
+(defn- test-def [os id browser]
+  {:os os
+   :cmd (->> ["bb test" id
+              (when browser (str "--browser=" browser))
+              (when (= "ubuntu" os) "--launch-virtual-display")]
+             (remove nil?)
+             (string/join " "))
+   :desc (->> [id os browser]
+              (remove nil?)
+              (string/join " "))})
+
+(defn- github-actions-matrix []
+  (let [oses ["macos" "ubuntu" "windows"]
+        ide-browsers ["chrome" "firefox"]
+        api-browsers ["chrome" "firefox" "edge" "safari"]]
+    (->> (concat
+          (for [os oses]
+            (test-def os "unit" nil))
+          (for [os oses
+                browser ide-browsers]
+            (test-def os "ide" browser))
+          (for [os oses
+                browser api-browsers
+                :when (not (or (and (= "ubuntu" os) (some #{browser} ["edge" "safari"]))
+                               (and (= "windows" os) (= "safari" browser))))]
+            (test-def os "api" browser)))
+         (sort-by :desc)
+         (into []))))
+
+(defn- launch-xvfb []
+  (if (fs/which "Xvfb")
+    (process/process "Xvfb :99 -screen 0 1024x768x24" {:out (fs/file "/dev/null")
+                                                       :err (fs/file "/dev/null")})
+    (status/die 1 "Xvfb not found"))
+  (let [deadline (+ (System/currentTimeMillis) 10000)]
+    (loop []
+      (let [{:keys [exit]} (shell/command {:out (fs/file "/dev/null")
+                                           :err (fs/file "/dev/null")
+                                           :continue true}
+                                          "xdpyinfo -display :99")]
+        (if (zero? exit)
+          (status/line :detail "Xvfb process looks good.")
+          (if (> (System/currentTimeMillis) deadline)
+            (status/die 1 "Failed to get status from Xvfb process")
+            (do
+              (status/line :detail "Waiting for Xvfb process.")
+              (Thread/sleep 500)
+              (recur))))))))
+
+(def args-usage "Valid args:
+  (api|ide) [--browser=<edge|safari|firefox|chrome>]... [--launch-virtual-display]
+  (unit|all) [--launch-virtual-display]
+  matrix-for-ci [--format=json]
+  --help
+
+Commands:
+  unit           Run only unit tests
+  api            Run only api tests, optionally specifying browsers to override defaults
+  ide            Run only ide tests, optionally specifying browsers to override defaults
+  all            Run all tests using browser defaults
+  matrix-for-ci  Return text matrix for GitHub Actions
+
+Options:
+  --launch-virtual-display   Launch a virtual display for browsers (use on linux only)
+  --help                     Show this help
+
+Notes:
+- ide tests default to firefox and chrome only.
+- api tests default browsers based on OS on which they are run.
+- launching a virtual display is necessary for GitHub Actions but not so for CircleCI")
+
+(defn -main [& args]
+  (when-let [opts (main/doc-arg-opt args-usage args)]
+    (cond
+      (get opts "matrix-for-ci")
+      (if (= "json" (get opts "--format"))
+        (status/line :detail (-> (github-actions-matrix)
+                                 (json/generate-string #_{:pretty true})))
+        (status/line :detail (->> (github-actions-matrix)
+                                  (doric/table [:os :cmd :desc]))))
+
+      :else
+      (let [lein-args (cond
+                        (get opts "api") "test :only etaoin.api-test"
+                        (get opts "ide") "test :only etaoin.ide-test"
+                        (get opts "unit") "test :unit"
+                        :else "test")
+            browsers (->> (get opts "--browser") (keep identity))
+            env (cond-> {}
+                  (seq browsers)
+                  (assoc (if (get opts "api")
+                           "ETAOIN_TEST_DRIVERS"
+                           "ETAOIN_IDE_TEST_DRIVERS")
+                         (mapv keyword browsers))
+
+                  (get opts "--launch-virtual-display")
+                  (assoc "DISPLAY" ":99.0"))
+            shell-opts (if (seq env)
+                         {:extra-env env}
+                         {})]
+        (when (get opts "--launch-virtual-display")
+          (status/line :head "Launching virtual display")
+          (launch-xvfb))
+        (status/line :head "Running tests")
+        (shell/command shell-opts (str "lein " lein-args))))))
+
+(main/when-invoked-as-script
+ (apply -main *command-line-args*))
+