From 5d5f9ecdbce36b698cc30859f276d7306ade8b80 Mon Sep 17 00:00:00 2001 From: lread Date: Mon, 23 May 2022 13:50:42 -0400 Subject: [PATCH] road to bb: http-client-lite & tests When running babashka, we now use the compatible http-client-lite. (Thanks borkdude your etaoin pod work was a very useful reference!) Reader conditionals select the existing http-client when running on from the JVM. Reader conditionals mean .cljc, which often means Clojure and ClojureScript, but in our case it means JVM Clojure and Babashka Clojure. Update configs for cljdoc and clj-kondo to let them know that this is not a ClojureScript project. Facility to test our bb implementation added via a new --platform option on our `bb test` task. I'm not sure of the best way to run babashka tests, but my first stab generates a runner.clj with appropriate namespaces and classpath then runs that. Etaoin does have some debug logging, so Babashka's log level is adjusted from the default of debug to info in the generated test runner. Finally adjusted our CI matrix to include the bb platform. Contributes to #380 --- .clj-kondo/config.edn | 2 +- .github/workflows/test.yml | 2 +- Makefile | 5 - deps.edn | 8 +- doc/cljdoc.edn | 4 + script/bb_test_runner.clj | 60 ++++++++++++ script/bb_test_runner_template.clj | 15 +++ script/test.clj | 121 ++++++++++++++++--------- src/etaoin/{client.clj => client.cljc} | 34 ++++--- 9 files changed, 186 insertions(+), 65 deletions(-) create mode 100644 doc/cljdoc.edn create mode 100644 script/bb_test_runner.clj create mode 100644 script/bb_test_runner_template.clj rename src/etaoin/{client.clj => client.cljc} (72%) diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 51b99827..e0993099 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -1,5 +1,5 @@ {:config-paths ^:replace [] ;; don't adopt any user preferences - + :cljc {:features [:clj]} :hooks ;; for now we'll use the simple macroexpand, can move to hooks for finer grained errors later {:macroexpand diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a1429777..fcd22f99 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: Test on: push: - branches: ['master'] + branches: ['lread-*'] pull_request: jobs: diff --git a/Makefile b/Makefile index 5e43bbab..471a45ce 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,3 @@ -;; TODO: lread move to bb tasks - .PHONY: kill kill: pkill chromedriver || true @@ -9,7 +7,6 @@ kill: IMAGE := etaoin -;; TODO: lread: have never tried, test, fix if necessary .PHONY: docker-build docker-build: docker build --no-cache -t ${IMAGE}:latest . @@ -20,7 +17,6 @@ check-host: $(error The HOST variable is not set, please do `export HOST=$$HOST` first) endif -;; TODO: lread: have never tried, test, fix if necessary # works only on mac + quartz .PHONY: docker-test-display docker-test-display: check-host @@ -32,7 +28,6 @@ docker-test-display: check-host bb test all || \ xhost - -;; TODO: lread: have never tried, test, fix if necessary .PHONY: docker-test docker-test: docker run --rm \ diff --git a/deps.edn b/deps.edn index c92fdd5f..2cec83b4 100644 --- a/deps.edn +++ b/deps.edn @@ -1,7 +1,8 @@ {:paths ["src"] :deps {org.clojure/clojure {:mvn/version "1.9.0"} ;; min clojure version babashka/fs {:mvn/version "0.1.6"} - clj-http/clj-http {:mvn/version "3.10.1"} + clj-http/clj-http {:mvn/version "3.10.1"} ;; for jvm use + org.clj-commons/clj-http-lite {:mvn/version "0.4.392"} ;; for babashka use cheshire/cheshire {:mvn/version "5.9.0"} org.clojure/tools.cli {:mvn/version "1.0.194"} org.clojure/tools.logging {:mvn/version "0.3.1"}} @@ -11,4 +12,7 @@ :test {:extra-paths ["test" "env/test/resources"] :extra-deps {io.github.cognitect-labs/test-runner {:git/tag "v0.5.1" :git/sha "dfb30dd"} ch.qos.logback/logback-classic {:mvn/version "1.3.0-alpha16"}} - :main-opts ["-m" "cognitect.test-runner"]}}} + :main-opts ["-m" "cognitect.test-runner"]} + ;; for babashka testing, needed for eatoin.ide + :bb-spec {:extra-deps {org.babashka/spec.alpha {:git/url "https://github.com/babashka/spec.alpha" + :sha "644a7fc216e43d5da87b07471b0f87d874107d1a"}}}}} diff --git a/doc/cljdoc.edn b/doc/cljdoc.edn new file mode 100644 index 00000000..ac80b051 --- /dev/null +++ b/doc/cljdoc.edn @@ -0,0 +1,4 @@ +{:cljdoc.doc/tree + [["Readme" {:file "README.adoc"}] + ["Changelog" {:file "CHANGELOG.adoc"}]] + :cljdoc/languages ["clj"]} diff --git a/script/bb_test_runner.clj b/script/bb_test_runner.clj new file mode 100644 index 00000000..9d8eced8 --- /dev/null +++ b/script/bb_test_runner.clj @@ -0,0 +1,60 @@ +(ns bb-test-runner + (:require [babashka.fs :as fs] + [clojure.string :as string] + [babashka.tasks :as tasks] + [helper.main :as main])) + +(def id->nses {:ide ['etaoin.ide-test] + :api ['etaoin.api-test] + :unit (->> "test/etaoin/unit" + fs/list-dir + (map #(fs/relativize "test" %)) + (map #(string/replace % #"\.clj$" "")) + (map #(string/replace % fs/file-separator ".")) + (map #(string/replace % "_" "-")) + sort)}) + +(def args-usage "Valid args: + (unit|api|ide|all) + --help + +Commands: + unit Run only unit tests + api Run only api tests + ide Run only ide tests + all Run all tests + +Options: + --help Show this help + +Intended to be called from test.clj where setup for such things as browser +web browser selection and virtual displays occur") + +(defn -main [& args] + (when-let [opts (main/doc-arg-opt args-usage args)] + (let [;; we only need bb-spec for ide and unit tests tests, so explicitly otherwise test without it + cp-aliases (if (or (get opts "all") (get opts "ide") (get opts "unit")) + ":test:bb-spec" + ":test") + nses (cond + (get opts "ide") (:ide id->nses) + (get opts "api") (:api id->nses) + (get opts "unit") (:unit id->nses) + (get opts "all") (mapcat second id->nses)) + runner (-> "script/bb_test_runner_template.clj" + slurp + (string/replace "{{cp-aliases}}" cp-aliases) + (string/replace "{{nses}}" (->> nses + (map #(str "'" %)) + (string/join " ")))) + test-runner-file (-> (fs/create-temp-file {:prefix "bb-test-runner" + :suffix ".clj"}) + fs/file)] + (spit test-runner-file runner) + (tasks/shell "bb" test-runner-file) + (try + (finally + (fs/delete-if-exists test-runner-file)))))) + +(main/when-invoked-as-script + (apply -main *command-line-args*)) diff --git a/script/bb_test_runner_template.clj b/script/bb_test_runner_template.clj new file mode 100644 index 00000000..11747017 --- /dev/null +++ b/script/bb_test_runner_template.clj @@ -0,0 +1,15 @@ +(require '[babashka.classpath :as cp] + '[babashka.tasks :as tasks] + '[clojure.test :as t] + '[taoensso.timbre :as timbre]) + +;; bb log level by default is debug, let's set it to info +;; TODO: maybe there is some different abstraction for this? +(alter-var-root #'timbre/*config* #(assoc %1 :min-level :info)) + +(cp/add-classpath (with-out-str (tasks/clojure "-A{{cp-aliases}} -Spath"))) + +(require {{nses}}) + +(let [test-results (t/run-tests {{nses}})] + (System/exit (+ (:fail test-results) (:error test-results)))) diff --git a/script/test.clj b/script/test.clj index 0345a0e7..af325fcd 100644 --- a/script/test.clj +++ b/script/test.clj @@ -8,32 +8,37 @@ [helper.shell :as shell] [lread.status-line :as status])) -(defn- test-def [os id browser] +(defn- test-def [os id platform browser] {:os os :cmd (->> ["bb test" id + (str "--platform=" platform) (when browser (str "--browser=" browser)) (when (= "ubuntu" os) "--launch-virtual-display")] (remove nil?) (string/join " ")) - :desc (->> [id os browser] + :desc (->> [platform id os browser] (remove nil?) (string/join " "))}) (defn- github-actions-matrix [] (let [oses ["macos" "ubuntu" "windows"] ide-browsers ["chrome" "firefox"] - api-browsers ["chrome" "firefox" "edge" "safari"]] + api-browsers ["chrome" "firefox" "edge" "safari"] + platforms ["jvm" "bb"]] (->> (concat - (for [os oses] - (test-def os "unit" nil)) (for [os oses + platform platforms] + (test-def os "unit" platform nil)) + (for [os oses + platform platforms browser ide-browsers] - (test-def os "ide" browser)) + (test-def os "ide" platform browser)) (for [os oses + platform platforms browser api-browsers :when (not (or (and (= "ubuntu" os) (some #{browser} ["edge" "safari"])) (and (= "windows" os) (= "safari" browser))))] - (test-def os "api" browser))) + (test-def os "api" platform browser))) (sort-by :desc) (into [])))) @@ -57,62 +62,88 @@ (Thread/sleep 500) (recur)))))))) -(def args-usage "Valid args: - (api|ide) [--browser=]... [--launch-virtual-display] - (unit|all) [--launch-virtual-display] +(def valid-browsers ["chrome" "firefox" "edge" "safari"]) +(def valid-platforms ["jvm" "bb"]) + +(defn valid-opts [opts] + (format "<%s>" (string/join "|" opts))) + +(def args-usage (-> "Valid args: + (api|ide) [--browser=BROWSER]... [--launch-virtual-display] [--platform=PLATFORM] + (unit|all) [--launch-virtual-display] [--platform=PLATFORM] matrix-for-ci [--format=json] --help Commands: - unit Run only unit tests - api Run only api tests, optionally specifying browsers to override defaults - ide Run only ide tests, optionally specifying browsers to override defaults + unit Run unit tests + api Run api tests + ide Run ide tests all Run all tests using browser defaults matrix-for-ci Return text matrix for GitHub Actions Options: - --launch-virtual-display Launch a virtual display for browsers (use on linux only) + --browser=BROWSER {{valid-browsers}} overrides defaults + --platform=PLATFORM {{valid-platforms}} [default: jvm] + --launch-virtual-display Launch a virtual display for browsers --help Show this help Notes: - ide tests default to firefox and chrome only. - api tests default browsers based on OS on which they are run. -- launching a virtual display is necessary for GitHub Actions but not so for CircleCI") +- launching a virtual display is necessary for GitHub Actions but not for CircleCI" + (string/replace "{{valid-browsers}}" (valid-opts valid-browsers)) + (string/replace "{{valid-platforms}}" (valid-opts valid-platforms)))) (defn -main [& args] (when-let [opts (main/doc-arg-opt args-usage args)] - (cond - (get opts "matrix-for-ci") - (if (= "json" (get opts "--format")) - (status/line :detail (-> (github-actions-matrix) - (json/generate-string #_{:pretty true}))) - (status/line :detail (->> (github-actions-matrix) - (doric/table [:os :cmd :desc])))) + (let [browsers (->> (get opts "--browser") (keep identity)) + platform (get opts "--platform")] + (when (or (not-every? (set valid-browsers) browsers) + (and platform (not (some #{platform} valid-platforms)))) + (status/die 1 args-usage)) + (cond + (get opts "matrix-for-ci") + (if (= "json" (get opts "--format")) + (status/line :detail (-> (github-actions-matrix) + (json/generate-string #_{:pretty true}))) + (status/line :detail (->> (github-actions-matrix) + (doric/table [:os :cmd :desc])))) - :else - (let [clojure-args (cond - (get opts "api") "-M:test --namespace etaoin.api-test" - (get opts "ide") "-M:test --namespace etaoin.ide-test" - (get opts "unit") "-M:test --namespace-regex '.*unit.*-test$'" - :else "-M:test") - browsers (->> (get opts "--browser") (keep identity)) - env (cond-> {} - (seq browsers) - (assoc (if (get opts "api") - "ETAOIN_TEST_DRIVERS" - "ETAOIN_IDE_TEST_DRIVERS") - (mapv keyword browsers)) + :else + (let [env (cond-> {} + (seq browsers) + (assoc (if (get opts "api") + "ETAOIN_TEST_DRIVERS" + "ETAOIN_IDE_TEST_DRIVERS") + (mapv keyword browsers)) - (get opts "--launch-virtual-display") - (assoc "DISPLAY" ":99.0")) - shell-opts (if (seq env) - {:extra-env env} - {})] - (when (get opts "--launch-virtual-display") - (status/line :head "Launching virtual display") - (launch-xvfb)) - (status/line :head "Running tests") - (shell/clojure shell-opts clojure-args))))) + (get opts "--launch-virtual-display") + (assoc "DISPLAY" ":99.0")) + shell-opts (if (seq env) + {:extra-env env} + {}) + test-id (cond + (get opts "api") "api" + (get opts "ide") "ide" + (get opts "unit") "unit" + :else "all")] + (when (get opts "--launch-virtual-display") + (status/line :head "Launching virtual display") + (launch-xvfb)) + (status/line :head "Running %s tests on %s%s" + test-id + platform + (if (seq browsers) + (str " against browsers: " (string/join ", " browsers)) + "")) + (if (= "jvm" platform) + (shell/clojure shell-opts + (case test-id + "api" "-M:test --namespace etaoin.api-test" + "ide" "-M:test --namespace etaoin.ide-test" + "unit" "-M:test --namespace-regex '.*unit.*-test$'" + "all" "-M:test")) + (shell/command shell-opts "bb" "script/bb_test_runner.clj" test-id))))))) (main/when-invoked-as-script (apply -main *command-line-args*)) diff --git a/src/etaoin/client.clj b/src/etaoin/client.cljc similarity index 72% rename from src/etaoin/client.clj rename to src/etaoin/client.cljc index c9c46512..a91811c9 100644 --- a/src/etaoin/client.clj +++ b/src/etaoin/client.cljc @@ -1,8 +1,9 @@ (ns etaoin.client (:require [clojure.string :as str] [clojure.tools.logging :as log] - [clj-http.client :as client] - [cheshire.core :refer [parse-string]] + #?(:bb [clj-http.lite.client :as client] + :clj [clj-http.client :as client]) + [cheshire.core :as json] [slingshot.slingshot :refer [throw+]])) ;; @@ -24,12 +25,20 @@ (def timeout (read-timeout)) (def default-api-params - {:as :json - :accept :json - :content-type :json - :socket-timeout (* 1000 timeout) - :conn-timeout (* 1000 timeout) - :debug false}) + #?(:bb + {:accept :json + :content-type :json + :socket-timeout (* 1000 timeout) ;; TODO: used by clj-http-lite? + :conn-timeout (* 1000 timeout) ;; TODO: used by clj-http-lite? + :debug false ;; TODO: used by clj-http-lite? + } + :clj + {:as :json + :accept :json + :content-type :json + :socket-timeout (* 1000 timeout) + :conn-timeout (* 1000 timeout) + :debug false})) ;; ;; helpers @@ -52,7 +61,7 @@ (defn- parse-json [body] (let [body* (str/replace body #"Invalid Command Method -" "")] (try - (parse-string body* true) + (json/parse-string body* true) (catch Throwable _ body)))) (defn- error-response [body] @@ -74,7 +83,9 @@ {:url url :method method :throw-exceptions false}) - (= :post method) (assoc :form-params (-> payload (or {})))) + (= :post method) + #?(:bb (assoc :body (json/generate-string (or payload {}))) + :clj (assoc :form-params (or payload {})))) _ (log/debugf "%s %s:%s %6s %s %s" (name driver-type) @@ -85,7 +96,8 @@ (-> payload (or ""))) resp (client/request params) - body (:body resp) + body #?(:bb (-> resp :body parse-json) + :clj (:body resp)) error (delay {:type :etaoin/http-error :status (:status resp) :driver driver