docs: user guide examples now os agnostic (#546)

* docs: user guide chord examples now os-agnostic

Keyboard chord examples formerly macOS specific.
Add Ubuntu and Windows for test-doc to CI matrix.

Weirdness: when testing locally on Linux, for sessions with no real
interaction, was sporadically getting socket timeout when trying to quit
session. For now, introduce a couple of waits in doc examples to appease
the mystery.

I expect I'll need to tweak a thing or two to get this working on CI.
Do I need to setup a virtual display for Ubuntu?

* tests: support running test-doc under linux/docker

Add virtual display support to test-doc.
Extract out common support to new script/virtual_display.clj
that is now shared between test and test-doc scripts.

Adjust dev docker image to:
- run from a non-root user
- copy over etaoin sources so that they are usable by this non-root user
- mimic Selenium docker images by always invoking chrome with the
--no-sandbox option

* docs: changelog [skip ci]

Closes #536

CHANGELOG.adoc

@@ -10,6 +10,7 @@ A release with an intentional breaking changes is marked with:
 * bump all deps to current versions
 * docs
 ** https://github.com/clj-commons/etaoin/issues/534[#534]: better describe `etaoin.api/select` and its alternatives
+** https://github.com/clj-commons/etaoin/issues/536[#536]: user guide examples are now all os agnostic and CI tested via test-doc-blocks on all supported OSes
 
 == v1.0.40
 

Dockerfile

@@ -1,7 +1,8 @@
-FROM clojure:temurin-17-tools-deps
+FROM clojure:temurin-17-tools-deps-bullseye-slim
 
 RUN apt-get -yqq update && \
     apt-get -yqq upgrade && \
+    apt-get -yqq install sudo && \
     apt-get -yqq install libnss3 && \
     apt-get -yqq install bzip2 && \
     apt-get -yqq install imagemagick && \
@@ -12,16 +13,45 @@ RUN apt-get -yqq update && \
     apt-get install -y fluxbox && \
     rm -rf /var/lib/apt/lists/*
 
-# This dockerfile is not amenable to layer updates because nothing changes
-# build with --no-cache
+# This dockerfile is not amenable to layer updates because although commands herein
+# don't change, the impact of them does. Basically we are saying: install latest of
+# things. Build this image with --no-cache to ensure you get latest as intented.
 
 # Install babashka, we use it for (and after) docker image building
 RUN curl -sLO https://raw.githubusercontent.com/babashka/babashka/master/install && \
     chmod +x install && \
     ./install && \
     rm ./install
 
-# Copy over etaoin only for its build support, delete after use
+# Image will default non-root user: etaoin-user
+RUN groupadd etaoin-user && \
+    useradd --create-home --shell /bin/bash --gid etaoin-user etaoin-user && \
+    usermod -a -G sudo etaoin-user && \
+    echo 'ALL ALL = (ALL) NOPASSWD: ALL' >> /etc/sudoers && \
+    echo 'etaoin-user:secret' | chpasswd
+ENV HOME=/home/etaoin-user
+
+# Copy over etaoin sources for docker image build support,
+# we'll delete after use here but copy over fresh etaoin sources
+# at image when run
 COPY ./ /etaoin
+RUN cd /etaoin && bb -docker-install
+
+# download deps to avoid repeating this work during image run
+RUN chown -R etaoin-user:etaoin-user /etaoin
+USER etaoin-user
+RUN cd /etaoin && bb download-deps
+
+# Create a spot to copy over fresh sources
+RUN mkdir /home/etaoin-user/etaoin
+
+USER root
+RUN rm -rf /etaoin
+
+USER etaoin-user
+
+WORKDIR /home/etaoin-user/etaoin
+
+COPY script/docker_entry.clj /bin
 
-RUN cd /etaoin && bb -docker-install && bb download-deps && rm -rf /etaoin
+ENTRYPOINT ["/usr/local/bin/bb", "/bin/docker_entry.clj"]

bb.edn

@@ -50,7 +50,7 @@
   test:bb        {:doc "Runs tests under Babashka [--help]"
                   :task test/test-bb}
   test-doc       {:doc "test code blocks in user guide"
-                  :task test-doc/-main}
+                  :task test-doc/test-doc}
   test-matrix    {:doc "Returns a test matrix for CI [--help]"
                   :task test-matrix/-main}
   drivers        {:doc "[list|kill] any running WebDrivers"
@@ -72,12 +72,10 @@
                    :task docker-install/-main}
   docker-run     {:doc "run etaoin docker image (specify no commmands for interactive)"
                   :task (apply shell/command {:continue true}
-                               "docker run -it --rm -v"
-                               (cond-> [(str (fs/cwd) ":/etaoin")
-                                        "-w" "/etaoin"]
-                                 (not (seq *command-line-args*)) (conj "--entrypoint" "/bin/bash")
-                                 :always (conj "etaoin:latest")
-                                 :always (concat *command-line-args*)))}
+                               "docker run -it --rm --shm-size=1gb"
+                               (str "-v" (fs/cwd) ":/etaoin")
+                               "etaoin:latest"
+                               *command-line-args*)}
   ci-release     {:doc "release tasks, use --help for args"
                   :task ci-release/-main }
   -gha-win-edge-workaround {:doc "Deal with Edge browser/Edge WebDriver version mismatch on GitHub Actions"

deps.edn

@@ -35,8 +35,12 @@
   ;; usage: test:test-docs
   :test-docs {:override-deps {org.clojure/clojure {:mvn/version "1.11.1"}}
               :extra-paths ["target/test-doc-blocks/test"]
-              :main-opts ["-m" "cognitect.test-runner"
-                          "-d" "target/test-doc-blocks/test"]}
+              :exec-fn cognitect.test-runner.api/test
+              :exec-args {:dirs ["target/test-doc-blocks/test"]}
+              :org.babashka/cli {:coerce {:nses [:symbol]
+                                          :patterns [:string]
+                                          :vars [:symbol]}}
+              :main-opts ["-m" "babashka.cli.exec"]}
 
   ;; for consistent linting we use a specific version of clj-kondo through the jvm
   :clj-kondo {:extra-deps {clj-kondo/clj-kondo {:mvn/version "2023.03.17"}}

doc/01-user-guide.adoc

@@ -1002,22 +1002,21 @@ A quick example of entering ordinary characters while holding Shift:
 ----
 
 The main input gets populated with "CAPS IS GREAT".
-Now maybe you'd like to delete the last word.
-Assuming you are using Chrome, this is done by pressing backspace holding Alt.
-Let's do that:
+Let's duplicate the text via select-all, copy, and paste keyboard shortcuts:
 
 [source,clojure]
 ----
-(e/fill-active driver (k/with-alt k/backspace))
+(if (= "Mac OS X" (System/getProperty "os.name"))
+  (e/fill-active driver (k/with-command "a") (k/with-command "c") k/arrow-right " " (k/with-command "v"))
+  (e/fill-active driver (k/with-ctrl "a") (k/with-ctrl "c") k/arrow-right " " (k/with-ctrl "v")))
 (e/get-element-value driver :active)
-;; => "CAPS IS "
+;; => "CAPS IS GREAT CAPS IS GREAT"
 ----
 
-Consider a more complex example which repeats real user behaviour.
-You'd like to delete everything from the input.
-First, you move the cursor to the very beginning of the input field.
-Then move it to the end holding shift so everything gets selected.
-Finally, you press delete to clear the selected text:
+And now let's clear the input by:
+1. moving the cursor to the beginning of the input field with the home key
+2. moving the cursor to the end field while holding shift to select all text
+3. deleting the selected text with the delete key
 
 [source,clojure]
 ----
@@ -1026,12 +1025,10 @@ Finally, you press delete to clear the selected text:
 ;; => ""
 ----
 
-There are also `with-ctrl` and `with-command` functions that act as you would expect.
-
 NOTE: These functions do not apply to the global browser's shortcuts.
 For example, neither "Command + R" nor "Command + T" reload the page or open a new tab.
 
-All the `etaoin.keys/with-*` functions are just wrappers upon the `etaoin.keys/chord` function that might be used for complex cases.
+The `etaoin.keys/with-*` functions are just wrappers for the `etaoin.keys/chord` function that might be used for complex cases.
 
 === File Uploading
 
@@ -2254,6 +2251,7 @@ or
 ;; you can also check if a driver is in headless mode:
 (e/headless? driver)
 ;; => true
+(e/wait 1) ;; seems to appease Firefox on Linux
 (e/quit driver)
 ----
 
@@ -2337,6 +2335,7 @@ Set a custom `User-Agent` header with the `:user-agent` option when creating a d
 ----
 (e/with-firefox {:user-agent "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"}
                 driver
+  (e/wait 1) ;; seems to appease Firefox on Linux
   (e/get-user-agent driver))
 ;; => "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)"
 ----
@@ -3034,12 +3033,17 @@ Reproduction:: When you try to start the chromedriver you get an error:
 
 Possible Cause::
 A common cause for Chrome to crash during startup is running Chrome as root user (administrator) on Linux.
-While it is possible to work around this issue by passing --no-sandbox flag when creating your WebDriver session, such a configuration is unsupported and highly discouraged.
-You need to configure your environment to run Chrome as a regular user instead.
+While it is possible to work around this issue by passing --no-sandbox flag when creating your WebDriver session, such a configuration is unsupported and discouraged.
+Ideally you would configure your environment to run Chrome as a regular user instead.
++
+NOTE: We have noticed that the Selenium docker images always invoke chrome with `--no-sandbox`, so the caveat is a little confusing.
+We've naively replicated this in our own dev docker images.
+Maybe `--no-sandbox` is acceptable, or even needed, when running from a docker container?
+If you have intel here, let us know!
 
-Potential Solution:: Run driver with an argument `--no-sandbox`.
+Potential Solution:: Run driver with argument `--no-sandbox`.
 Caution!
-This bypasses OS security model.
+This bypasses the OS security model.
 +
 [source,clojure]
 ----

doc/02-developer-guide.adoc

@@ -160,11 +160,9 @@ And then at the interactive prompt:
 bb test:jvm --suites ide --browsers firefox
 ----
 
-NOTE: `docker-run` plunks you automatically into `/etaoin` which maps to the etaoin project root
+NOTE: `docker-run` copies etaoin project files into `/home/etaoin-user/etaoin` which will be your work dir.
 
-There is nothing magic about the babashka docker tasks.
-They are here for our convenience.
-Feel free to run docker commands directly if that is more your thing.
+The docker image is catered to running Etaoin tests.
 
 === WebDriver Processes
 

script/docker_entry.clj

@@ -0,0 +1,40 @@
+(ns docker-entry
+  ;; keep deps to built in to avoid having to bring over more sources to docker image
+  (:require [babashka.fs :as fs]
+            [babashka.process :as proc]))
+
+(defn copy-etaoin-sources
+  "If the conditions are right, copy over mounted etaoin sources, if not warn."
+  []
+  ;; some sanity checks first
+  (let [errors (reduce (fn [errors check]
+                         (if-let [error (check)]
+                           (conj errors error)
+                           errors))
+                       []
+                       [#(when (not (fs/exists? "/etaoin"))
+                           "/etaoin sources not mounted")
+                        #(when (not (= "/home/etaoin-user/etaoin" (str (fs/cwd))))
+                           "expected cwd to be /home/etaoin-user/etaoin")])]
+
+    (if (seq errors)
+      (do
+        (println "* WARNING: etaoin sources not copied:")
+        (run! #(println "-" %) errors))
+      (do
+        (println "copying mounted etaoin sources")
+        (fs/copy-tree "/etaoin" ".")))))
+
+(defn -main
+  "Docker image entry point script to run at docker image launch.
+  We cannot simply use a -v mounted /etaoin due to permissions, so we copy over
+  the a mounted /etaoin to a spot where we do have full rights."
+  [& args]
+  (copy-etaoin-sources)
+  ;; and now run provided command, else bash if none specified
+  (let [cmd (if (seq args) args "/bin/bash")]
+    (println "running command:" cmd)
+    (proc/exec cmd)))
+
+(when (= *file* (System/getProperty "babashka.file"))
+  (apply -main *command-line-args*))

script/docker_install.clj

@@ -3,6 +3,7 @@
             [babashka.fs :as fs]
             [cheshire.core :as json]
             [clojure.java.io :as io]
+            [clojure.string :as string]
             [helper.main :as main]
             [helper.shell :as shell]
             [lread.status-line :as status]))
@@ -42,6 +43,26 @@
     (shell/command "apt-get -yqq update")
     (shell/command "apt-get -yqq install google-chrome-stable")))
 
+(defn- wrap-chrome
+  "The Selenium chrome docker image wraps the chrome launcher, so I'm going with the flow here.
+  I don't fully understand if --no-sandbox is required for docker images, but if Selenium is doing it,
+  I'm not interested in figuring out if we don't need to as well.
+
+  They also do the umask thing... so mimicing that as well."
+  []
+  (status/line :head "Wrapping chrome launcher")
+  (let [launcher (-> (shell/command {:out :string}
+                                    "readlink -f /usr/bin/google-chrome")
+                     :out
+                     (string/trim))
+        launcher-orig-renamed (str launcher "-base")]
+    (fs/move launcher launcher-orig-renamed)
+    (spit launcher (string/join "\n"
+                                ["#!/bin/bash"
+                                 "umask 002"
+                                 (format "exec -a \"$0\" \"%s\" --no-sandbox \"$@\"" launcher-orig-renamed)]))
+    (shell/command "chmod +x" launcher)))
+
 (defn- install-geckodriver []
   (status/line :head "Installing geckodriver")
   (let [dl-file "/tmp/geckodriver_linux64.tar.gz"
@@ -77,6 +98,7 @@
     (status/die 1 "Expected to be run from DockerFile"))
   (install-chromedriver)
   (install-chrome)
+  (wrap-chrome)
   (install-geckodriver)
   (install-firefox))
 

script/helper/os.clj

@@ -1,5 +1,6 @@
 (ns helper.os
-  (:require [clojure.string :as string]))
+  (:require [babashka.fs :as fs]
+            [clojure.string :as string]))
 
 (defn get-os []
   (let [os-name (string/lower-case (System/getProperty "os.name"))]
@@ -9,3 +10,6 @@
       #"(nix|nux|aix)" :unix
       #"sunos" :solaris
       :unknown)))
+
+(defn running-in-docker? []
+  (fs/exists? "/.dockerenv"))

script/helper/virtual_display.clj

@@ -0,0 +1,40 @@
+(ns helper.virtual-display
+  (:require [babashka.fs :as fs]
+            [babashka.process :as process]
+            [helper.shell :as shell]
+            [lread.status-line :as status]))
+
+(defn- launch-xvfb []
+  (if (fs/which "Xvfb")
+    (process/process "Xvfb :99 -screen 0 1024x768x24" {:out (fs/file "/dev/null")
+                                                       :err (fs/file "/dev/null")})
+    (status/die 1 "Xvfb not found"))
+  (let [deadline (+ (System/currentTimeMillis) 10000)]
+    (loop []
+      (let [{:keys [exit]} (shell/command {:out (fs/file "/dev/null")
+                                           :err (fs/file "/dev/null")
+                                           :continue true}
+                                          "xdpyinfo -display :99")]
+        (if (zero? exit)
+          (status/line :detail "Xvfb process looks good.")
+          (if (> (System/currentTimeMillis) deadline)
+            (status/die 1 "Failed to get status from Xvfb process")
+            (do
+              (status/line :detail "Waiting for Xvfb process.")
+              (Thread/sleep 500)
+              (recur))))))))
+
+(defn- launch-fluxbox []
+  (if (fs/which "fluxbox")
+    (process/process "fluxbox -display :99" {:out (fs/file "/dev/null")
+                                             :err (fs/file "/dev/null")})
+    (status/die 1 "fluxbox not found")))
+
+(defn launch []
+  (status/line :head "Launching virtual display")
+  (launch-xvfb)
+  (launch-fluxbox))
+
+(defn extra-env "Returns env vars required for programs using launched virtual display"
+  []
+  {"DISPLAY" ":99.0"})