diff --git a/CHANGELOG.adoc b/CHANGELOG.adoc index 07147e89..84d6229c 100644 --- a/CHANGELOG.adoc +++ b/CHANGELOG.adoc @@ -91,6 +91,7 @@ Other Changes * https://github.com/clj-commons/etaoin/issues/380[#380]: Etaoin is now Babashka compatible! * https://github.com/clj-commons/etaoin/issues/413[#413]: Etaoin now exports a clj-kondo config to help with the linting of its many handy macros +* https://github.com/clj-commons/etaoin/pull/357[#357]: Add support for connecting to a remote WebDriver via `:webdriver-url` (thanks https://github.com/verma[@verma] for the PR and https://github.com/mjmeintjes[@mjmeintjes] for the example usage!) * https://github.com/clj-commons/etaoin/issues/383[#383]: Drop testing for Safari on Windows, Apple no longer releases Safari for Windows * https://github.com/clj-commons/etaoin/issues/388[#388]: Drop testing for PhantomJS, development has long ago stopped for PhantomJS * https://github.com/clj-commons/etaoin/issues/387[#387]: No longer testing multiple key modifiers for a single webdriver send keys request diff --git a/README.adoc b/README.adoc index 8631651c..81557f11 100644 --- a/README.adoc +++ b/README.adoc @@ -85,6 +85,8 @@ Can be `alpha`, `beta`, `rc1`, etc. * https://github.com/daveyarwood[Dave Yarwood] * https://github.com/jkrasnay[John Krasnay] * https://github.com/kidd[Raimon Grau] +* https://github.com/verma[Uday Verma] +* https://github.com/mjmeintjes[Matt Meintjes] === Current Maintainers diff --git a/doc/01-user-guide.adoc b/doc/01-user-guide.adoc index adf5ca89..46c9fbb4 100644 --- a/doc/01-user-guide.adoc +++ b/doc/01-user-guide.adoc @@ -997,6 +997,16 @@ An exception will be thrown if the local file is not found. (e/upload-file driver file-input my-file) ---- +When interacting with a remote WebDriver process, you'll need to avoid the local file existence check by using `remote-file` like so: + +//:test-doc-blocks/skip +[source,clojure] +---- +(e/upload-file driver file-input (e/remote-file "/yes/i/really/do/exist.png")) +---- +The remote file is assumed to exist where the WebDriver is running. +The WebDriver will throw an error if it does not exist. + === Scrolling Etaoin includes functions to scroll the web page. @@ -1938,10 +1948,12 @@ Here, for example, we set an explicit path to the chrome WebDriver binary: a|`:host` for *WebDriver* process. When: -* omitted, creates a new local WebDriver process. +* omitted, creates a new local WebDriver process (unless `:webdriver-url` was specified). * specified, attempts to connect to an existing running WebDriver process. See <>. +Alternative: see `:webdriver-url` below. + Example: `:host "192.68.1.12"` | @@ -1958,6 +1970,19 @@ a| Varies by vendor: * edge `17556` * phantom `8910` +a| `:webdriver-url` for *WebDriver* process. When: + +* omitted, creates a new local WebDriver process (unless `:host` was specified). +* specified, attempts to connect to an existing running WebDriver process. + +See <>. + +Alternative: see `:host` above. + +Example: `"https://chrome.browserless.io/webdriver"` + +| + | `:path-driver` to *WebDriver* binary. + Typically used if your WebDriver is not on the PATH. @@ -2254,11 +2279,13 @@ To fine tune the proxy you use the original https://www.w3.org/TR/webdriver/#pro To connect to an existing WebDriver, specify the `:host` parameter. -TIP: When the `:host` parameter is not specified Etaoin will create a new WebDriver process. +TIP: When neither the `:host` nor the `:webdriver-url` parameter is specified Etaoin will launch a new WebDriver process. The `:host` can be a hostname (localhost, some.remote.host.net) or an IP address (127.0.0.1, 183.102.156.31). If the port is not specified, the <> `:port` is assumed. +Both `:host` and `:port` are ignored if `:webdriver-url` is specified. + Example: //:test-doc-blocks/skip @@ -2269,6 +2296,19 @@ Example: ;; Connect to an existing geckodriver process on remote most on default port (def driver (e/firefox {:host "192.168.1.11"})) ;; the default port for firefox is 4444 + +;; Connect to a chrome instance on browserless.io via :webdriver-url +;; (replace YOUR-API-TOKEN with a valid browserless.io api token if you want to try this out) +(e2/with-chrome [driver + {:webdriver-url "https://chrome.browserless.io/webdriver" + :capabilities {"browserless:token" "YOUR-API-TOKEN" + "chromeOptions" {"args" ["--no-sandbox"]}}}] + (e/go driver "https://en.wikipedia.org/") + (e/wait-visible driver [{:id :simpleSearch} {:tag :input :name :search}]) + (e/fill driver {:tag :input :name :search} "Clojure programming language") + (e/fill driver {:tag :input :name :search} k/enter) + (e/get-title driver)) +;; => "Clojure programming language - Search results - Wikipedia" ---- === Setting the Browser Profile [[browser-profile]] diff --git a/src/etaoin/api.clj b/src/etaoin/api.clj index 336f5a45..eb3e0065 100644 --- a/src/etaoin/api.clj +++ b/src/etaoin/api.clj @@ -59,7 +59,7 @@ - [[fill]] [[fill-active]] [[fill-el]] [[fill-multi]] - [[fill-human]] [[fill-human-el]] [[fill-human-multi]] - [[select]] [[selected?]] [[selected-el?]] - - [[upload-file]] + - [[upload-file]] [[remote-file]] - [[disabled?]] [[enabled?]] - [[clear]] [[clear-el]] - [[submit]] @@ -232,7 +232,7 @@ (defn get-status "Returns `driver` status. - Indicates readiness to create a new session. + Can indicate readiness to create a new session. The return varies for different driver implementations." [driver] @@ -2207,8 +2207,12 @@ ;; (defn running? - "Return true if `driver` seems accessable via its host and port." + "Return true if `driver` seems accessable via its `:host` and `:port`. + + Throws if using `:webdriver-url`" [driver] + (when (:webdriver-url driver) + (throw (ex-info "Not supported for driver using :webdriver-url" {}))) (util/connectable? (:host driver) (:port driver))) @@ -2586,11 +2590,15 @@ (merge {:message message} opt)))) (defn wait-running - "Waits until `driver` is [[running?]]. + "Waits until `driver` is reachable via its host and port via [[running?]]. + + Throws if using `:webdriver-url`. - `opt`: see [[wait-predicate]] opt." [driver & [opt]] - (log/debugf "Waiting for %s:%s is running" + (when (:webdriver-url driver) + (throw (ex-info "Not supported for driver using :webdriver-url" {}))) + (log/debugf "Waiting until %s:%s is running" (:host driver) (:port driver)) (wait-predicate #(running? driver) opt)) @@ -2952,13 +2960,26 @@ ;; file upload ;; +(defrecord RemoteFile [file]) +(defn remote-file + "Wraps `remote-file-path` for use with [[upload-file]] to avoid local file existence check. + + Example usage: + ```Clojure + (upload-file (remote-file \"C:/Users/hello/url.txt\")) + ```" + [remote-file-path] + (RemoteFile. remote-file-path)) + (defmulti upload-file - "Have `driver` attach a local file `path` to a file input field found by query `q`. + "Have `driver` attach a file `path` to a file input field element found by query `q`. Arguments: - `q` see [[query]] for details; - - `file` is either a string or java.io.File object that references a local file. The file should exist. + - `file` + - when a string or java.io.File object, the file must exist locally. + - when [[remote-file]] file is assumed to exist remotely and no local existence check is performed. Under the hood, we send the file's name as a sequence of keys to the input." {:arglists '([driver q path])} @@ -2969,6 +2990,11 @@ [driver q path] (upload-file driver q (fs/file path))) +(defmethod upload-file RemoteFile + [driver q path] + ;; directly send the path without any local file existence validation + (fill driver q (:file path))) + (defmethod upload-file java.io.File [driver q ^java.io.File file] (let [path (.getAbsolutePath file) @@ -3289,10 +3315,14 @@ from the `defaults` global map if is not passed. If there is no port in that map, a random-generated port is used. + -- `:webdriver-url` is a URL to a web-driver service. This URL is + generally provided by web-driver service providers. When specified the + `:host` and `:port` parameters are ignored. + -- `:locator` is a string determs what algorithm to use by default when finding elements on a page. `default-locator` variable is used if not passed." - [type & [{:keys [port host locator]}]] + [type & [{:keys [port host webdriver-url locator]}]] (let [port (or port (if host (get-in defaults [type :port]) @@ -3300,12 +3330,15 @@ host (or host "127.0.0.1") url (make-url host port) locator (or locator default-locator) - driver {:type type - :host host - :port port - :url url - :locator locator}] - (log/debugf "Created driver: %s %s:%s" (name type) host port) + driver {:type type + :host host + :port port + :url url + :webdriver-url webdriver-url + :locator locator}] + (if webdriver-url + (log/debugf "Created driver: %s %s" (name type) (util/strip-url-creds webdriver-url)) + (log/debugf "Created driver: %s %s:%s" (name type) host port)) driver)) (defn- proxy-env @@ -3316,7 +3349,6 @@ http (assoc :http http) ssl (assoc :ssl ssl)))) - (defn- -run-driver "Runs a driver process locally. @@ -3442,7 +3474,8 @@ to the browser's process. See https://www.w3.org/TR/webdriver/#capabilities" - [driver & [{:keys [url + [driver & [{:keys [webdriver-url + url size args prefs @@ -3453,7 +3486,8 @@ capabilities load-strategy desired-capabilities]}]] - (wait-running driver) + (when (not webdriver-url) + (wait-running driver)) (let [type (:type driver) caps (get-in defaults [type :capabilities]) proxy (proxy-env proxy) @@ -3509,11 +3543,12 @@ `opt` is optionally, see [Driver Options](/doc/01-user-guide.adoc#driver-options)." ([type] (boot-driver type {})) - ([type {:keys [host] :as opt}] + ([type {:keys [host webdriver-url] :as opt}] (cond-> type - true (-create-driver opt) - (not host) (-run-driver opt) - true (-connect-driver opt)))) + true (-create-driver opt) + (and (not host) + (not webdriver-url)) (-run-driver opt) + true (-connect-driver opt)))) (defn quit "Have `driver` close the current session, then, if Etaoin launched it, kill the WebDriver process." @@ -3638,6 +3673,8 @@ `(with-driver :chrome ~opt ~bind ~@body)) + + (defmacro with-edge "Executes `body` with an Edge driver session bound to `bind`. diff --git a/src/etaoin/impl/client.cljc b/src/etaoin/impl/client.cljc index 7d0c9679..0f819447 100644 --- a/src/etaoin/impl/client.cljc +++ b/src/etaoin/impl/client.cljc @@ -3,6 +3,7 @@ [cheshire.core :as json] [clojure.string :as str] [clojure.tools.logging :as log] + [etaoin.impl.util :as util] #?(:bb [clj-http.lite.client :as client] :clj [clj-http.client :as client]) [slingshot.slingshot :refer [throw+]])) @@ -74,28 +75,29 @@ ;; (defn call - [{driver-type :type :keys [host port] :as driver} + [{driver-type :type :keys [host port webdriver-url] :as driver} method path-args payload] (let [path (get-url-path path-args) - url (format "http://%s:%s/%s" host port path) + url (if webdriver-url + (format "%s/%s" webdriver-url path) + (format "http://%s:%s/%s" host port path)) params (cond-> (merge - default-api-params - {:url url - :method method - :throw-exceptions false}) + default-api-params + {:url url + :method method + :throw-exceptions false}) (= :post method) #?(:bb (assoc :body (.getBytes (json/generate-string (or payload {})) "UTF-8")) :clj (assoc :form-params (or payload {})))) - - _ (log/debugf "%s %s:%s %6s %s %s" + _ (log/debugf "%s %s %6s %s %s" (name driver-type) - host - port + (if webdriver-url + (util/strip-url-creds webdriver-url) + (str host ":" port)) (-> method name str/upper-case) path (-> payload (or ""))) - resp (client/request params) body #?(:bb (-> resp :body parse-json) :clj (:body resp)) @@ -103,6 +105,7 @@ :status (:status resp) :driver driver :response (error-response body) + :webdriver-url webdriver-url :host host :port port :method method diff --git a/src/etaoin/impl/util.clj b/src/etaoin/impl/util.clj index 9d6d52ad..8b7e774f 100644 --- a/src/etaoin/impl/util.clj +++ b/src/etaoin/impl/util.clj @@ -69,3 +69,16 @@ ~@body (finally (.delete tmp#))))) + +(defn strip-url-creds + "Return `url` with any http credentials stripped, https://user:pass@hello.com -> https://hello.com. + Use when logging urls to avoid spilling secrets." + ^String [^String url] + (let [u (java.net.URL. url)] + (.toExternalForm + (java.net.URL. + (.getProtocol u) + (.getHost u) + (.getPort u) + (.getFile u) + (.getRef u))))) diff --git a/test/etaoin/unit/proc_test.clj b/test/etaoin/unit/proc_test.clj index 9608937f..a9636f8b 100644 --- a/test/etaoin/unit/proc_test.clj +++ b/test/etaoin/unit/proc_test.clj @@ -55,12 +55,24 @@ (is (= 2 (get-count-chromedriver-instances)))) (proc/kill process))) -(deftest test-process-forking-connect-existing +(deftest test-process-forking-connect-existing-host (let [port 9999 process (proc/run ["chromedriver" (format "--port=%d" port)]) _ (e/wait-running {:port port :host "localhost"}) + ;; should connect, not launch driver (e/chrome {:host "localhost" :port port :args ["--no-sandbox"]})] - (e/wait-running driver) + (is (= 1 (get-count-chromedriver-instances))) + (e/quit driver) + (proc/kill process))) + +(deftest test-process-forking-connect-existing-webdriver-url + (let [port 9999 + process (proc/run ["chromedriver" (format "--port=%d" port)]) + ;; normally would not call wait-running for a remote service, we are simulating here and want + ;; to make sure the process we launched is up and running + _ (e/wait-running {:port port :host "localhost"}) + ;; should connect, not launch + driver (e/chrome {:webdriver-url (format "http://localhost:%d" port) :args ["--no-sandbox"]})] (is (= 1 (get-count-chromedriver-instances))) (e/quit driver) (proc/kill process)))