From 27611e9d02764593484a1c8c9aac151c907145cc Mon Sep 17 00:00:00 2001 From: Oldes Huhuman Date: Thu, 4 Jan 2024 15:30:39 +0100 Subject: [PATCH] FEAT: including source of `webdriver` and `websocket` modules --- README.md | 2 + src/boot/sysobj.reb | 2 + src/modules/webdriver.reb | 218 +++++++++++++++++++++++++++++++++ src/modules/websocket.reb | 247 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 469 insertions(+) create mode 100644 src/modules/webdriver.reb create mode 100644 src/modules/websocket.reb diff --git a/README.md b/README.md index 9679918b21..74a6aa6aca 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,8 @@ It is possible to extend Rebol functionality using external modules (native and * [Rebol/OpenCV](https://github.com/Oldes/Rebol-OpenCV) - Computer Vision Library * [Rebol/SQLite](https://github.com/Siskin-framework/Rebol-SQLite) - SQL database engine * [Rebol/Triangulate](https://github.com/Siskin-framework/Rebol-Triangulate) - Two-Dimensional Quality Mesh Generator and Delaunay Triangulator +* [Rebol/WebDriver](https://github.com/Oldes/Rebol-WebDriver) - WebDriver scheme for automating Chromium based browser sessions +* [Rebol/WebSocket](https://github.com/Oldes/Rebol-WebSocket) - WebSocket scheme and codec It should be noted that on macOS it may be required to resign _downloaded native extensions_ using command like: ``` diff --git a/src/boot/sysobj.reb b/src/boot/sysobj.reb index 03dea94929..5e819683de 100644 --- a/src/boot/sysobj.reb +++ b/src/boot/sysobj.reb @@ -271,6 +271,8 @@ modules: object [ mime-field: https://src.rebol.tech/mezz/codec-mime-field.reb mime-types: https://src.rebol.tech/mezz/codec-mime-types.reb quoted-printable: https://src.rebol.tech/mezz/codec-quoted-printable.reb + webdriver: https://src.rebol.tech/modules/webdriver.reb + websocket: https://src.rebol.tech/modules/websocket.reb ;; and.. window: none ;- internal extension for gui (on Windows so far!) ] diff --git a/src/modules/webdriver.reb b/src/modules/webdriver.reb new file mode 100644 index 0000000000..b11979f6d7 --- /dev/null +++ b/src/modules/webdriver.reb @@ -0,0 +1,218 @@ +Rebol [ + Title: "WebDriver (chrome) scheme" + Type: module + Name: webdriver + Date: 03-Jan-2024 + Version: 0.1.0 + Author: @Oldes + Home: https://github.com/Oldes/Rebol-WebDriver + Rights: http://opensource.org/licenses/Apache-2.0 + Purpose: {Can be used to automate browser sessions.} + History: [ + 03-Jan-2024 "Oldes" {Initial version} + ] + Needs: [ + 3.11.0 ;; Minimal Rebol version required by WebScocket module + websocket + json + ] + Notes: { + Currently only `chrome` scheme is implemented which is supposed to be working + with Chromium, Chrome and other Blink-based browsers. + + The browser must be started with `remote-debugging` enabled. + + For example on macOS using a Brave browser: + ```terminal + /Applications/Brave\ Browser.app/Contents/MacOS/Brave\ Browser --remote-debugging-port=9222 + ``` + + Available methods are documented here: https://chromedevtools.github.io/devtools-protocol/ + } +] + +system/options/log/chrome: 4 + +;; internal functions.... +read-and-wait: function[ + "Wait specified time while processing read events" + port [port!] "Internal websocket port of the webdrive scheme" + time [time!] +][ + start: now/precise + end: start + time + until [ + read port + wait [port time] + + process-packets port + + time: difference end now/precise + time <= 0:0:0 + ] +] + +process-packets: function[ + "Process incomming webscocket packets of the webdrive scheme" + conn [port!] +][ + port: conn/parent ;; outter webdrive scheme + ctx: port/extra + foreach packet conn/data [ + try/with [ + packet: decode 'json packet + either packet/id [ + ctx/pending: ctx/pending - 1 + append port/data packet + ][ port/actor/on-method packet ] + ] :print + ] + clear conn/data +] + +ws-decode: :codecs/ws/decode + +;- The Chrome scheme --------------------------------------------------------------- +sys/make-scheme [ + name: 'chrome + title: "Chrome WebDriver API" + spec: object [title: scheme: ref: host: none port: 9222] + + actor: [ + open: func [port [port!] /local ctx spec host conn data port-spec][ + spec: port/spec + spec/host: any [spec/host "localhost"] + spec/port: any [spec/port 9222] + + port/data: copy [] ;; holds decoded websocket responses + port/extra: ctx: context [ + host: rejoin [http:// spec/host #":" spec/port] + version: none + browser: none + counter: 0 + pending: 0 ;; increments when a new method is sent, decremented when response is received + req: #(id: 0 method: none params: #[none]) ;; used to send a command (to avoid cerating a new map) + page-info: none ;; holds resolved info from an attached page + page-conn: none ;; webscocket connection to an attached page + ] + + ctx/version: data: try/with [ + decode 'json read ctx/host/json/version + ][ + sys/log/error 'CHROME "Failed to get browser info!" + sys/log/error 'CHROME system/state/last-error + return none + ] + + ctx/browser: conn: open as url! data/webSocketDebuggerUrl + conn/parent: port + wait [conn 15] + sys/log/more 'CHROME "Browser connection opened." + port + ] + open?: func[port /local ctx][ + all [ + ctx: port/extra + any [ctx/browser ctx/page-conn] + ] + ] + close: func[port /local ctx][ + ctx: port/extra + if ctx/port-conn [ + try [close ctx/port-conn wait [ctx/page-conn 1]] + ctx/port-conn: ctx/port-info: none + + ] + if ctx/browser [ + try [close ctx/browser wait [ctx/browser 1]] + ctx/browser: none + ] + port + ] + + write: func[port data /local ctx url time method params conn][ + unless block? data [data: reduce [data]] + + sys/log/info 'CHROME ["WRITE:" as-green mold/flat data] + + clear port/data + + ctx: port/extra + either open? ctx/browser [ + parse data [some [ + + set url: url! ( + ;- Open a new target (page) + try/with [ + ctx/page-info: decode 'json write join ctx/host/json/new? url [PUT] + ;?? ctx/page-info + append port/data ctx/page-info + + ctx/page-conn: conn: open as url! ctx/page-info/webSocketDebuggerUrl + ;conn/awake: :ws-web-awake + conn/parent: port + wait [conn 15] + conn: none + ] :print + ) + | set time: [time! | decimal! | integer!] ( + ;- Wait some time while processing incomming messages + time: to time! time + sys/log/info 'CHROME ["WAIT" as-green time] + read-and-wait any [ctx/page-conn ctx/browser] time + ) + | + set method: word! set params: opt [map! | block!] ( + ;- Send a command with optional options + if block? params [params: make map! reduce/no-set params] + sys/log/info 'CHROME ["Command:" as-red method as-green mold/flat/part params 100] + ;; resusing `req` value for all commands as it is just used to form a json anyway + ctx/req/id: ctx/counter: ctx/counter + 1 ;; each command has an unique id + ctx/req/method: method + ctx/req/params: params + ctx/pending: ctx/pending + 1 + write conn: any [ctx/page-conn ctx/browser] ctx/req + ;; don't wake up until received responses for all command requests + forever [ + ;@@TODO: handle the timeout to awoid infinite loop! + wait [conn 15] ;; wait for any events + process-packets conn ;; process incomming websocket messages + if ctx/pending <= 0 [break] ;; exit the loop if there are no pending requests + read conn ;; keep reading + ] + ) + ]] + either 1 = length? port/data [first port/data][port/data] + ][ sys/log/error 'CHROME "Not open!"] + ] + + read: func[port /local ctx conn packet][ + ;; waits for any number of incomming messages + if all [ + ctx: port/extra + conn: any [ctx/page-conn ctx/browser] + ][ + clear port/data + read conn + wait [conn 1] ;; don't wait more then 1 second if there are no incomming messages + process-packets conn + ] + port/data + ] + + pick: func[port value /local result][ + ;; just a shortcut to get a single result direcly + unless block? value [value: reduce [value]] + result: write port value + if block? result [result: last result] + result/result + ] + + on-method: func[packet][ + ;; this function is supposed to be user defined and used to process incomming messages + ;; in this case it just prints its content... + sys/log/info 'CHROME [as-red packet/method mold packet/params] + ] + ] +] + diff --git a/src/modules/websocket.reb b/src/modules/websocket.reb new file mode 100644 index 0000000000..3fc21f2aca --- /dev/null +++ b/src/modules/websocket.reb @@ -0,0 +1,247 @@ +Rebol [ + Title: "WebSocket scheme and codec" + Type: module + Name: websocket + Date: 01-Jan-2024 + Version: 0.1.0 + Author: @Oldes + Exports: [http-server decode-target to-CLF-idate] + Home: https://github.com/Oldes/Rebol-WebSocket + Rights: http://opensource.org/licenses/Apache-2.0 + Purpose: {Communicate with a server over WebSocket's connection.} + History: [ + 01-Jan-2024 "Oldes" {Initial version} + ] + Needs: [3.11.0] +] + +;--- WebSocket Codec -------------------------------------------------- +append system/options/log [ws: 4] +system/options/quiet: false +register-codec [ + name: 'ws + type: 'text + title: "WebSocket" + + encode: function/with [ + "Encodes one WebSocket message." + data [binary! any-string! word! map!] + /no-mask + ][ + case [ + data = 'ping [return #{81801B1F519C}] + data = 'close [return #{888260D19A196338}] + map? data [data: to-json data] + word? data [data: form data] + ] + out: clear #{} + ;; first byte has FIN bit and an opcode (if data are string or binary data) + byte1: either binary? data [2#10000010][2#10000001] ;; final binary/string + unless binary? data [data: to binary! data] + len: length? data + either no-mask [ + binary/write out case [ + len < 0#007E [[UI8 :byte1 UI8 :len :data]] + len <= 0#FFFF [[UI8 :byte1 UI8 126 UI16 :len :data]] + 'else [[UI8 :byte1 UI8 127 UI64 :len :data]] + ] + ][ + ;; update a mask... + repeat i 4 [mask/:i: 1 + random 254] ;; avoiding zero + data: data xor mask + binary/write out case [ + len < 0#007E [byte2: 2#10000000 | len [UI8 :byte1 UI8 :byte2 :mask :data]] + len <= 0#FFFF [[UI8 :byte1 UI8 254 UI16 :len :mask :data]] + 'else [[UI8 :byte1 UI8 255 UI64 :len :mask :data]] + ] + ] + out + ][ + mask: #{00000000} + out: make binary! 100 + ] + + decode: function [ + "Decodes WebSocket messages from a given input." + data [binary!] "Consumed data are removed! (modified)" + ][ + out: clear [] + ;; minimal WebSocket message has 2 bytes at least (when no masking involved) + while [2 < length? data][ + final?: data/1 & 2#10000000 = 2#10000000 + opcode: data/1 & 2#00001111 + mask?: data/2 & 2#10000000 = 2#10000000 + len: data/2 & 2#01111111 + data: skip data 2 + + ;@@ Not removing bytes until we make sure, that there is enough data! + case [ + len = 126 [ + ;; there must be at least 2 bytes for the message length + if 2 >= length? data [break] + len: binary/read data 'UI16 + data: skip data 2 + ] + len = 127 [ + if 8 >= length? data [break] + len: binary/read data 'UI64 + data: skip data 8 + ] + ] + if (4 + length? data) < len [break] + data: truncate data ;; removes already processed bytes from the head + either mask? [ + masks: take/part data 4 + temp: masks xor take/part data len + if len < 4 [truncate/part temp len] ;; the mask was longer then the message + ][ temp: take/part data len ] + if all [final? opcode = 1] [try [temp: to string! temp]] + append append append out :final? :opcode :temp + ] + out + ] +] + +ws-encode: :codecs/ws/encode +ws-decode: :codecs/ws/decode + +;--- WebSocket Scheme ------------------------------------------------- +ws-conn-awake: func [event /local port extra parent spec temp] [ + port: event/port + unless parent: port/parent [return true] + extra: parent/extra + sys/log/more 'WS ["==TCP-event:" as-red event/type] + either extra/handshake [ + switch event/type [ + read [ + append parent/data port/data + clear port/data + ] + ] + insert system/ports/system make event! [ type: event/type port: parent ] + port + ][ + switch/default event/type [ + ;- Upgrading from HTTP to WS...] + read [ + ;print ["^/read:" length? port/data] + append parent/data port/data + clear port/data + ;probe to string! parent/data + either find parent/data #{0D0A0D0A} [ + ;; parse response header... + try/with [ + ;; skip the first line and construct response fields + extra/fields: temp: construct find/tail parent/data #{0D0A} + unless all [ + "websocket" = select temp 'Upgrade + "Upgrade" = select temp 'Connection + extra/key = select temp 'Sec-WebSocket-Accept + ][ + insert system/ports/system make event! [ type: 'error port: parent ] + return true + ] + + ] :print + + clear port/data + clear parent/data + extra/handshake: true + insert system/ports/system make event! [ type: 'connect port: parent ] + ][ + ;; missing end of the response header... + read port + ] + ] + wrote [read port] + lookup [open port] + connect [ + spec: parent/spec + extra/key: enbase/part checksum form now/precise 'sha1 64 16 + write port ajoin [ + {GET } spec/path spec/target { HTTP/1.1} CRLF + {Host: } spec/host if spec/port [join #":" spec/port] CRLF + {Upgrade: websocket} CRLF + {Connection: Upgrade} CRLF + {Sec-WebSocket-Key: } extra/key CRLF + {Sec-WebSocket-Protocol: chat, superchat} CRLF + {Sec-WebSocket-Version: 13} CRLF + CRLF + ] + extra/key: enbase checksum join extra/key "258EAFA5-E914-47DA-95CA-C5AB0DC85B11" 'sha1 64 + ] + ][true] + ] +] +sys/make-scheme [ + name: 'ws + title: "Websocket" + spec: make system/standard/port-spec-net [] + awake: func [event /local port extra parent spec temp] [ + port: event/port + sys/log/debug 'WS ["== WS-event:" as-red event/type] + switch event/type [ + read [ + sys/log/debug 'WS ["== raw-data:" as-blue port/data] + ws-decode port/data + ] + wrote [] + connect [ + ;; optional validation of response headers + ?? port/extra/fields + ] + error [ + print "closing..." + try [close port/extra/connection] + ;wait port/extra/connection + ] + ] + true + ] + actor: [ + open: func [port [port!] /local spec host conn port-spec][ + spec: port/spec + port/extra: context [ + connection: + key: + handshake: + fields: none + ] + port/data: make binary! 200 + ;; `ref` is used in logging and errors + conn: make port/spec [ref: none] + conn/scheme: 'tcp + port-spec: if spec/port [join #":" spec/port] + conn/ref: as url! ajoin [conn/scheme "://" spec/host port-spec] + spec/ref: as url! ajoin ["ws://" spec/host port-spec] + port/extra/connection: conn: make port! conn + conn/parent: port + conn/awake: :ws-conn-awake + open conn + port + ] + open?: func[port /local ctx][ + all [ + ctx: port/extra + ctx/handshake + open? ctx/connection + ] + ] + close: func[port][ + close port/extra/connection + ] + write: func[port data][ + sys/log/debug 'WS ["write:" as-green mold data] + either open? port [ + write port/extra/connection ws-encode data + ][ sys/log/error 'WS "Not open!"] + + ] + read: func[port][ + either open? port [ + read port/extra/connection + ][ sys/log/error 'WS "Not open!"] + + ] + ] +] \ No newline at end of file