diff --git a/agent/main/lib/Browser.ts b/agent/main/lib/Browser.ts index e415f2bcb..502639849 100755 --- a/agent/main/lib/Browser.ts +++ b/agent/main/lib/Browser.ts @@ -375,12 +375,8 @@ export default class Browser extends TypedEventEmitter implement } } if (options.useRemoteDebuggingPort) { - const randomPort = (min: number, max: number): number => { - return Math.floor(Math.random() * (max - min) + min); - }; - const port = randomPort(9900, 9999); - launchArgs.push(`--remote-debugging-port=${port}`); - this.engine.remoteDebuggingPort = port; + this.engine.useRemoteDebuggingPort = true; + launchArgs.push('--remote-debugging-port=0'); } else { launchArgs.push('--remote-debugging-pipe'); } diff --git a/agent/main/lib/BrowserProcess.ts b/agent/main/lib/BrowserProcess.ts index 906305972..f0b044c64 100755 --- a/agent/main/lib/BrowserProcess.ts +++ b/agent/main/lib/BrowserProcess.ts @@ -11,31 +11,36 @@ import { arch } from 'os'; import ShutdownHandler from '@ulixee/commons/lib/ShutdownHandler'; import { PipeTransport } from './PipeTransport'; import env from '../env'; -import { PortTransport } from './PortTransport'; +import { WebsocketTransport } from './WebsocketTransport'; const { log } = Log(module); export default class BrowserProcess extends TypedEventEmitter<{ close: void }> { - public readonly transport: PipeTransport | PortTransport; + public readonly transport: PipeTransport | WebsocketTransport; public isProcessFunctionalPromise = new Resolvable(); public launchStderr: string[] = []; private processKilled = false; private readonly launchedProcess: ChildProcess; + private remoteDebuggingUrl?: Resolvable; - constructor(private browserEngine: IBrowserEngine, private processEnv?: NodeJS.ProcessEnv) { + constructor( + private browserEngine: IBrowserEngine, + private processEnv?: NodeJS.ProcessEnv, + ) { super(); bindFunctions(this); this.launchedProcess = this.launch(); this.bindProcessEvents(); - if (browserEngine.remoteDebuggingPort) { - this.transport = new PortTransport(browserEngine.remoteDebuggingPort); + if (browserEngine.useRemoteDebuggingPort) { + this.remoteDebuggingUrl = new Resolvable(); + this.transport = new WebsocketTransport(this.remoteDebuggingUrl.promise); } else { this.transport = new PipeTransport(this.launchedProcess); } - + this.transport.connectedPromise .then(() => this.isProcessFunctionalPromise.resolve(true)) .catch(err => setTimeout(() => this.isProcessFunctionalPromise.reject(err), 1.1e3)); @@ -94,6 +99,14 @@ export default class BrowserProcess extends TypedEventEmitter<{ close: void }> { }); readline.createInterface({ input: stderr }).on('line', line => { if (!line) return; + + if (this.remoteDebuggingUrl?.isResolved === false) { + const match = line.match(/DevTools listening on (.*)/); + if (match) { + this.remoteDebuggingUrl.resolve(match[1].trim()); + } + } + this.launchStderr.push(line); // don't grow in perpetuity! if (this.launchStderr.length > 100) { diff --git a/agent/main/lib/PortTransport.ts b/agent/main/lib/PortTransport.ts deleted file mode 100644 index 569dbcb13..000000000 --- a/agent/main/lib/PortTransport.ts +++ /dev/null @@ -1,69 +0,0 @@ -import Log from '@ulixee/commons/lib/Logger'; -import Resolvable from '@ulixee/commons/lib/Resolvable'; -import { w3cwebsocket as WebsocketClient } from 'websocket'; -import IConnectionTransport from '../interfaces/IConnectionTransport'; - -const { log } = Log(module); - -export class PortTransport implements IConnectionTransport { - isClosed = false; - - connectedPromise = new Resolvable(); - client: any; - - public onMessageFn: (message: string) => void; - public readonly onCloseFns: (() => void)[] = []; - - constructor(private readonly port: number) { - this.connect(this.port).catch(() => undefined); - } - - send(message: string): boolean { - if (this.client) { - this.client.send(message); - return true; - } - return false; - } - - close(): void { - if (this.isClosed) return; - this.isClosed = true; - } - - private emit(message): void { - if (this.onMessageFn) { - setImmediate(this.onMessageFn, message); - } - } - - private onData(message: string): void { - this.emit(message); - } - - private async connect(port: number): Promise { - try { - await new Promise(resolve => setTimeout(resolve, 1000)); - const response = await fetch(`http://localhost:${port}/json/version`); - const data = await response.json(); - const url = data.webSocketDebuggerUrl; - - const client = new WebsocketClient(url); - - client.onerror = () => { - if (!this.connectedPromise.isResolved) - this.connectedPromise.reject(new Error('Failed to connect')); - }; - client.onopen = () => { - this.client = client; - client.onmessage = e => { - const msgData = e.data; - this.onData(msgData); - }; - if (!this.connectedPromise.isResolved) this.connectedPromise.resolve(); - }; - } catch (error) { - if (!this.connectedPromise.isResolved) this.connectedPromise.reject(error); - } - } -} diff --git a/agent/main/lib/WebsocketTransport.ts b/agent/main/lib/WebsocketTransport.ts new file mode 100644 index 000000000..4a3df9b55 --- /dev/null +++ b/agent/main/lib/WebsocketTransport.ts @@ -0,0 +1,134 @@ +import Log from '@ulixee/commons/lib/Logger'; +import Resolvable from '@ulixee/commons/lib/Resolvable'; +import * as WebSocket from 'ws'; +import EventSubscriber from '@ulixee/commons/lib/EventSubscriber'; +import IConnectionTransport from '../interfaces/IConnectionTransport'; + + +// export class PortTransport implements IConnectionTransport { +// isClosed = false; + +// connectedPromise = new Resolvable(); +// client: any; + +// public onMessageFn: (message: string) => void; +// public readonly onCloseFns: (() => void)[] = []; + +// constructor(urlPromise: Promise) { +// urlPromise +// .then(url => this.connect(url)) +// .catch(error => { +// if (!this.connectedPromise.isResolved) this.connectedPromise.reject(error); +// }); +// } + +// send(message: string): boolean { +// if (this.client) { +// this.client.send(message); +// return true; +// } +// return false; +// } + +// close(): void { +// if (this.isClosed) return; +// this.isClosed = true; +// } + +// private emit(message): void { +// if (this.onMessageFn) { +// setImmediate(this.onMessageFn, message); +// } +// } + +// private onData(message: string): void { +// this.emit(message); +// } + +// private connect(url: string): void { +// const client = new WebSocket(url); + +// client.onerror = () => { +// if (!this.connectedPromise.isResolved) +// this.connectedPromise.reject(new Error('Failed to connect')); +// }; +// client.onopen = () => { +// this.client = client; +// client.onmessage = e => { +// const msgData = e.data; +// this.onData(msgData); +// }; +// if (!this.connectedPromise.isResolved) this.connectedPromise.resolve(); +// }; +// } +// } + +const { log } = Log(module); + +export class WebsocketTransport implements IConnectionTransport { + public get url(): string { + return this.webSocket.url; + } + + public onMessageFn: (message: string) => void; + public readonly onCloseFns: (() => void)[] = []; + public connectedPromise = new Resolvable(); + public isClosed = false; + + private events = new EventSubscriber(); + private webSocket?: WebSocket; + + constructor(urlPromise: Promise) { + urlPromise + .then(url => this.connect(url)) + .catch(error => { + if (!this.connectedPromise.isResolved) this.connectedPromise.reject(error); + }); + } + + send(message: string): boolean { + if (this.webSocket?.readyState === WebSocket.OPEN) { + this.webSocket.send(message); + return true; + } + return false; + } + + close(): void { + this.isClosed = true; + this.events.close(); + try { + this.webSocket?.close(); + } catch {} + } + + private onClosed(): void { + log.stats('WebSocketTransport.Closed'); + for (const close of this.onCloseFns) close(); + } + + private onMessage(event: string): void { + this.onMessageFn?.(event); + } + + private connect(url: string): void { + url = url.replace('localhost', '127.0.0.1'); + this.webSocket = new WebSocket(url, [], { + perMessageDeflate: false, + followRedirects: true, + }); + this.webSocket.once('open', this.connectedPromise.resolve); + this.webSocket.once('error', err => this.connectedPromise.reject(err, true)); + this.events.on(this.webSocket, 'message', this.onMessage.bind(this)); + this.events.once(this.webSocket, 'close', this.onClosed.bind(this)); + this.events.once(this.webSocket, 'error', error => { + if (!this.connectedPromise.isResolved) this.connectedPromise.reject(error, true); + if (this.isClosed) return; + if (error.code !== 'EPIPE') { + log.error('WebsocketTransport.error', { error, sessionId: null }); + } + }); + } +} + + diff --git a/agent/main/package.json b/agent/main/package.json index f02e344a6..6f289cc64 100644 --- a/agent/main/package.json +++ b/agent/main/package.json @@ -13,7 +13,7 @@ "devtools-protocol": "^0.0.1137505", "nanoid": "^3.3.6", "tough-cookie": "^4.1.3", - "websocket": "^1.0.34" + "ws": "^8.17.0" }, "devDependencies": { "@ulixee/unblocked-agent-testing": "2.0.0-alpha.28", diff --git a/specification/agent/browser/IBrowserEngine.ts b/specification/agent/browser/IBrowserEngine.ts index c121d41e5..b52c534fb 100644 --- a/specification/agent/browser/IBrowserEngine.ts +++ b/specification/agent/browser/IBrowserEngine.ts @@ -10,6 +10,6 @@ export default interface IBrowserEngine { isHeaded?: boolean; isHeadlessNew?: boolean; - remoteDebuggingPort?: number; + useRemoteDebuggingPort?: boolean; verifyLaunchable?(): Promise; } diff --git a/yarn.lock b/yarn.lock index ea5d2e109..60201bf17 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2851,13 +2851,6 @@ buffers@~0.1.1: resolved "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz" integrity sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ== -bufferutil@^4.0.1: - version "4.0.7" - resolved "https://registry.yarnpkg.com/bufferutil/-/bufferutil-4.0.7.tgz#60c0d19ba2c992dd8273d3f73772ffc894c153ad" - integrity sha512-kukuqc39WOHtdxtw4UScxF/WVnMFVSQVKhtx3AjZJzhd0RGZZldcrfSEbVsWWe6KNH253574cq5F+wpv0G9pJw== - dependencies: - node-gyp-build "^4.3.0" - builtins@^1.0.3: version "1.0.3" resolved "https://registry.npmjs.org/builtins/-/builtins-1.0.3.tgz" @@ -3505,14 +3498,6 @@ csv-parser@^2.3.2: minimist "^1.2.0" through2 "^3.0.1" -d@1, d@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/d/-/d-1.0.1.tgz#8698095372d58dbee346ffd0c7093f99f8f9eb5a" - integrity sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA== - dependencies: - es5-ext "^0.10.50" - type "^1.0.1" - damerau-levenshtein@^1.0.8: version "1.0.8" resolved "https://registry.npmjs.org/damerau-levenshtein/-/damerau-levenshtein-1.0.8.tgz" @@ -3549,7 +3534,7 @@ debug@4, debug@4.3.4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, de dependencies: ms "2.1.2" -debug@^2.2.0, debug@^2.6.9: +debug@^2.6.9: version "2.6.9" resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== @@ -3960,32 +3945,6 @@ es-to-primitive@^1.2.1: is-date-object "^1.0.1" is-symbol "^1.0.2" -es5-ext@^0.10.35, es5-ext@^0.10.50: - version "0.10.62" - resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.62.tgz#5e6adc19a6da524bf3d1e02bbc8960e5eb49a9a5" - integrity sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA== - dependencies: - es6-iterator "^2.0.3" - es6-symbol "^3.1.3" - next-tick "^1.1.0" - -es6-iterator@^2.0.3: - version "2.0.3" - resolved "https://registry.yarnpkg.com/es6-iterator/-/es6-iterator-2.0.3.tgz#a7de889141a05a94b0854403b2d0a0fbfa98f3b7" - integrity sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g== - dependencies: - d "1" - es5-ext "^0.10.35" - es6-symbol "^3.1.1" - -es6-symbol@^3.1.1, es6-symbol@^3.1.3: - version "3.1.3" - resolved "https://registry.yarnpkg.com/es6-symbol/-/es6-symbol-3.1.3.tgz#bad5d3c1bcdac28269f4cb331e431c78ac705d18" - integrity sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA== - dependencies: - d "^1.0.1" - ext "^1.1.2" - escalade@^3.1.1: version "3.1.1" resolved "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz" @@ -4320,13 +4279,6 @@ expect@^29.0.0, expect@^29.7.0: jest-message-util "^29.7.0" jest-util "^29.7.0" -ext@^1.1.2: - version "1.7.0" - resolved "https://registry.yarnpkg.com/ext/-/ext-1.7.0.tgz#0ea4383c0103d60e70be99e9a7f11027a33c4f5f" - integrity sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw== - dependencies: - type "^2.7.2" - extend@~3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" @@ -6760,11 +6712,6 @@ neo-async@^2.6.2: resolved "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz" integrity sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw== -next-tick@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.1.0.tgz#1836ee30ad56d67ef281b22bd199f709449b35eb" - integrity sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ== - node-fetch@2.6.7, node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz" @@ -6772,11 +6719,6 @@ node-fetch@2.6.7, node-fetch@^2.6.1, node-fetch@^2.6.7: dependencies: whatwg-url "^5.0.0" -node-gyp-build@^4.3.0: - version "4.8.0" - resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.8.0.tgz#3fee9c1731df4581a3f9ead74664369ff00d26dd" - integrity sha512-u6fs2AEUljNho3EYTJNBfImO5QTo/J/1Etd+NVdCj7qWKUSN/bSLkZwhDv7I+w/MSC6qJ4cknepkAYykDdK8og== - node-gyp@^5.0.2: version "5.1.1" resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-5.1.1.tgz#eb915f7b631c937d282e33aed44cb7a025f62a3e" @@ -8899,16 +8841,6 @@ type-is@^1.6.16, type-is@^1.6.4: media-typer "0.3.0" mime-types "~2.1.24" -type@^1.0.1: - version "1.2.0" - resolved "https://registry.yarnpkg.com/type/-/type-1.2.0.tgz#848dd7698dafa3e54a6c479e759c4bc3f18847a0" - integrity sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg== - -type@^2.7.2: - version "2.7.2" - resolved "https://registry.yarnpkg.com/type/-/type-2.7.2.tgz#2376a15a3a28b1efa0f5350dcf72d24df6ef98d0" - integrity sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw== - typed-array-buffer@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/typed-array-buffer/-/typed-array-buffer-1.0.0.tgz" @@ -9085,13 +9017,6 @@ url-parse@^1.5.3: querystringify "^2.1.1" requires-port "^1.0.0" -utf-8-validate@^5.0.2: - version "5.0.10" - resolved "https://registry.yarnpkg.com/utf-8-validate/-/utf-8-validate-5.0.10.tgz#d7d10ea39318171ca982718b6b96a8d2442571a2" - integrity sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ== - dependencies: - node-gyp-build "^4.3.0" - util-deprecate@^1.0.1, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" @@ -9184,18 +9109,6 @@ webidl-conversions@^6.1.0: resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-6.1.0.tgz#9111b4d7ea80acd40f5270d666621afa78b69514" integrity sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w== -websocket@^1.0.34: - version "1.0.34" - resolved "https://registry.yarnpkg.com/websocket/-/websocket-1.0.34.tgz#2bdc2602c08bf2c82253b730655c0ef7dcab3111" - integrity sha512-PRDso2sGwF6kM75QykIesBijKSVceR6jL2G8NGYyq2XrItNC2P5/qL5XeR056GhA+Ly7JMFvJb9I312mJfmqnQ== - dependencies: - bufferutil "^4.0.1" - debug "^2.2.0" - es5-ext "^0.10.50" - typedarray-to-buffer "^3.1.5" - utf-8-validate "^5.0.2" - yaeti "^0.0.6" - whatwg-url@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz" @@ -9387,6 +9300,11 @@ ws@^7.2.0, ws@^7.4.6, ws@^7.5.9: resolved "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz" integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== +ws@^8.17.0: + version "8.17.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.17.0.tgz#d145d18eca2ed25aaf791a183903f7be5e295fea" + integrity sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow== + xregexp@4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/xregexp/-/xregexp-4.0.0.tgz" @@ -9402,11 +9320,6 @@ y18n@^5.0.5: resolved "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== -yaeti@^0.0.6: - version "0.0.6" - resolved "https://registry.yarnpkg.com/yaeti/-/yaeti-0.0.6.tgz#f26f484d72684cf42bedfb76970aa1608fbf9577" - integrity sha512-MvQa//+KcZCUkBTIC9blM+CU9J2GzuTytsOUwf2lidtvkx/6gnEp1QvJv34t9vdjhFmha/mUiNDbN0D0mJWdug== - yallist@^3.0.0, yallist@^3.0.2, yallist@^3.1.1: version "3.1.1" resolved "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz"