From 939f1732a17582961f48b116b1c57fe0832a753b Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 11 Jul 2024 18:32:03 +0200 Subject: [PATCH 01/30] WIP --- package-lock.json | 70 ++++++ packages/devtools-proxy-support/.depcheckrc | 8 + packages/devtools-proxy-support/.eslintignore | 2 + packages/devtools-proxy-support/.eslintrc.js | 8 + packages/devtools-proxy-support/.mocharc.js | 1 + .../devtools-proxy-support/.prettierignore | 3 + .../devtools-proxy-support/.prettierrc.json | 1 + packages/devtools-proxy-support/LICENSE | 201 ++++++++++++++++++ packages/devtools-proxy-support/package.json | 67 ++++++ packages/devtools-proxy-support/src/agent.ts | 3 + packages/devtools-proxy-support/src/fetch.ts | 3 + .../devtools-proxy-support/src/index.spec.ts | 0 packages/devtools-proxy-support/src/index.ts | 10 + .../src/proxy-options.ts | 43 ++++ packages/devtools-proxy-support/src/tunnel.ts | 19 ++ .../devtools-proxy-support/tsconfig-lint.json | 5 + packages/devtools-proxy-support/tsconfig.json | 9 + 17 files changed, 453 insertions(+) create mode 100644 packages/devtools-proxy-support/.depcheckrc create mode 100644 packages/devtools-proxy-support/.eslintignore create mode 100644 packages/devtools-proxy-support/.eslintrc.js create mode 100644 packages/devtools-proxy-support/.mocharc.js create mode 100644 packages/devtools-proxy-support/.prettierignore create mode 100644 packages/devtools-proxy-support/.prettierrc.json create mode 100644 packages/devtools-proxy-support/LICENSE create mode 100644 packages/devtools-proxy-support/package.json create mode 100644 packages/devtools-proxy-support/src/agent.ts create mode 100644 packages/devtools-proxy-support/src/fetch.ts create mode 100644 packages/devtools-proxy-support/src/index.spec.ts create mode 100644 packages/devtools-proxy-support/src/index.ts create mode 100644 packages/devtools-proxy-support/src/proxy-options.ts create mode 100644 packages/devtools-proxy-support/src/tunnel.ts create mode 100644 packages/devtools-proxy-support/tsconfig-lint.json create mode 100644 packages/devtools-proxy-support/tsconfig.json diff --git a/package-lock.json b/package-lock.json index afc04f53..2e712d62 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5725,6 +5725,10 @@ "resolved": "packages/devtools-connect", "link": true }, + "node_modules/@mongodb-js/devtools-proxy-support": { + "resolved": "packages/devtools-proxy-support", + "link": true + }, "node_modules/@mongodb-js/devtools-scripts": { "resolved": "scripts", "link": true @@ -25193,6 +25197,42 @@ "integrity": "sha512-Rsk5qQHJ9eowMH28Jwhe8HEbmdYDX4lwoMWshiCXugjtHqMD9ZbiqSDLxcsfdqsETPzVUtX5s1Z5kStiIM6l4A==", "dev": true }, + "packages/devtools-proxy-support": { + "version": "0.1.0", + "license": "Apache-2.0", + "devDependencies": { + "@mongodb-js/eslint-config-devtools": "0.9.10", + "@mongodb-js/mocha-config-devtools": "^1.0.3", + "@mongodb-js/prettier-config-devtools": "^1.0.1", + "@mongodb-js/tsconfig-devtools": "^1.0.2", + "@types/chai": "^4.2.21", + "@types/mocha": "^9.1.1", + "@types/node": "^17.0.35", + "@types/sinon-chai": "^3.2.5", + "chai": "^4.3.6", + "depcheck": "^1.4.1", + "eslint": "^7.25.0", + "gen-esm-wrapper": "^1.1.0", + "mocha": "^8.4.0", + "nyc": "^15.1.0", + "prettier": "^2.3.2", + "sinon": "^9.2.3", + "typescript": "^5.0.4" + } + }, + "packages/devtools-proxy-support/node_modules/typescript": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", + "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "packages/dl-center": { "name": "@mongodb-js/dl-center", "version": "1.1.5", @@ -31574,6 +31614,36 @@ } } }, + "@mongodb-js/devtools-proxy-support": { + "version": "file:packages/devtools-proxy-support", + "requires": { + "@mongodb-js/eslint-config-devtools": "0.9.10", + "@mongodb-js/mocha-config-devtools": "^1.0.3", + "@mongodb-js/prettier-config-devtools": "^1.0.1", + "@mongodb-js/tsconfig-devtools": "^1.0.2", + "@types/chai": "^4.2.21", + "@types/mocha": "^9.1.1", + "@types/node": "^17.0.35", + "@types/sinon-chai": "^3.2.5", + "chai": "^4.3.6", + "depcheck": "^1.4.1", + "eslint": "^7.25.0", + "gen-esm-wrapper": "^1.1.0", + "mocha": "^8.4.0", + "nyc": "^15.1.0", + "prettier": "^2.3.2", + "sinon": "^9.2.3", + "typescript": "^5.0.4" + }, + "dependencies": { + "typescript": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", + "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", + "dev": true + } + } + }, "@mongodb-js/devtools-scripts": { "version": "file:scripts", "requires": { diff --git a/packages/devtools-proxy-support/.depcheckrc b/packages/devtools-proxy-support/.depcheckrc new file mode 100644 index 00000000..48bf9af6 --- /dev/null +++ b/packages/devtools-proxy-support/.depcheckrc @@ -0,0 +1,8 @@ +ignores: + - '@mongodb-js/prettier-config-devtools' + - '@mongodb-js/tsconfig-devtools' + - '@types/chai' + - '@types/sinon-chai' + - 'sinon' +ignore-patterns: + - 'dist' diff --git a/packages/devtools-proxy-support/.eslintignore b/packages/devtools-proxy-support/.eslintignore new file mode 100644 index 00000000..85a8a75e --- /dev/null +++ b/packages/devtools-proxy-support/.eslintignore @@ -0,0 +1,2 @@ +.nyc-output +dist diff --git a/packages/devtools-proxy-support/.eslintrc.js b/packages/devtools-proxy-support/.eslintrc.js new file mode 100644 index 00000000..83296d73 --- /dev/null +++ b/packages/devtools-proxy-support/.eslintrc.js @@ -0,0 +1,8 @@ +module.exports = { + root: true, + extends: ['@mongodb-js/eslint-config-devtools'], + parserOptions: { + tsconfigRootDir: __dirname, + project: ['./tsconfig-lint.json'], + }, +}; diff --git a/packages/devtools-proxy-support/.mocharc.js b/packages/devtools-proxy-support/.mocharc.js new file mode 100644 index 00000000..64afeb1f --- /dev/null +++ b/packages/devtools-proxy-support/.mocharc.js @@ -0,0 +1 @@ +module.exports = require('@mongodb-js/mocha-config-devtools'); diff --git a/packages/devtools-proxy-support/.prettierignore b/packages/devtools-proxy-support/.prettierignore new file mode 100644 index 00000000..4d28df66 --- /dev/null +++ b/packages/devtools-proxy-support/.prettierignore @@ -0,0 +1,3 @@ +.nyc_output +dist +coverage diff --git a/packages/devtools-proxy-support/.prettierrc.json b/packages/devtools-proxy-support/.prettierrc.json new file mode 100644 index 00000000..dfae21d0 --- /dev/null +++ b/packages/devtools-proxy-support/.prettierrc.json @@ -0,0 +1 @@ +"@mongodb-js/prettier-config-devtools" diff --git a/packages/devtools-proxy-support/LICENSE b/packages/devtools-proxy-support/LICENSE new file mode 100644 index 00000000..5e0fd33c --- /dev/null +++ b/packages/devtools-proxy-support/LICENSE @@ -0,0 +1,201 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, +and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by +the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all +other entities that control, are controlled by, or are under common +control with that entity. For the purposes of this definition, +"control" means (i) the power, direct or indirect, to cause the +direction or management of such entity, whether by contract or +otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity +exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, +including but not limited to software source code, documentation +source, and configuration files. + +"Object" form shall mean any form resulting from mechanical +transformation or translation of a Source form, including but +not limited to compiled object code, generated documentation, +and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or +Object form, made available under the License, as indicated by a +copyright notice that is included in or attached to the work +(an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object +form, that is based on (or derived from) the Work and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. For the purposes +of this License, Derivative Works shall not include works that remain +separable from, or merely link (or bind by name) to the interfaces of, +the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including +the original version of the Work and any modifications or additions +to that Work or Derivative Works thereof, that is intentionally +submitted to Licensor for inclusion in the Work by the copyright owner +or by an individual or Legal Entity authorized to submit on behalf of +the copyright owner. For the purposes of this definition, "submitted" +means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, +and issue tracking systems that are managed by, or on behalf of, the +Licensor for the purpose of discussing and improving the Work, but +excluding communication that is conspicuously marked or otherwise +designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity +on behalf of whom a Contribution has been received by Licensor and +subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the +Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except as stated in this section) patent license to make, have made, +use, offer to sell, sell, import, and otherwise transfer the Work, +where such license applies only to those patent claims licensable +by such Contributor that are necessarily infringed by their +Contribution(s) alone or by combination of their Contribution(s) +with the Work to which such Contribution(s) was submitted. If You +institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work +or a Contribution incorporated within the Work constitutes direct +or contributory patent infringement, then any patent licenses +granted to You under this License for that Work shall terminate +as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the +Work or Derivative Works thereof in any medium, with or without +modifications, and in Source or Object form, provided that You +meet the following conditions: + +(a) You must give any other recipients of the Work or +Derivative Works a copy of this License; and + +(b) You must cause any modified files to carry prominent notices +stating that You changed the files; and + +(c) You must retain, in the Source form of any Derivative Works +that You distribute, all copyright, patent, trademark, and +attribution notices from the Source form of the Work, +excluding those notices that do not pertain to any part of +the Derivative Works; and + +(d) If the Work includes a "NOTICE" text file as part of its +distribution, then any Derivative Works that You distribute must +include a readable copy of the attribution notices contained +within such NOTICE file, excluding those notices that do not +pertain to any part of the Derivative Works, in at least one +of the following places: within a NOTICE text file distributed +as part of the Derivative Works; within the Source form or +documentation, if provided along with the Derivative Works; or, +within a display generated by the Derivative Works, if and +wherever such third-party notices normally appear. The contents +of the NOTICE file are for informational purposes only and +do not modify the License. You may add Your own attribution +notices within Derivative Works that You distribute, alongside +or as an addendum to the NOTICE text from the Work, provided +that such additional attribution notices cannot be construed +as modifying the License. + +You may add Your own copyright statement to Your modifications and +may provide additional or different license terms and conditions +for use, reproduction, or distribution of Your modifications, or +for any such Derivative Works as a whole, provided Your use, +reproduction, and distribution of the Work otherwise complies with +the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, +any Contribution intentionally submitted for inclusion in the Work +by You to the Licensor shall be under the terms and conditions of +this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify +the terms of any separate license agreement you may have executed +with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade +names, trademarks, service marks, or product names of the Licensor, +except as required for reasonable and customary use in describing the +origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or +agreed to in writing, Licensor provides the Work (and each +Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied, including, without limitation, any warranties or conditions +of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +PARTICULAR PURPOSE. You are solely responsible for determining the +appropriateness of using or redistributing the Work and assume any +risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, +whether in tort (including negligence), contract, or otherwise, +unless required by applicable law (such as deliberate and grossly +negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, +incidental, or consequential damages of any character arising as a +result of this License or out of the use or inability to use the +Work (including but not limited to damages for loss of goodwill, +work stoppage, computer failure or malfunction, or any and all +other commercial damages or losses), even if such Contributor +has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing +the Work or Derivative Works thereof, You may choose to offer, +and charge a fee for, acceptance of support, warranty, indemnity, +or other liability obligations and/or rights consistent with this +License. However, in accepting such obligations, You may act only +on Your own behalf and on Your sole responsibility, not on behalf +of any other Contributor, and only if You agree to indemnify, +defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason +of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following +boilerplate notice, with the fields enclosed by brackets "{}" +replaced with your own identifying information. (Don't include +the brackets!) The text should be enclosed in the appropriate +comment syntax for the file format. We also recommend that a +file or class name and description of purpose be included on the +same "printed page" as the copyright notice for easier +identification within third-party archives. + +Copyright {yyyy} {name of copyright owner} + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/packages/devtools-proxy-support/package.json b/packages/devtools-proxy-support/package.json new file mode 100644 index 00000000..04c184fb --- /dev/null +++ b/packages/devtools-proxy-support/package.json @@ -0,0 +1,67 @@ +{ + "name": "@mongodb-js/devtools-proxy-support", + "description": "Proxy/tunneling support utilities for MongoDB Developer Tools", + "author": { + "name": "MongoDB Inc", + "email": "compass@mongodb.com" + }, + "publishConfig": { + "access": "public" + }, + "bugs": { + "url": "https://jira.mongodb.org/projects/COMPASS/issues", + "email": "compass@mongodb.com" + }, + "homepage": "https://github.com/mongodb-js/devtools-shared", + "version": "0.1.0", + "repository": { + "type": "git", + "url": "https://github.com/mongodb-js/devtools-shared.git" + }, + "files": [ + "dist" + ], + "license": "Apache-2.0", + "main": "dist/index.js", + "exports": { + "require": "./dist/index.js", + "import": "./dist/.esm-wrapper.mjs" + }, + "types": "./dist/index.d.ts", + "scripts": { + "bootstrap": "npm run compile", + "prepublishOnly": "npm run compile", + "compile": "tsc -p tsconfig.json && gen-esm-wrapper . ./dist/.esm-wrapper.mjs", + "typecheck": "tsc --noEmit", + "eslint": "eslint", + "prettier": "prettier", + "lint": "npm run eslint . && npm run prettier -- --check .", + "depcheck": "depcheck", + "check": "npm run typecheck && npm run lint && npm run depcheck", + "check-ci": "npm run check", + "test": "mocha", + "test-cov": "nyc -x \"**/*.spec.*\" --reporter=lcov --reporter=text --reporter=html npm run test", + "test-watch": "npm run test -- --watch", + "test-ci": "npm run test-cov", + "reformat": "npm run prettier -- --write ." + }, + "devDependencies": { + "@mongodb-js/eslint-config-devtools": "0.9.10", + "@mongodb-js/mocha-config-devtools": "^1.0.3", + "@mongodb-js/prettier-config-devtools": "^1.0.1", + "@mongodb-js/tsconfig-devtools": "^1.0.2", + "@types/chai": "^4.2.21", + "@types/mocha": "^9.1.1", + "@types/node": "^17.0.35", + "@types/sinon-chai": "^3.2.5", + "chai": "^4.3.6", + "depcheck": "^1.4.1", + "eslint": "^7.25.0", + "gen-esm-wrapper": "^1.1.0", + "mocha": "^8.4.0", + "nyc": "^15.1.0", + "prettier": "^2.3.2", + "sinon": "^9.2.3", + "typescript": "^5.0.4" + } +} diff --git a/packages/devtools-proxy-support/src/agent.ts b/packages/devtools-proxy-support/src/agent.ts new file mode 100644 index 00000000..6992b16c --- /dev/null +++ b/packages/devtools-proxy-support/src/agent.ts @@ -0,0 +1,3 @@ +export function createAgent( + proxyOptions: DevtoolsProxyOptions +): Agent | undefined {} diff --git a/packages/devtools-proxy-support/src/fetch.ts b/packages/devtools-proxy-support/src/fetch.ts new file mode 100644 index 00000000..b75db72e --- /dev/null +++ b/packages/devtools-proxy-support/src/fetch.ts @@ -0,0 +1,3 @@ +export function createFetch( + proxyOptions: DevtoolsProxyOptions +): (url: string, fetchOptions: FetchOptions) => Promise {} diff --git a/packages/devtools-proxy-support/src/index.spec.ts b/packages/devtools-proxy-support/src/index.spec.ts new file mode 100644 index 00000000..e69de29b diff --git a/packages/devtools-proxy-support/src/index.ts b/packages/devtools-proxy-support/src/index.ts new file mode 100644 index 00000000..57e1b69a --- /dev/null +++ b/packages/devtools-proxy-support/src/index.ts @@ -0,0 +1,10 @@ +export { + DevtoolsProxyOptions, + DevtoolsProxyOptionsSecrets, + translateToElectronProxyConfig, + extractProxySecrets, + mergeProxySecrets, +} from './proxy-options'; +export { Tunnel, setupSocks5Tunnel } from './tunnel'; +export { createAgent } from './agent'; +export { createFetch } from './fetch'; diff --git a/packages/devtools-proxy-support/src/proxy-options.ts b/packages/devtools-proxy-support/src/proxy-options.ts new file mode 100644 index 00000000..00ade3e8 --- /dev/null +++ b/packages/devtools-proxy-support/src/proxy-options.ts @@ -0,0 +1,43 @@ +// Should be an opaque type, but TS does not support those. +export type DevtoolsProxyOptionsSecrets = string; + +export interface DevtoolsProxyOptions { + // Can be an ssh://, socks5://, http://, https:// or pac<...>:// URL + // Everything besides ssh:// gets forwarded to the `proxy-agent` npm package + proxy?: string; + // With the semantics of the NO_PROXY environment variable, + // defaults to `localhost,127.0.0.1,::1` + noProxyHosts?: string; + // Takes effect if `proxy` was not specified. + useEnvironmentVariableProxies?: boolean; + sshOptions?: { + identityKeyFile?: string; + identityKeyPassphrase?: string; + }; +} + +// https://www.electronjs.org/docs/latest/api/structures/proxy-config +interface ElectronProxyConfig { + mode: 'direct' | 'auto_detect' | 'pac_script' | 'fixed_servers' | 'system'; + pacScript?: string; + proxyRules?: string; + proxyBypassRules?: string; +} + +export function translateToElectronProxyConfig( + proxyOptions: DevtoolsProxyOptions +): ElectronProxyConfig {} + +// These mirror our secrets extraction/merging logic in Compass +export function extractProxySecrets(proxyOptions: DevtoolsProxyOptions): { + proxyOptions: Partial; + secrets: DevtoolsProxyOptionsSecrets; +} {} + +export function mergeProxySecrets({ + proxyOptions, + secrets, +}: { + proxyOptions: Partial; + secrets: DevtoolsProxyOptionsSecrets; +}): DevtoolsProxyOptions {} diff --git a/packages/devtools-proxy-support/src/tunnel.ts b/packages/devtools-proxy-support/src/tunnel.ts new file mode 100644 index 00000000..259d5569 --- /dev/null +++ b/packages/devtools-proxy-support/src/tunnel.ts @@ -0,0 +1,19 @@ +export interface Tunnel { + on(ev: 'forwardingError', cb: (err: Error) => void): void; + on(ev: 'error', cb: (err: Error) => void): void; + + close(): Promise; + + options: { + // These can safely be assigned to driver MongoClientOptinos + proxyHost: string; + proxyPort: number; + proxyUsername: string; + proxyPassword: string; + }; +} + +export function setupSocks5Tunnel( + proxyOptions: DevtoolsProxyOptions, + target = 'mongodb://' +): Promise {} diff --git a/packages/devtools-proxy-support/tsconfig-lint.json b/packages/devtools-proxy-support/tsconfig-lint.json new file mode 100644 index 00000000..6bdef84f --- /dev/null +++ b/packages/devtools-proxy-support/tsconfig-lint.json @@ -0,0 +1,5 @@ +{ + "extends": "./tsconfig.json", + "include": ["**/*"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/devtools-proxy-support/tsconfig.json b/packages/devtools-proxy-support/tsconfig.json new file mode 100644 index 00000000..21899b27 --- /dev/null +++ b/packages/devtools-proxy-support/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "@mongodb-js/tsconfig-devtools/tsconfig.common.json", + "compilerOptions": { + "outDir": "dist", + "allowJs": true + }, + "include": ["src/**/*"], + "exclude": ["./src/**/*.spec.*"] +} From 74404cde0a20f440b1c17d55bf0ac4a74181d858 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 11 Jul 2024 19:22:12 +0200 Subject: [PATCH 02/30] WIP --- packages/devtools-proxy-support/src/proxy-options.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/devtools-proxy-support/src/proxy-options.ts b/packages/devtools-proxy-support/src/proxy-options.ts index 00ade3e8..c244fa30 100644 --- a/packages/devtools-proxy-support/src/proxy-options.ts +++ b/packages/devtools-proxy-support/src/proxy-options.ts @@ -26,7 +26,10 @@ interface ElectronProxyConfig { export function translateToElectronProxyConfig( proxyOptions: DevtoolsProxyOptions -): ElectronProxyConfig {} +): ElectronProxyConfig { + if (proxyOptions.proxy) { + } +} // These mirror our secrets extraction/merging logic in Compass export function extractProxySecrets(proxyOptions: DevtoolsProxyOptions): { From a0a30c026868a1a669a7a54c8fa6a68eb13abb14 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 18 Jul 2024 15:41:59 +0200 Subject: [PATCH 03/30] WIP --- package-lock.json | 581 +++++++++++++++++- packages/devtools-proxy-support/package.json | 6 + packages/devtools-proxy-support/src/agent.ts | 34 +- .../src/env-var-proxies.ts | 0 packages/devtools-proxy-support/src/fetch.ts | 12 +- packages/devtools-proxy-support/src/index.ts | 2 +- .../src/proxy-options.ts | 193 +++++- .../devtools-proxy-support/src/socksv5.d.ts | 12 + packages/devtools-proxy-support/src/ssh.ts | 115 ++++ packages/devtools-proxy-support/src/tunnel.ts | 348 ++++++++++- packages/devtools-proxy-support/tsconfig.json | 3 +- 11 files changed, 1261 insertions(+), 45 deletions(-) create mode 100644 packages/devtools-proxy-support/src/env-var-proxies.ts create mode 100644 packages/devtools-proxy-support/src/socksv5.d.ts create mode 100644 packages/devtools-proxy-support/src/ssh.ts diff --git a/package-lock.json b/package-lock.json index 2e712d62..09f55fb9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6796,6 +6796,11 @@ "node": ">= 6" } }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==" + }, "node_modules/@tsconfig/node10": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", @@ -8334,11 +8339,27 @@ "node": "*" } }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/ast-types-flow": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", "integrity": "sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag==" }, + "node_modules/ast-types/node_modules/tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" + }, "node_modules/astral-regex": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", @@ -8559,6 +8580,14 @@ } ] }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -10017,6 +10046,14 @@ "node": ">=0.10" } }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "engines": { + "node": ">= 14" + } + }, "node_modules/data-urls": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz", @@ -10595,6 +10632,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -11309,6 +11359,34 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/escodegen/node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/eslint": { "version": "7.32.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.32.0.tgz", @@ -12635,10 +12713,9 @@ "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "node_modules/fs-extra": { - "version": "11.1.1", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz", - "integrity": "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==", - "dev": true, + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -13197,6 +13274,20 @@ "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" } }, + "node_modules/get-uri": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.3.tgz", + "integrity": "sha512-BzUrJBS9EcUb4cFol8r4W3v1cPsSyajLSthNkz5BxbpDcHN5tIrM10E2eNvfnvBn3DaT3DUgx0OpsBKkaOpanw==", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4", + "fs-extra": "^11.2.0" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -15331,7 +15422,6 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", - "dev": true, "dependencies": { "universalify": "^2.0.0" }, @@ -18275,6 +18365,14 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "dev": true }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/next-tick": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", @@ -19726,6 +19824,84 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/pac-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.2.tgz", + "integrity": "sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg==", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.5", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-proxy-agent/node_modules/socks-proxy-agent": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz", + "integrity": "sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==", + "dependencies": { + "agent-base": "^7.1.1", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/package-hash": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/package-hash/-/package-hash-4.0.0.tgz", @@ -20469,11 +20645,84 @@ "node": ">= 0.10" } }, + "node_modules/proxy-agent": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.4.0.tgz", + "integrity": "sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.3", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.0.1", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-agent/node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/proxy-agent/node_modules/socks-proxy-agent": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz", + "integrity": "sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==", + "dependencies": { + "agent-base": "^7.1.1", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "dev": true + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, "node_modules/psl": { "version": "1.9.0", @@ -22254,9 +22503,9 @@ "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" }, "node_modules/socks": { - "version": "2.7.3", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.3.tgz", - "integrity": "sha512-vfuYK48HXCTFD03G/1/zkIls3Ebr2YNa4qU9gHDZdblHLiqhJrJGkY3+0Nx0JpN9qBhJbVObc1CNciT1bIZJxw==", + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", + "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", "dependencies": { "ip-address": "^9.0.5", "smart-buffer": "^4.2.0" @@ -22280,6 +22529,44 @@ "node": ">= 10" } }, + "node_modules/socksv5": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/socksv5/-/socksv5-0.0.6.tgz", + "integrity": "sha512-tQpQ0MdNQAsQBDhCXy3OvGGJikh9QOl3PkbwT4POJiQCm/fK4z9AxKQQRG8WLeF6talphnPrSWiZRpTl42rApg==", + "bundleDependencies": [ + "ipv6" + ], + "dependencies": { + "ipv6": "*" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/socksv5/node_modules/ipv6": { + "version": "3.1.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "cli": "0.4.x", + "cliff": "0.1.x", + "sprintf": "0.1.x" + }, + "bin": { + "ipv6": "bin/ipv6.js", + "ipv6grep": "bin/ipv6grep.js" + }, + "engines": { + "node": "*" + } + }, + "node_modules/socksv5/node_modules/ipv6/node_modules/sprintf": { + "version": "0.1.3", + "inBundle": true, + "engines": { + "node": ">=0.2.4" + } + }, "node_modules/sort-keys": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-2.0.0.tgz", @@ -22296,7 +22583,7 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, + "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -23580,7 +23867,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", - "dev": true, "engines": { "node": ">= 10.0.0" } @@ -25198,8 +25484,15 @@ "dev": true }, "packages/devtools-proxy-support": { + "name": "@mongodb-js/devtools-proxy-support", "version": "0.1.0", "license": "Apache-2.0", + "dependencies": { + "agent-base": "^7.1.1", + "proxy-agent": "^6.4.0", + "socksv5": "^0.0.6", + "ssh2": "^1.15.0" + }, "devDependencies": { "@mongodb-js/eslint-config-devtools": "0.9.10", "@mongodb-js/mocha-config-devtools": "^1.0.3", @@ -25220,6 +25513,17 @@ "typescript": "^5.0.4" } }, + "packages/devtools-proxy-support/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, "packages/devtools-proxy-support/node_modules/typescript": { "version": "5.5.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", @@ -31625,6 +31929,7 @@ "@types/mocha": "^9.1.1", "@types/node": "^17.0.35", "@types/sinon-chai": "^3.2.5", + "agent-base": "^7.1.1", "chai": "^4.3.6", "depcheck": "^1.4.1", "eslint": "^7.25.0", @@ -31632,10 +31937,21 @@ "mocha": "^8.4.0", "nyc": "^15.1.0", "prettier": "^2.3.2", + "proxy-agent": "^6.4.0", "sinon": "^9.2.3", + "socksv5": "^0.0.6", + "ssh2": "^1.15.0", "typescript": "^5.0.4" }, "dependencies": { + "agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "requires": { + "debug": "^4.3.4" + } + }, "typescript": { "version": "5.5.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", @@ -33252,6 +33568,11 @@ "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==" }, + "@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==" + }, "@tsconfig/node10": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", @@ -34536,6 +34857,21 @@ "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==" }, + "ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "requires": { + "tslib": "^2.0.1" + }, + "dependencies": { + "tslib": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" + } + } + }, "ast-types-flow": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", @@ -34711,6 +35047,11 @@ "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, + "basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==" + }, "bcrypt-pbkdf": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", @@ -35813,6 +36154,11 @@ "assert-plus": "^1.0.0" } }, + "data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==" + }, "data-urls": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz", @@ -36226,6 +36572,16 @@ "object-keys": "^1.1.1" } }, + "degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "requires": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + } + }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -36792,6 +37148,24 @@ "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", "dev": true }, + "escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "requires": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2", + "source-map": "~0.6.1" + }, + "dependencies": { + "estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==" + } + } + }, "eslint": { "version": "7.32.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.32.0.tgz", @@ -37783,10 +38157,9 @@ "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "fs-extra": { - "version": "11.1.1", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz", - "integrity": "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==", - "dev": true, + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", "requires": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -38207,6 +38580,17 @@ "resolve-pkg-maps": "^1.0.0" } }, + "get-uri": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.3.tgz", + "integrity": "sha512-BzUrJBS9EcUb4cFol8r4W3v1cPsSyajLSthNkz5BxbpDcHN5tIrM10E2eNvfnvBn3DaT3DUgx0OpsBKkaOpanw==", + "requires": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4", + "fs-extra": "^11.2.0" + } + }, "getpass": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", @@ -39789,7 +40173,6 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", - "dev": true, "requires": { "graceful-fs": "^4.1.6", "universalify": "^2.0.0" @@ -42228,6 +42611,11 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "dev": true }, + "netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==" + }, "next-tick": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", @@ -43325,6 +43713,68 @@ "p-reduce": "^2.0.0" } }, + "pac-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.0.2.tgz", + "integrity": "sha512-BFi3vZnO9X5Qt6NRz7ZOaPja3ic0PhlsmCRYLOpN11+mWBCR6XJDqW5RF3j8jm4WGGQZtBA+bTfxYzeKW73eHg==", + "requires": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.5", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.4" + }, + "dependencies": { + "agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "requires": { + "debug": "^4.3.4" + } + }, + "http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "requires": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + } + }, + "https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "requires": { + "agent-base": "^7.0.2", + "debug": "4" + } + }, + "socks-proxy-agent": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz", + "integrity": "sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==", + "requires": { + "agent-base": "^7.1.1", + "debug": "^4.3.4", + "socks": "^2.8.3" + } + } + } + }, + "pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "requires": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + } + }, "package-hash": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/package-hash/-/package-hash-4.0.0.tgz", @@ -43888,11 +44338,68 @@ } } }, + "proxy-agent": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.4.0.tgz", + "integrity": "sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ==", + "requires": { + "agent-base": "^7.0.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.3", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.0.1", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.2" + }, + "dependencies": { + "agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "requires": { + "debug": "^4.3.4" + } + }, + "http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "requires": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + } + }, + "https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "requires": { + "agent-base": "^7.0.2", + "debug": "4" + } + }, + "lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==" + }, + "socks-proxy-agent": { + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.4.tgz", + "integrity": "sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==", + "requires": { + "agent-base": "^7.1.1", + "debug": "^4.3.4", + "socks": "^2.8.3" + } + } + } + }, "proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", - "dev": true + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, "psl": { "version": "1.9.0", @@ -45255,9 +45762,9 @@ } }, "socks": { - "version": "2.7.3", - "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.3.tgz", - "integrity": "sha512-vfuYK48HXCTFD03G/1/zkIls3Ebr2YNa4qU9gHDZdblHLiqhJrJGkY3+0Nx0JpN9qBhJbVObc1CNciT1bIZJxw==", + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.3.tgz", + "integrity": "sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==", "requires": { "ip-address": "^9.0.5", "smart-buffer": "^4.2.0" @@ -45274,6 +45781,31 @@ "socks": "^2.6.2" } }, + "socksv5": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/socksv5/-/socksv5-0.0.6.tgz", + "integrity": "sha512-tQpQ0MdNQAsQBDhCXy3OvGGJikh9QOl3PkbwT4POJiQCm/fK4z9AxKQQRG8WLeF6talphnPrSWiZRpTl42rApg==", + "requires": { + "ipv6": "*" + }, + "dependencies": { + "ipv6": { + "version": "3.1.1", + "bundled": true, + "requires": { + "cli": "0.4.x", + "cliff": "0.1.x", + "sprintf": "0.1.x" + }, + "dependencies": { + "sprintf": { + "version": "0.1.3", + "bundled": true + } + } + } + } + }, "sort-keys": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-2.0.0.tgz", @@ -45287,7 +45819,7 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true + "devOptional": true }, "source-map-js": { "version": "1.0.2", @@ -46276,8 +46808,7 @@ "universalify": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", - "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", - "dev": true + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==" }, "unpipe": { "version": "1.0.0", diff --git a/packages/devtools-proxy-support/package.json b/packages/devtools-proxy-support/package.json index 04c184fb..7f20960e 100644 --- a/packages/devtools-proxy-support/package.json +++ b/packages/devtools-proxy-support/package.json @@ -45,6 +45,12 @@ "test-ci": "npm run test-cov", "reformat": "npm run prettier -- --write ." }, + "dependencies": { + "proxy-agent": "^6.4.0", + "agent-base": "^7.1.1", + "ssh2": "^1.15.0", + "socksv5": "^0.0.6" + }, "devDependencies": { "@mongodb-js/eslint-config-devtools": "0.9.10", "@mongodb-js/mocha-config-devtools": "^1.0.3", diff --git a/packages/devtools-proxy-support/src/agent.ts b/packages/devtools-proxy-support/src/agent.ts index 6992b16c..6f590734 100644 --- a/packages/devtools-proxy-support/src/agent.ts +++ b/packages/devtools-proxy-support/src/agent.ts @@ -1,3 +1,35 @@ +import { ProxyAgent } from 'proxy-agent'; +import type { Agent } from 'https'; +import type { DevtoolsProxyOptions } from './proxy-options'; +import { proxyForUrl } from './proxy-options'; +import type { ClientRequest } from 'http'; +import type { TcpNetConnectOpts } from 'net'; +import type { ConnectionOptions } from 'tls'; +import type { Duplex } from 'stream'; +import { SSHAgent } from './ssh'; + +export type AgentWithInitialize = Agent & { + // This is genuinely custom for our usage (to allow establishing an SSH tunnel + // first before starting to push connections through it) + initialize?(): Promise; + + // This is just part of the regular Agent interface, used by Node.js itself, + // but missing from @types/node + createSocket( + req: ClientRequest, + options: TcpNetConnectOpts | ConnectionOptions, + cb: (err: Error | null, s?: Duplex) => void + ): void; +}; + export function createAgent( proxyOptions: DevtoolsProxyOptions -): Agent | undefined {} +): AgentWithInitialize { + if (proxyOptions.proxy && new URL(proxyOptions.proxy).protocol === 'ssh:') { + return new SSHAgent(proxyOptions); + } + const getProxyForUrl = proxyForUrl(proxyOptions); + return new ProxyAgent({ + getProxyForUrl, + }); +} diff --git a/packages/devtools-proxy-support/src/env-var-proxies.ts b/packages/devtools-proxy-support/src/env-var-proxies.ts new file mode 100644 index 00000000..e69de29b diff --git a/packages/devtools-proxy-support/src/fetch.ts b/packages/devtools-proxy-support/src/fetch.ts index b75db72e..c88f8679 100644 --- a/packages/devtools-proxy-support/src/fetch.ts +++ b/packages/devtools-proxy-support/src/fetch.ts @@ -1,3 +1,13 @@ +import type { RequestInit, Response } from 'node-fetch'; +import fetch from 'node-fetch'; +import { createAgent } from './agent'; +import type { DevtoolsProxyOptions } from './proxy-options'; + export function createFetch( proxyOptions: DevtoolsProxyOptions -): (url: string, fetchOptions: FetchOptions) => Promise {} +): (url: string, fetchOptions?: RequestInit) => Promise { + const agent = createAgent(proxyOptions); + return async (url, fetchOptions) => { + return await fetch(url, { agent, ...fetchOptions }); + }; +} diff --git a/packages/devtools-proxy-support/src/index.ts b/packages/devtools-proxy-support/src/index.ts index 57e1b69a..ed2541ef 100644 --- a/packages/devtools-proxy-support/src/index.ts +++ b/packages/devtools-proxy-support/src/index.ts @@ -5,6 +5,6 @@ export { extractProxySecrets, mergeProxySecrets, } from './proxy-options'; -export { Tunnel, setupSocks5Tunnel } from './tunnel'; +export { Tunnel, TunnelOptions, setupSocks5Tunnel } from './tunnel'; export { createAgent } from './agent'; export { createFetch } from './fetch'; diff --git a/packages/devtools-proxy-support/src/proxy-options.ts b/packages/devtools-proxy-support/src/proxy-options.ts index c244fa30..f1dd008f 100644 --- a/packages/devtools-proxy-support/src/proxy-options.ts +++ b/packages/devtools-proxy-support/src/proxy-options.ts @@ -18,29 +18,210 @@ export interface DevtoolsProxyOptions { // https://www.electronjs.org/docs/latest/api/structures/proxy-config interface ElectronProxyConfig { - mode: 'direct' | 'auto_detect' | 'pac_script' | 'fixed_servers' | 'system'; + mode?: 'direct' | 'auto_detect' | 'pac_script' | 'fixed_servers' | 'system'; pacScript?: string; proxyRules?: string; proxyBypassRules?: string; } +function proxyConfForEnvVars( + env: Record = process.env +): { map: Map; noProxy: string } { + const map = new Map(); + let noProxy = ''; + for (const [_key, value] of Object.entries(env)) { + if (value === undefined) continue; + const key = _key.toUpperCase(); + if (key.endsWith('_PROXY') && key !== 'NO_PROXY') { + map.set(key.replace(/_PROXY$/, '').toLowerCase(), value || 'direct://'); + } + if (key === 'NO_PROXY') noProxy = value; + } + return { map, noProxy }; +} + +function shouldProxy(noProxy: string, url: URL): boolean { + if (!noProxy) return true; + if (noProxy === '*') return false; + for (const noProxyItem of noProxy.split(/[\s,]/)) { + let { host, port } = + noProxyItem.match(/(?.+)(:(?\d+)$)?/)?.groups ?? {}; + if (!host) { + host = noProxyItem; + port = ''; + } + if (port && url.port !== port) continue; + if ( + host === url.hostname || + (host.startsWith('*') && url.hostname.endsWith(host.substring(1))) + ) + return false; + } + return true; +} + +export function proxyForUrl( + proxyOptions: DevtoolsProxyOptions +): (url: string) => string { + if (proxyOptions.proxy) { + const proxyUrl = proxyOptions.proxy; + if (new URL(proxyUrl).protocol === 'direct:') return () => ''; + return (target: string) => { + if (shouldProxy(proxyOptions.noProxyHosts || '', new URL(target))) { + return proxyUrl; + } + return ''; + }; + } + + if (proxyOptions.useEnvironmentVariableProxies) { + const { map, noProxy } = proxyConfForEnvVars(); + return (target: string) => { + const url = new URL(target); + const protocol = url.protocol.replace(/:$/, ''); + const combinedNoProxyRules = [noProxy, proxyOptions.noProxyHosts] + .filter(Boolean) + .join(','); + const proxyForProtocol = map.get(protocol); + if (proxyForProtocol && shouldProxy(combinedNoProxyRules, url)) { + return proxyForProtocol; + } + return ''; + }; + } + + return () => ''; +} + export function translateToElectronProxyConfig( proxyOptions: DevtoolsProxyOptions ): ElectronProxyConfig { if (proxyOptions.proxy) { + const url = new URL(proxyOptions.proxy); + if (url.protocol === 'ssh:') { + throw new Error( + `Using ssh:// proxies for generic browser proxy usage is not supported (translating '${redactUrl( + url + )}')` + ); + } + if (url.username || url.password) { + throw new Error( + `Using authenticated proxies for generic browser proxy usage is not supported (translating '${redactUrl( + url + )}')` + ); + } + if (url.protocol.startsWith('pac+')) { + url.protocol = url.protocol.replace('pac+', ''); + return { + mode: 'pac_script', + pacScript: url.toString(), + proxyBypassRules: proxyOptions.noProxyHosts, + }; + } + if ( + url.protocol !== 'http:' && + url.protocol !== 'https:' && + url.protocol !== 'socks5:' + ) { + throw new Error( + `Unsupported proxy protocol (translating '${redactUrl(url)}')` + ); + } + return { + mode: 'fixed_servers', + proxyRules: url.toString(), + proxyBypassRules: proxyOptions.noProxyHosts, + }; + } + + if (proxyOptions.useEnvironmentVariableProxies) { + const proxyRules: string[] = []; + const proxyBypassRules = [proxyOptions.noProxyHosts]; + const { map, noProxy } = proxyConfForEnvVars(); + for (const [key, value] of map) proxyBypassRules.push(`${key}=${value}`); + proxyBypassRules.push(noProxy); + + return { + mode: 'fixed_servers', + proxyBypassRules: proxyBypassRules.filter(Boolean).join(',') || undefined, + proxyRules: proxyRules.join(';'), + }; } + + return {}; +} + +interface DevtoolsProxyOptionsSecretsInternal { + username?: string; + password?: string; + sshIdentityKeyPassphrase?: string; } // These mirror our secrets extraction/merging logic in Compass -export function extractProxySecrets(proxyOptions: DevtoolsProxyOptions): { - proxyOptions: Partial; +export function extractProxySecrets( + proxyOptions: Readonly +): { + proxyOptions: DevtoolsProxyOptions; secrets: DevtoolsProxyOptionsSecrets; -} {} +} { + const secrets: DevtoolsProxyOptionsSecretsInternal = {}; + if (proxyOptions.proxy) { + const proxyUrl = new URL(proxyOptions.proxy); + ({ username: secrets.username, password: secrets.password } = proxyUrl); + proxyUrl.username = proxyUrl.password = ''; + proxyOptions = { ...proxyOptions, proxy: proxyUrl.toString() }; + } + if (proxyOptions.sshOptions) { + secrets.sshIdentityKeyPassphrase = + proxyOptions.sshOptions.identityKeyPassphrase; + proxyOptions = { + ...proxyOptions, + sshOptions: { + ...proxyOptions.sshOptions, + identityKeyPassphrase: undefined, + }, + }; + } + return { + secrets: JSON.stringify(secrets), + proxyOptions: proxyOptions, + }; +} export function mergeProxySecrets({ proxyOptions, secrets, }: { - proxyOptions: Partial; + proxyOptions: Readonly; secrets: DevtoolsProxyOptionsSecrets; -}): DevtoolsProxyOptions {} +}): DevtoolsProxyOptions { + const parsedSecrets: DevtoolsProxyOptionsSecretsInternal = + JSON.parse(secrets); + if ( + (parsedSecrets.username || parsedSecrets.password) && + proxyOptions.proxy + ) { + const proxyUrl = new URL(proxyOptions.proxy); + proxyUrl.username = parsedSecrets.username || ''; + proxyUrl.password = parsedSecrets.password || ''; + proxyOptions = { ...proxyOptions, proxy: proxyUrl.toString() }; + } + if (parsedSecrets.sshIdentityKeyPassphrase) { + proxyOptions = { + ...proxyOptions, + sshOptions: { + ...proxyOptions.sshOptions, + identityKeyPassphrase: parsedSecrets.sshIdentityKeyPassphrase, + }, + }; + } + return proxyOptions; +} + +function redactUrl(urlOrString: URL | string): string { + const url = new URL(urlOrString.toString()); + url.username = url.password = '(credential)'; + return url.toString(); +} diff --git a/packages/devtools-proxy-support/src/socksv5.d.ts b/packages/devtools-proxy-support/src/socksv5.d.ts new file mode 100644 index 00000000..9c9cc3ab --- /dev/null +++ b/packages/devtools-proxy-support/src/socksv5.d.ts @@ -0,0 +1,12 @@ +declare module 'socksv5/lib/server' { + const mod: any; + export = mod; +} +declare module 'socksv5/lib/auth/None' { + const mod: any; + export = mod; +} +declare module 'socksv5/lib/auth/UserPassword' { + const mod: any; + export = mod; +} diff --git a/packages/devtools-proxy-support/src/ssh.ts b/packages/devtools-proxy-support/src/ssh.ts new file mode 100644 index 00000000..73a894f7 --- /dev/null +++ b/packages/devtools-proxy-support/src/ssh.ts @@ -0,0 +1,115 @@ +import { Agent as AgentBase } from 'agent-base'; +import type { DevtoolsProxyOptions } from './proxy-options'; +import type { AgentWithInitialize } from './agent'; +import type { ClientRequest } from 'http'; +import type { Duplex } from 'stream'; +import type { ClientChannel, ConnectConfig } from 'ssh2'; +import { Client as SshClient } from 'ssh2'; +import { once } from 'events'; +import { promises as fs } from 'fs'; + +export class SSHAgent extends AgentBase implements AgentWithInitialize { + private readonly proxyOptions: Readonly; + private readonly url: URL; + private sshClient: SshClient; + private connected = false; + private connectingPromise?: Promise; + private closed = false; + private forwardOut: ( + srcIP: string, + srcPort: number, + dstIP: string, + dstPort: number + ) => Promise; + + constructor(options: DevtoolsProxyOptions) { + super(); + this.proxyOptions = options; + this.url = new URL(options.proxy ?? ''); + this.sshClient = new SshClient(); + this.sshClient.on('close', () => { + log.info(mongoLogId(1_001_000_252), this.logCtx, 'sshClient closed'); + this.connected = false; + }); + + this.forwardOut = promisify(this.sshClient.forwardOut.bind(this.sshClient)); + } + + async initialize(): Promise { + if (this.connected) { + debug('already connected'); + return; + } + + if (this.connectingPromise) { + debug('reusing connectingPromise'); + return this.connectingPromise; + } + + if (this.closed) { + // A socks5 request could come in after we deliberately closed the connection. Don't reconnect in that case. + throw new Error('Disconnected.'); + } + + const sshConnectConfig: ConnectConfig = { + readyTimeout: 20000, + keepaliveInterval: 20000, + host: this.url.hostname, + port: +this.url.port || 22, + username: this.url.username || undefined, + password: this.url.password || undefined, + privateKey: this.proxyOptions.sshOptions?.identityKeyFile + ? await fs.readFile(this.proxyOptions.sshOptions.identityKeyFile) + : undefined, + passphrase: this.proxyOptions.sshOptions?.identityKeyPassphrase, + }; + + log.info( + mongoLogId(1_001_000_257), + this.logCtx, + 'Establishing new SSH connection' + ); + + this.connectingPromise = Promise.race([ + once(this.sshClient, 'error').then(([err]) => { + throw err; + }), + (() => { + const waitForReady = once(this.sshClient, 'ready').then(() => {}); // eslint-disable-line @typescript-eslint/no-empty-function + this.sshClient.connect(sshConnectConfig); + return waitForReady; + })(), + ]); + + try { + await this.connectingPromise; + } catch (err) { + this.emit('forwardingError', err); + log.error( + mongoLogId(1_001_000_258), + this.logCtx, + 'Failed to establish new SSH connection', + { error: (err as any)?.stack ?? String(err) } + ); + delete this.connectingPromise; + throw err; + } + + delete this.connectingPromise; + this.connected = true; + log.info( + mongoLogId(1_001_000_259), + this.logCtx, + 'Finished establishing new SSH connection' + ); + } + + override async connect(req: ClientRequest): Promise { + await this.initialize(); + + const host = req.getHeader('host') as string; + const url = new URL(req.path, `tcp://${host}`); + + return await this.forwardOut('127.0.0.1', 0, url.hostname, +url.port); + } +} diff --git a/packages/devtools-proxy-support/src/tunnel.ts b/packages/devtools-proxy-support/src/tunnel.ts index 259d5569..be89ffae 100644 --- a/packages/devtools-proxy-support/src/tunnel.ts +++ b/packages/devtools-proxy-support/src/tunnel.ts @@ -1,19 +1,347 @@ +import { EventEmitter, once } from 'events'; +import type { DevtoolsProxyOptions } from './proxy-options'; +import { proxyForUrl } from './proxy-options'; +import type { Agent } from 'https'; +import type { AgentWithInitialize } from './agent'; +import { createAgent } from './agent'; + +// The socksv5 module is not bundle-able by itself, so we get the +// subpackages directly +import socks5Server from 'socksv5/lib/server'; +import socks5AuthNone from 'socksv5/lib/auth/None'; +import socks5AuthUserPassword from 'socksv5/lib/auth/UserPassword'; +import { promisify } from 'util'; +import { isIPv6 } from 'net'; +import type { Socket } from 'net'; +import type { Duplex } from 'stream'; +import type { ClientRequest } from 'http'; + +export interface TunnelOptions { + // These can safely be assigned to driver MongoClientOptinos + proxyHost: string; + proxyPort: number; + proxyUsername?: string; + proxyPassword?: string; +} + +function getTunnelOptions(config: Partial): TunnelOptions { + return { + proxyHost: '127.0.0.1', + proxyPort: 0, + proxyUsername: undefined, + proxyPassword: undefined, + ...config, + }; +} + +type ErrorWithOrigin = Error & { origin?: string }; + export interface Tunnel { on(ev: 'forwardingError', cb: (err: Error) => void): void; on(ev: 'error', cb: (err: Error) => void): void; close(): Promise; - options: { - // These can safely be assigned to driver MongoClientOptinos - proxyHost: string; - proxyPort: number; - proxyUsername: string; - proxyPassword: string; - }; + readonly config: Readonly; +} + +function createFakeHttpClientRequest(dstAddr: string, dstPort: number) { + return { + host: dstAddr, + protocol: 'http', + method: 'GET', + path: '/', + getHeader(name) { + return name === 'host' + ? `${isIPv6(dstAddr) ? `[${dstAddr}]` : dstAddr}:${dstPort}` + : undefined; + }, + } as ClientRequest; } -export function setupSocks5Tunnel( - proxyOptions: DevtoolsProxyOptions, +let idCounter = 0; +class Socks5Server extends EventEmitter implements Tunnel { + private readonly agent: AgentWithInitialize; + private server: any; + private serverListen: (port?: number, host?: string) => Promise; + private serverClose: () => Promise; + private connections: Set = new Set(); + private logCtx = `tunnel-${idCounter++}`; + private rawConfig: TunnelOptions; + private closed = false; + private agentInitialized = false; + private agentInitPromise?: Promise; + + constructor(agent: Agent, tunnelOptions: Partial) { + super(); + this.agent = agent; + this.rawConfig = getTunnelOptions(tunnelOptions); + + this.server = socks5Server.createServer(this.socks5Request.bind(this)); + + if (this.rawConfig.proxyUsername) { + this.server.useAuth( + socks5AuthUserPassword( + (user: string, pass: string, cb: (success: boolean) => void) => { + const success = + this.rawConfig.proxyUsername === user && + this.rawConfig.proxyPassword === pass; + log.info( + mongoLogId(1_001_000_253), + this.logCtx, + 'Validated auth parameters', + { success } + ); + queueMicrotask(() => cb(success)); + } + ) + ); + } else { + log.info(mongoLogId(1_001_000_254), this.logCtx, 'Skipping auth setup'); + this.server.useAuth(socks5AuthNone()); + } + + this.serverListen = promisify(this.server.listen.bind(this.server)); + this.serverClose = promisify(this.server.close.bind(this.server)); + + for (const eventName of ['close', 'error', 'listening'] as const) { + this.server.on(eventName, this.emit.bind(this, eventName)); + } + } + + get config(): TunnelOptions { + const serverAddress = this.server.address(); + + return { + ...this.rawConfig, + proxyPort: + (typeof serverAddress !== 'string' && serverAddress?.port) || + this.rawConfig.proxyPort, + }; + } + + async listen(): Promise { + const { proxyHost, proxyPort } = this.rawConfig; + + log.info( + mongoLogId(1_001_000_255), + this.logCtx, + 'Listening for Socks5 connections', + { proxyHost, proxyPort } + ); + + const listeningPromise = this.serverListen(proxyPort, proxyHost); + try { + await Promise.all([listeningPromise, this.ensureAgentInitialized()]); + this.agentInitialized = true; + } catch (err: unknown) { + try { + await listeningPromise; + } finally { + await this.close(); + } + throw err; + } + } + + private async ensureAgentInitialized() { + if (this.agentInitialized) { + debug('agent already connected'); + return; + } + + if (this.agentInitPromise) { + debug('reusing agentInitPromise'); + return this.agentInitPromise; + } + + if (this.closed) { + // A socks5 request could come in after we deliberately closed the connection. Don't reconnect in that case. + throw new Error('Disconnected.'); + } + + try { + await (this.agentInitPromise = this.agent.initialize?.()); + } catch (err) { + this.emit('forwardingError', err); + log.error( + mongoLogId(1_001_000_258), + this.logCtx, + 'Failed to establish new SSH connection', + { error: (err as any)?.stack ?? String(err) } + ); + delete this.agentInitPromise; + await this.serverClose(); + throw err; + } + + delete this.agentInitPromise; + this.agentInitialized = true; + log.info( + mongoLogId(1_001_000_259), + this.logCtx, + 'Finished establishing new SSH connection' + ); + } + + private async closeOpenConnections() { + const waitForClose: Promise[] = []; + for (const socket of this.connections) { + waitForClose.push(once(socket, 'close')); + socket.destroy(); + } + await Promise.all(waitForClose); + this.connections.clear(); + } + + async close(): Promise { + this.closed = true; + + log.info(mongoLogId(1_001_000_256), this.logCtx, 'Closing SSH tunnel'); + const [maybeError] = await Promise.all([ + // If we catch anything, just return the error instead of throwing, we + // want to await on closing the connections before re-throwing server + // close error + this.serverClose().catch((e) => e), + this.agent.destroy?.(), + this.closeOpenConnections(), + ]); + + if (maybeError) { + throw maybeError; + } + } + + private async forwardOut(dstAddr: string, dstPort: number): Promise { + const channel = await promisify(this.agent.createSocket.bind(this.agent))( + createFakeHttpClientRequest(dstAddr, dstPort), + { + host: dstAddr, + port: dstPort, + } + ); + if (!channel) + throw new Error(`Could not create channel to ${dstAddr}:${dstPort}`); + return channel; + } + + private async socks5Request( + info: any, + accept: (intercept: true) => Socket, + deny: () => void + ): Promise { + const { srcAddr, srcPort, dstAddr, dstPort } = info; + const logMetadata = { srcAddr, srcPort, dstAddr, dstPort }; + log.info( + mongoLogId(1_001_000_260), + this.logCtx, + 'Received Socks5 fowarding request', + { + ...logMetadata, + } + ); + let socket: Socket | null = null; + + try { + await this.ensureAgentInitialized(); + + let channel: Duplex; + try { + channel = await this.forwardOut(dstAddr, dstPort); + } catch (err) { + // XXXX + if ((err as Error).message === 'Not connected') { + this.agentInitialized = false; + log.error( + mongoLogId(1_001_000_261), + this.logCtx, + 'Error forwarding Socks5 request, retrying', + { + ...logMetadata, + error: (err as Error).stack, + } + ); + await this.ensureAgentInitialized(); + channel = await this.forwardOut(dstAddr, dstPort); + } else { + throw err; + } + } + + log.info( + mongoLogId(1_001_000_262), + this.logCtx, + 'Opened SSH channel and accepting socks5 request', + { + ...logMetadata, + } + ); + + socket = accept(true); + this.connections.add(socket); + + socket.on('error', (err: ErrorWithOrigin) => { + log.error( + mongoLogId(1_001_000_263), + this.logCtx, + 'Error on Socks5 stream socket', + { + ...logMetadata, + error: (err as Error).stack, + } + ); + err.origin = err.origin ?? 'connection'; + this.emit('forwardingError', err); + }); + + socket.once('close', () => { + log.info( + mongoLogId(1_001_000_264), + this.logCtx, + 'Socks5 stream socket closed', + { + ...logMetadata, + } + ); + this.connections.delete(socket as Socket); + }); + + socket.pipe(channel).pipe(socket); + } catch (err) { + this.emit('forwardingError', err); + log.error( + mongoLogId(1_001_000_265), + this.logCtx, + 'Error establishing SSH channel for Socks5 request', + { + ...logMetadata, + error: (err as Error).stack, + } + ); + deny(); + if (socket) { + (err as any).origin = 'ssh-client'; + socket.destroy(err as any); + } + } + } +} + +export async function setupSocks5Tunnel( + proxyOptions: DevtoolsProxyOptions | AgentWithInitialize, + tunnelOptions?: Partial, target = 'mongodb://' -): Promise {} +): Promise { + let agent: AgentWithInitialize; + if ('createConnection' in proxyOptions) { + agent = proxyOptions as AgentWithInitialize; + } else { + if (!proxyForUrl(proxyOptions as DevtoolsProxyOptions)(target)) + return undefined; + agent = createAgent(proxyOptions as DevtoolsProxyOptions); + } + + const server = new Socks5Server(agent, { ...tunnelOptions }); + await server.listen(); + return server; +} diff --git a/packages/devtools-proxy-support/tsconfig.json b/packages/devtools-proxy-support/tsconfig.json index 21899b27..156674a5 100644 --- a/packages/devtools-proxy-support/tsconfig.json +++ b/packages/devtools-proxy-support/tsconfig.json @@ -2,7 +2,8 @@ "extends": "@mongodb-js/tsconfig-devtools/tsconfig.common.json", "compilerOptions": { "outDir": "dist", - "allowJs": true + "allowJs": true, + "strict": true }, "include": ["src/**/*"], "exclude": ["./src/**/*.spec.*"] From 8f989f76491d11ac5d923ad61f70c4dc656c6eb8 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 18 Jul 2024 19:06:21 +0200 Subject: [PATCH 04/30] WIP --- packages/devtools-proxy-support/src/agent.ts | 7 +- packages/devtools-proxy-support/src/index.ts | 3 +- .../devtools-proxy-support/src/logging.ts | 225 ++++++++++++++++++ .../src/{tunnel.ts => socks5.ts} | 125 +++------- packages/devtools-proxy-support/src/ssh.ts | 81 +++++-- 5 files changed, 317 insertions(+), 124 deletions(-) create mode 100644 packages/devtools-proxy-support/src/logging.ts rename packages/devtools-proxy-support/src/{tunnel.ts => socks5.ts} (72%) diff --git a/packages/devtools-proxy-support/src/agent.ts b/packages/devtools-proxy-support/src/agent.ts index 6f590734..66ee2153 100644 --- a/packages/devtools-proxy-support/src/agent.ts +++ b/packages/devtools-proxy-support/src/agent.ts @@ -7,11 +7,14 @@ import type { TcpNetConnectOpts } from 'net'; import type { ConnectionOptions } from 'tls'; import type { Duplex } from 'stream'; import { SSHAgent } from './ssh'; +import type { ProxyLogEmitter } from './logging'; +import type { EventEmitter } from 'events'; export type AgentWithInitialize = Agent & { // This is genuinely custom for our usage (to allow establishing an SSH tunnel // first before starting to push connections through it) initialize?(): Promise; + logger?: ProxyLogEmitter; // This is just part of the regular Agent interface, used by Node.js itself, // but missing from @types/node @@ -20,7 +23,9 @@ export type AgentWithInitialize = Agent & { options: TcpNetConnectOpts | ConnectionOptions, cb: (err: Error | null, s?: Duplex) => void ): void; -}; + + // http.Agent is an EventEmitter +} & Partial; export function createAgent( proxyOptions: DevtoolsProxyOptions diff --git a/packages/devtools-proxy-support/src/index.ts b/packages/devtools-proxy-support/src/index.ts index ed2541ef..839ea778 100644 --- a/packages/devtools-proxy-support/src/index.ts +++ b/packages/devtools-proxy-support/src/index.ts @@ -5,6 +5,7 @@ export { extractProxySecrets, mergeProxySecrets, } from './proxy-options'; -export { Tunnel, TunnelOptions, setupSocks5Tunnel } from './tunnel'; +export { Tunnel, TunnelOptions, setupSocks5Tunnel } from './socks5'; export { createAgent } from './agent'; export { createFetch } from './fetch'; +export { ProxyEventMap, ProxyLogEmitter, hookLogger } from './logging'; diff --git a/packages/devtools-proxy-support/src/logging.ts b/packages/devtools-proxy-support/src/logging.ts new file mode 100644 index 00000000..4974ca9c --- /dev/null +++ b/packages/devtools-proxy-support/src/logging.ts @@ -0,0 +1,225 @@ +interface BaseSocks5RequestMetadata { + srcAddr: string; + srcPort: number; + dstAddr: string; + dstPort: number; +} + +export interface ProxyEventMap { + 'socks5:authentication-complete': (ev: { success: boolean }) => void; + 'socks5:skip-auth-setup': () => void; + 'socks5:start-listening': (ev: { + proxyHost: string; + proxyPort: number; + }) => void; + 'socks5:forwarding-error': ( + ev: { error: string } & Partial + ) => void; + 'socks5:agent-initialized': () => void; + 'socks5:closing-tunnel': () => void; + 'socks5:got-forwarding-request': (ev: BaseSocks5RequestMetadata) => void; + 'socks5:accepted-forwarding-request': (ev: BaseSocks5RequestMetadata) => void; + 'socks5:failed-forwarding-request': ( + ev: { error: string } & Partial + ) => void; + 'socks5:forwarded-socket-closed': (ev: BaseSocks5RequestMetadata) => void; + + 'ssh:client-closed': () => void; + 'ssh:establishing-conection': (ev: { + host: string | undefined; + port: number | undefined; + password: boolean; + passphrase: boolean; + privateKey: boolean; + }) => void; + 'ssh:failed-connection': (ev: { error: string }) => void; + 'ssh:established-connection': () => void; + 'ssh:failed-forward': (ev: { + error: string; + host: string; + retryableError: boolean; + retriesLeft: number; + }) => void; +} + +export type ProxyEventArgs = + ProxyEventMap[K] extends (...args: infer P) => any ? P : never; + +export interface ProxyLogEmitter { + // TypeScript uses something like this itself for its EventTarget definitions. + on(event: K, listener: ProxyEventMap[K]): this; + off?( + event: K, + listener: ProxyEventMap[K] + ): this; + once( + event: K, + listener: ProxyEventMap[K] + ): this; + emit( + event: K, + ...args: ProxyEventArgs + ): unknown; +} + +interface MongoLogWriter { + info(c: string, id: unknown, ctx: string, msg: string, attr?: any): void; + warn(c: string, id: unknown, ctx: string, msg: string, attr?: any): void; + error(c: string, id: unknown, ctx: string, msg: string, attr?: any): void; + mongoLogId(this: void, id: number): unknown; +} + +let idCounter = 0; +export function hookLogger( + emitter: ProxyLogEmitter, + logCtx: string, + log: MongoLogWriter +): void { + logCtx = `${logCtx}-${idCounter++}`; + const { mongoLogId } = log; + + emitter.on('socks5:authentication-complete', (ev) => { + log.info( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_253), + `socks5-${logCtx}`, + 'Validated auth parameters', + { ...ev } + ); + }); + + emitter.on('socks5:skip-auth-setup', () => { + log.info( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_254), + `socks5-${logCtx}`, + 'Skipping auth setup' + ); + }); + + emitter.on('socks5:start-listening', (ev) => { + log.info( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_255), + `socks5-${logCtx}`, + 'Listening for Socks5 connections', + { ...ev } + ); + }); + + emitter.on('socks5:forwarding-error', (ev) => { + log.error( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_259), + `socks5-${logCtx}`, + 'Failed to establish new SSH connection', + { ...ev } + ); + }); + + emitter.on('socks5:agent-initialized', () => { + log.info( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_259), + `socks5-${logCtx}`, + 'Finished initializing agent' + ); + }); + + emitter.on('socks5:closing-tunnel', () => { + log.info( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_256), + `socks5-${logCtx}`, + 'Closing Socks5 tunnel' + ); + }); + + emitter.on('socks5:got-forwarding-request', (ev) => { + log.info( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_260), + `socks5-${logCtx}`, + 'Received Socks5 fowarding request', + { ...ev } + ); + }); + + emitter.on('socks5:accepted-forwarding-request', (ev) => { + log.info( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_262), + `socks5-${logCtx}`, + 'Established outbound connection and accepting socks5 request', + { ...ev } + ); + }); + + emitter.on('socks5:failed-forwarding-request', (ev) => { + log.error( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_265), + `socks5-${logCtx}`, + 'Error establishing outbound connection for socks5 request', + { ...ev } + ); + }); + + emitter.on('socks5:forwarded-socket-closed', (ev) => { + log.info( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_264), + `socks5-${logCtx}`, + 'Socks5 stream socket closed', + { ...ev } + ); + }); + + emitter.on('ssh:client-closed', () => { + log.info( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_252), + `ssh-${logCtx}`, + 'sshClient closed' + ); + }); + + emitter.on('ssh:establishing-conection', (ev) => { + log.info( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_257), + `ssh-${logCtx}`, + 'Establishing new SSH connection', + { ...ev } + ); + }); + emitter.on('ssh:failed-connection', (ev) => { + log.info( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_258), + `ssh-${logCtx}`, + 'Failed to establish new SSH connection', + { ...ev } + ); + }); + emitter.on('ssh:established-connection', () => { + log.info( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_259), + `ssh-${logCtx}`, + 'Finished establishing new SSH connection' + ); + }); + + emitter.on('ssh:failed-forward', (ev) => { + log.error( + 'DEVTOOLS-PROXY', + mongoLogId(1_001_000_261), + `ssh-${logCtx}`, + 'Error forwarding Socks5 request, retrying', + { + ...ev, + } + ); + }); +} diff --git a/packages/devtools-proxy-support/src/tunnel.ts b/packages/devtools-proxy-support/src/socks5.ts similarity index 72% rename from packages/devtools-proxy-support/src/tunnel.ts rename to packages/devtools-proxy-support/src/socks5.ts index be89ffae..d1650c80 100644 --- a/packages/devtools-proxy-support/src/tunnel.ts +++ b/packages/devtools-proxy-support/src/socks5.ts @@ -1,7 +1,6 @@ import { EventEmitter, once } from 'events'; import type { DevtoolsProxyOptions } from './proxy-options'; import { proxyForUrl } from './proxy-options'; -import type { Agent } from 'https'; import type { AgentWithInitialize } from './agent'; import { createAgent } from './agent'; @@ -15,6 +14,7 @@ import { isIPv6 } from 'net'; import type { Socket } from 'net'; import type { Duplex } from 'stream'; import type { ClientRequest } from 'http'; +import type { ProxyLogEmitter } from './logging'; export interface TunnelOptions { // These can safely be assigned to driver MongoClientOptinos @@ -37,6 +37,7 @@ function getTunnelOptions(config: Partial): TunnelOptions { type ErrorWithOrigin = Error & { origin?: string }; export interface Tunnel { + readonly logger: ProxyLogEmitter; on(ev: 'forwardingError', cb: (err: Error) => void): void; on(ev: 'error', cb: (err: Error) => void): void; @@ -59,22 +60,24 @@ function createFakeHttpClientRequest(dstAddr: string, dstPort: number) { } as ClientRequest; } -let idCounter = 0; class Socks5Server extends EventEmitter implements Tunnel { + public logger: ProxyLogEmitter = new EventEmitter() private readonly agent: AgentWithInitialize; private server: any; private serverListen: (port?: number, host?: string) => Promise; private serverClose: () => Promise; private connections: Set = new Set(); - private logCtx = `tunnel-${idCounter++}`; private rawConfig: TunnelOptions; private closed = false; private agentInitialized = false; private agentInitPromise?: Promise; - constructor(agent: Agent, tunnelOptions: Partial) { + constructor(agent: AgentWithInitialize, tunnelOptions: Partial) { super(); this.agent = agent; + if (agent.logger) + this.logger = agent.logger + agent.on?.('error', (err: Error) => this.emit('forwardingError', err)) this.rawConfig = getTunnelOptions(tunnelOptions); this.server = socks5Server.createServer(this.socks5Request.bind(this)); @@ -86,18 +89,13 @@ class Socks5Server extends EventEmitter implements Tunnel { const success = this.rawConfig.proxyUsername === user && this.rawConfig.proxyPassword === pass; - log.info( - mongoLogId(1_001_000_253), - this.logCtx, - 'Validated auth parameters', - { success } - ); + this.logger.emit('socks5:authentication-complete', {success}); queueMicrotask(() => cb(success)); } ) ); } else { - log.info(mongoLogId(1_001_000_254), this.logCtx, 'Skipping auth setup'); + this.logger.emit('socks5:skip-auth-setup'); this.server.useAuth(socks5AuthNone()); } @@ -123,12 +121,7 @@ class Socks5Server extends EventEmitter implements Tunnel { async listen(): Promise { const { proxyHost, proxyPort } = this.rawConfig; - log.info( - mongoLogId(1_001_000_255), - this.logCtx, - 'Listening for Socks5 connections', - { proxyHost, proxyPort } - ); + this.logger.emit('socks5:start-listening', { proxyHost, proxyPort }) const listeningPromise = this.serverListen(proxyPort, proxyHost); try { @@ -146,12 +139,10 @@ class Socks5Server extends EventEmitter implements Tunnel { private async ensureAgentInitialized() { if (this.agentInitialized) { - debug('agent already connected'); return; } if (this.agentInitPromise) { - debug('reusing agentInitPromise'); return this.agentInitPromise; } @@ -164,12 +155,8 @@ class Socks5Server extends EventEmitter implements Tunnel { await (this.agentInitPromise = this.agent.initialize?.()); } catch (err) { this.emit('forwardingError', err); - log.error( - mongoLogId(1_001_000_258), - this.logCtx, - 'Failed to establish new SSH connection', - { error: (err as any)?.stack ?? String(err) } - ); + this.logger.emit('socks5:forwarding-error', + { error: (err as any)?.stack ?? String(err) }) delete this.agentInitPromise; await this.serverClose(); throw err; @@ -177,11 +164,7 @@ class Socks5Server extends EventEmitter implements Tunnel { delete this.agentInitPromise; this.agentInitialized = true; - log.info( - mongoLogId(1_001_000_259), - this.logCtx, - 'Finished establishing new SSH connection' - ); + this.logger.emit('socks5:agent-initialized') } private async closeOpenConnections() { @@ -197,7 +180,7 @@ class Socks5Server extends EventEmitter implements Tunnel { async close(): Promise { this.closed = true; - log.info(mongoLogId(1_001_000_256), this.logCtx, 'Closing SSH tunnel'); + this.logger.emit('socks5:closing-tunnel') const [maybeError] = await Promise.all([ // If we catch anything, just return the error instead of throwing, we // want to await on closing the connections before re-throwing server @@ -232,92 +215,42 @@ class Socks5Server extends EventEmitter implements Tunnel { ): Promise { const { srcAddr, srcPort, dstAddr, dstPort } = info; const logMetadata = { srcAddr, srcPort, dstAddr, dstPort }; - log.info( - mongoLogId(1_001_000_260), - this.logCtx, - 'Received Socks5 fowarding request', - { - ...logMetadata, - } - ); + this.logger.emit('socks5:got-forwarding-request', {...logMetadata}); let socket: Socket | null = null; try { await this.ensureAgentInitialized(); - let channel: Duplex; - try { - channel = await this.forwardOut(dstAddr, dstPort); - } catch (err) { - // XXXX - if ((err as Error).message === 'Not connected') { - this.agentInitialized = false; - log.error( - mongoLogId(1_001_000_261), - this.logCtx, - 'Error forwarding Socks5 request, retrying', - { - ...logMetadata, - error: (err as Error).stack, - } - ); - await this.ensureAgentInitialized(); - channel = await this.forwardOut(dstAddr, dstPort); - } else { - throw err; - } - } + const channel = await this.forwardOut(dstAddr, dstPort); - log.info( - mongoLogId(1_001_000_262), - this.logCtx, - 'Opened SSH channel and accepting socks5 request', - { - ...logMetadata, - } - ); + this.logger.emit('socks5:accepted-forwarding-request', {...logMetadata}) socket = accept(true); this.connections.add(socket); socket.on('error', (err: ErrorWithOrigin) => { - log.error( - mongoLogId(1_001_000_263), - this.logCtx, - 'Error on Socks5 stream socket', - { - ...logMetadata, - error: (err as Error).stack, - } - ); - err.origin = err.origin ?? 'connection'; + err.origin ??= 'connection'; + this.logger.emit('socks5:forwarding-error', + { + ...logMetadata, + error: String((err as Error).stack), + }) this.emit('forwardingError', err); }); socket.once('close', () => { - log.info( - mongoLogId(1_001_000_264), - this.logCtx, - 'Socks5 stream socket closed', - { - ...logMetadata, - } - ); + this.logger.emit('socks5:forwarded-socket-closed', { ...logMetadata}) this.connections.delete(socket as Socket); }); socket.pipe(channel).pipe(socket); } catch (err) { + this.emit('socks5:failed-forwarding-request', + { + ...logMetadata, + error: String((err as Error).stack), + }); this.emit('forwardingError', err); - log.error( - mongoLogId(1_001_000_265), - this.logCtx, - 'Error establishing SSH channel for Socks5 request', - { - ...logMetadata, - error: (err as Error).stack, - } - ); deny(); if (socket) { (err as any).origin = 'ssh-client'; diff --git a/packages/devtools-proxy-support/src/ssh.ts b/packages/devtools-proxy-support/src/ssh.ts index 73a894f7..d3c37f37 100644 --- a/packages/devtools-proxy-support/src/ssh.ts +++ b/packages/devtools-proxy-support/src/ssh.ts @@ -5,10 +5,13 @@ import type { ClientRequest } from 'http'; import type { Duplex } from 'stream'; import type { ClientChannel, ConnectConfig } from 'ssh2'; import { Client as SshClient } from 'ssh2'; -import { once } from 'events'; +import EventEmitter, { once } from 'events'; import { promises as fs } from 'fs'; +import { promisify } from 'util'; +import type { ProxyLogEmitter } from './logging'; export class SSHAgent extends AgentBase implements AgentWithInitialize { + public logger: ProxyLogEmitter; private readonly proxyOptions: Readonly; private readonly url: URL; private sshClient: SshClient; @@ -22,13 +25,17 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { dstPort: number ) => Promise; - constructor(options: DevtoolsProxyOptions) { + constructor(options: DevtoolsProxyOptions, logger?: ProxyLogEmitter) { super(); + (this as AgentWithInitialize).on?.('error', () => { + //Errors should not crash the process + }); + this.logger = logger ?? new EventEmitter(); this.proxyOptions = options; this.url = new URL(options.proxy ?? ''); this.sshClient = new SshClient(); this.sshClient.on('close', () => { - log.info(mongoLogId(1_001_000_252), this.logCtx, 'sshClient closed'); + this.logger.emit('ssh:client-closed'); this.connected = false; }); @@ -37,12 +44,10 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { async initialize(): Promise { if (this.connected) { - debug('already connected'); return; } if (this.connectingPromise) { - debug('reusing connectingPromise'); return this.connectingPromise; } @@ -64,11 +69,13 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { passphrase: this.proxyOptions.sshOptions?.identityKeyPassphrase, }; - log.info( - mongoLogId(1_001_000_257), - this.logCtx, - 'Establishing new SSH connection' - ); + this.logger.emit('ssh:establishing-conection', { + host: sshConnectConfig.host, + port: sshConnectConfig.port, + password: !!sshConnectConfig.passphrase, + privateKey: !!sshConnectConfig.privateKey, + passphrase: !!sshConnectConfig.passphrase, + }); this.connectingPromise = Promise.race([ once(this.sshClient, 'error').then(([err]) => { @@ -84,32 +91,54 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { try { await this.connectingPromise; } catch (err) { - this.emit('forwardingError', err); - log.error( - mongoLogId(1_001_000_258), - this.logCtx, - 'Failed to establish new SSH connection', - { error: (err as any)?.stack ?? String(err) } - ); + (this as AgentWithInitialize).emit?.('error', err); + this.logger.emit('ssh:failed-connection', { + error: (err as any)?.stack ?? String(err), + }); delete this.connectingPromise; throw err; } delete this.connectingPromise; this.connected = true; - log.info( - mongoLogId(1_001_000_259), - this.logCtx, - 'Finished establishing new SSH connection' - ); + this.logger.emit('ssh:established-connection'); } override async connect(req: ClientRequest): Promise { - await this.initialize(); + return await this._connect(req); + } + + private async _connect(req: ClientRequest, retriesLeft = 1): Promise { + let host = ''; + try { + // Using the `host` header matches what proxy-agent does + host = req.getHeader('host') as string; + const url = new URL(req.path, `tcp://${host}`); - const host = req.getHeader('host') as string; - const url = new URL(req.path, `tcp://${host}`); + await this.initialize(); + + return await this.forwardOut('127.0.0.1', 0, url.hostname, +url.port); + } catch (err: unknown) { + const retryableError = (err as Error).message === 'Not connected'; + this.logger.emit('ssh:failed-forward', { + host, + error: String((err as Error).stack), + retryableError, + retriesLeft, + }); + if (retryableError) { + this.connected = false; + if (retriesLeft > 0) { + await this.initialize(); + return await this._connect(req, retriesLeft - 1); + } + } + throw err; + } + } - return await this.forwardOut('127.0.0.1', 0, url.hostname, +url.port); + destroy(): void { + this.closed = true; + this.sshClient.end(); } } From 85ad7167ad40b8331fc755c5aa56c82ba2876c1f Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 18 Jul 2024 20:06:44 +0200 Subject: [PATCH 05/30] WIP --- .../src/env-var-proxies.ts | 0 packages/devtools-proxy-support/src/socks5.ts | 41 ++++++++++--------- 2 files changed, 22 insertions(+), 19 deletions(-) delete mode 100644 packages/devtools-proxy-support/src/env-var-proxies.ts diff --git a/packages/devtools-proxy-support/src/env-var-proxies.ts b/packages/devtools-proxy-support/src/env-var-proxies.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/packages/devtools-proxy-support/src/socks5.ts b/packages/devtools-proxy-support/src/socks5.ts index d1650c80..ab319fce 100644 --- a/packages/devtools-proxy-support/src/socks5.ts +++ b/packages/devtools-proxy-support/src/socks5.ts @@ -61,7 +61,7 @@ function createFakeHttpClientRequest(dstAddr: string, dstPort: number) { } class Socks5Server extends EventEmitter implements Tunnel { - public logger: ProxyLogEmitter = new EventEmitter() + public logger: ProxyLogEmitter = new EventEmitter(); private readonly agent: AgentWithInitialize; private server: any; private serverListen: (port?: number, host?: string) => Promise; @@ -72,12 +72,14 @@ class Socks5Server extends EventEmitter implements Tunnel { private agentInitialized = false; private agentInitPromise?: Promise; - constructor(agent: AgentWithInitialize, tunnelOptions: Partial) { + constructor( + agent: AgentWithInitialize, + tunnelOptions: Partial + ) { super(); this.agent = agent; - if (agent.logger) - this.logger = agent.logger - agent.on?.('error', (err: Error) => this.emit('forwardingError', err)) + if (agent.logger) this.logger = agent.logger; + agent.on?.('error', (err: Error) => this.emit('forwardingError', err)); this.rawConfig = getTunnelOptions(tunnelOptions); this.server = socks5Server.createServer(this.socks5Request.bind(this)); @@ -89,7 +91,7 @@ class Socks5Server extends EventEmitter implements Tunnel { const success = this.rawConfig.proxyUsername === user && this.rawConfig.proxyPassword === pass; - this.logger.emit('socks5:authentication-complete', {success}); + this.logger.emit('socks5:authentication-complete', { success }); queueMicrotask(() => cb(success)); } ) @@ -121,7 +123,7 @@ class Socks5Server extends EventEmitter implements Tunnel { async listen(): Promise { const { proxyHost, proxyPort } = this.rawConfig; - this.logger.emit('socks5:start-listening', { proxyHost, proxyPort }) + this.logger.emit('socks5:start-listening', { proxyHost, proxyPort }); const listeningPromise = this.serverListen(proxyPort, proxyHost); try { @@ -155,8 +157,9 @@ class Socks5Server extends EventEmitter implements Tunnel { await (this.agentInitPromise = this.agent.initialize?.()); } catch (err) { this.emit('forwardingError', err); - this.logger.emit('socks5:forwarding-error', - { error: (err as any)?.stack ?? String(err) }) + this.logger.emit('socks5:forwarding-error', { + error: (err as any)?.stack ?? String(err), + }); delete this.agentInitPromise; await this.serverClose(); throw err; @@ -164,7 +167,7 @@ class Socks5Server extends EventEmitter implements Tunnel { delete this.agentInitPromise; this.agentInitialized = true; - this.logger.emit('socks5:agent-initialized') + this.logger.emit('socks5:agent-initialized'); } private async closeOpenConnections() { @@ -180,7 +183,7 @@ class Socks5Server extends EventEmitter implements Tunnel { async close(): Promise { this.closed = true; - this.logger.emit('socks5:closing-tunnel') + this.logger.emit('socks5:closing-tunnel'); const [maybeError] = await Promise.all([ // If we catch anything, just return the error instead of throwing, we // want to await on closing the connections before re-throwing server @@ -215,7 +218,7 @@ class Socks5Server extends EventEmitter implements Tunnel { ): Promise { const { srcAddr, srcPort, dstAddr, dstPort } = info; const logMetadata = { srcAddr, srcPort, dstAddr, dstPort }; - this.logger.emit('socks5:got-forwarding-request', {...logMetadata}); + this.logger.emit('socks5:got-forwarding-request', { ...logMetadata }); let socket: Socket | null = null; try { @@ -223,30 +226,30 @@ class Socks5Server extends EventEmitter implements Tunnel { const channel = await this.forwardOut(dstAddr, dstPort); - this.logger.emit('socks5:accepted-forwarding-request', {...logMetadata}) + this.logger.emit('socks5:accepted-forwarding-request', { + ...logMetadata, + }); socket = accept(true); this.connections.add(socket); socket.on('error', (err: ErrorWithOrigin) => { err.origin ??= 'connection'; - this.logger.emit('socks5:forwarding-error', - { + this.logger.emit('socks5:forwarding-error', { ...logMetadata, error: String((err as Error).stack), - }) + }); this.emit('forwardingError', err); }); socket.once('close', () => { - this.logger.emit('socks5:forwarded-socket-closed', { ...logMetadata}) + this.logger.emit('socks5:forwarded-socket-closed', { ...logMetadata }); this.connections.delete(socket as Socket); }); socket.pipe(channel).pipe(socket); } catch (err) { - this.emit('socks5:failed-forwarding-request', - { + this.emit('socks5:failed-forwarding-request', { ...logMetadata, error: String((err as Error).stack), }); From 01b018da05a139979b73e43cbbce12f69ab17347 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Mon, 22 Jul 2024 21:23:12 +0200 Subject: [PATCH 06/30] WIP --- package-lock.json | 308 ++++++++++++++++++ packages/devtools-proxy-support/package.json | 4 +- .../devtools-proxy-support/src/agent.spec.ts | 243 ++++++++++++++ packages/devtools-proxy-support/src/agent.ts | 19 ++ .../devtools-proxy-support/src/fetch.spec.ts | 44 +++ packages/devtools-proxy-support/src/fetch.ts | 65 +++- .../devtools-proxy-support/src/index.spec.ts | 0 packages/devtools-proxy-support/src/index.ts | 8 +- packages/devtools-proxy-support/src/socks5.ts | 13 +- .../test/fixtures/ca.crt | 29 ++ .../test/fixtures/server.bundle.pem | 161 +++++++++ .../test/fixtures/sshd.key | 15 + .../devtools-proxy-support/test/helpers.ts | 194 +++++++++++ .../devtools-proxy-support/test/socksv5.d.ts | 12 + 14 files changed, 1094 insertions(+), 21 deletions(-) create mode 100644 packages/devtools-proxy-support/src/agent.spec.ts create mode 100644 packages/devtools-proxy-support/src/fetch.spec.ts delete mode 100644 packages/devtools-proxy-support/src/index.spec.ts create mode 100644 packages/devtools-proxy-support/test/fixtures/ca.crt create mode 100644 packages/devtools-proxy-support/test/fixtures/server.bundle.pem create mode 100644 packages/devtools-proxy-support/test/fixtures/sshd.key create mode 100644 packages/devtools-proxy-support/test/helpers.ts create mode 100644 packages/devtools-proxy-support/test/socksv5.d.ts diff --git a/package-lock.json b/package-lock.json index 09f55fb9..af455079 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7889,6 +7889,18 @@ "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dev": true, + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -10940,6 +10952,84 @@ "readable-stream": "^2.0.2" } }, + "node_modules/duplexpair": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/duplexpair/-/duplexpair-1.0.2.tgz", + "integrity": "sha512-6DHuWdEGHNcuSqrn926rWJcRsTDrb+ugw0hx/trAxCH48z9WlFqDtwtbiEMq/KGFYQWzLs1VA0I6KUkuIgCoXw==", + "dev": true, + "dependencies": { + "readable-stream": "^4.5.2" + } + }, + "node_modules/duplexpair/node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/duplexpair/node_modules/readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "dev": true, + "dependencies": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/duplexpair/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/duplexpair/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -12105,6 +12195,15 @@ "es5-ext": "~0.10.14" } }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/eventemitter3": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", @@ -12395,6 +12494,28 @@ "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", "dev": true }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, "node_modules/figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -12669,6 +12790,17 @@ "node": ">= 0.12" } }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -18440,6 +18572,24 @@ "integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==", "devOptional": true }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } + }, "node_modules/node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", @@ -20520,6 +20670,15 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "dev": true, + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -24084,6 +24243,14 @@ "defaults": "^1.0.3" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", @@ -25489,6 +25656,7 @@ "license": "Apache-2.0", "dependencies": { "agent-base": "^7.1.1", + "node-fetch": "^3.3.2", "proxy-agent": "^6.4.0", "socksv5": "^0.0.6", "ssh2": "^1.15.0" @@ -25504,6 +25672,7 @@ "@types/sinon-chai": "^3.2.5", "chai": "^4.3.6", "depcheck": "^1.4.1", + "duplexpair": "^1.0.2", "eslint": "^7.25.0", "gen-esm-wrapper": "^1.1.0", "mocha": "^8.4.0", @@ -25524,6 +25693,31 @@ "node": ">= 14" } }, + "packages/devtools-proxy-support/node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "engines": { + "node": ">= 12" + } + }, + "packages/devtools-proxy-support/node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, "packages/devtools-proxy-support/node_modules/typescript": { "version": "5.5.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", @@ -31932,9 +32126,11 @@ "agent-base": "^7.1.1", "chai": "^4.3.6", "depcheck": "^1.4.1", + "duplexpair": "^1.0.2", "eslint": "^7.25.0", "gen-esm-wrapper": "^1.1.0", "mocha": "^8.4.0", + "node-fetch": "^3.3.2", "nyc": "^15.1.0", "prettier": "^2.3.2", "proxy-agent": "^6.4.0", @@ -31952,6 +32148,21 @@ "debug": "^4.3.4" } }, + "data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==" + }, + "node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "requires": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + } + }, "typescript": { "version": "5.5.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", @@ -34528,6 +34739,15 @@ "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==" }, + "abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dev": true, + "requires": { + "event-target-shim": "^5.0.0" + } + }, "accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -36803,6 +37023,55 @@ "readable-stream": "^2.0.2" } }, + "duplexpair": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/duplexpair/-/duplexpair-1.0.2.tgz", + "integrity": "sha512-6DHuWdEGHNcuSqrn926rWJcRsTDrb+ugw0hx/trAxCH48z9WlFqDtwtbiEMq/KGFYQWzLs1VA0I6KUkuIgCoXw==", + "dev": true, + "requires": { + "readable-stream": "^4.5.2" + }, + "dependencies": { + "buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "dev": true, + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "readable-stream": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.5.2.tgz", + "integrity": "sha512-yjavECdqeZ3GLXNgRXgeQEdz9fvDDkNKyHnbHRFtOr7/LcfgBcmct7t/ET+HaCTqfh06OzoAxrkN/IfjJBVe+g==", + "dev": true, + "requires": { + "abort-controller": "^3.0.0", + "buffer": "^6.0.3", + "events": "^3.3.0", + "process": "^0.11.10", + "string_decoder": "^1.3.0" + } + }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "requires": { + "safe-buffer": "~5.2.0" + } + } + } + }, "eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -37678,6 +37947,12 @@ "es5-ext": "~0.10.14" } }, + "event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "dev": true + }, "eventemitter3": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", @@ -37920,6 +38195,15 @@ "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", "dev": true }, + "fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "requires": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + } + }, "figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", @@ -38133,6 +38417,14 @@ "mime-types": "^2.1.12" } }, + "formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "requires": { + "fetch-blob": "^3.1.2" + } + }, "forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -42673,6 +42965,11 @@ "integrity": "sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==", "devOptional": true }, + "node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==" + }, "node-fetch": { "version": "2.6.7", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", @@ -44238,6 +44535,12 @@ "integrity": "sha512-++Vn7NS4Xf9NacaU9Xq3URUuqZETPsf8L4j5/ckhaRYsfPeRyzGw+iDjFhV/Jr3uNmTvvddEJFWh5R1gRgUH8A==", "dev": true }, + "process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", + "dev": true + }, "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", @@ -46981,6 +47284,11 @@ "defaults": "^1.0.3" } }, + "web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==" + }, "webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", diff --git a/packages/devtools-proxy-support/package.json b/packages/devtools-proxy-support/package.json index 7f20960e..fdfd3849 100644 --- a/packages/devtools-proxy-support/package.json +++ b/packages/devtools-proxy-support/package.json @@ -49,7 +49,8 @@ "proxy-agent": "^6.4.0", "agent-base": "^7.1.1", "ssh2": "^1.15.0", - "socksv5": "^0.0.6" + "socksv5": "^0.0.6", + "node-fetch": "^3.3.2" }, "devDependencies": { "@mongodb-js/eslint-config-devtools": "0.9.10", @@ -62,6 +63,7 @@ "@types/sinon-chai": "^3.2.5", "chai": "^4.3.6", "depcheck": "^1.4.1", + "duplexpair": "^1.0.2", "eslint": "^7.25.0", "gen-esm-wrapper": "^1.1.0", "mocha": "^8.4.0", diff --git a/packages/devtools-proxy-support/src/agent.spec.ts b/packages/devtools-proxy-support/src/agent.spec.ts new file mode 100644 index 00000000..67550686 --- /dev/null +++ b/packages/devtools-proxy-support/src/agent.spec.ts @@ -0,0 +1,243 @@ +import { createAgent } from './'; +import type { Agent, IncomingMessage } from 'http'; +import { get as httpGet } from 'http'; +import { get as httpsGet } from 'https'; +import { expect } from 'chai'; +import sinon from 'sinon'; +import { HTTPServerProxyTestSetup } from '../test/helpers'; +import socks5AuthNone from 'socksv5/lib/auth/None'; +import socks5AuthUserPassword from 'socksv5/lib/auth/UserPassword'; + +describe('createAgent', function () { + let setup: HTTPServerProxyTestSetup; + let agents: Agent[]; + + const get = async ( + url: string, + agent: Agent + ): Promise => { + const getFn = new URL(url).protocol === 'https:' ? httpsGet : httpGet; + const options = { + agent, + ca: setup.tlsOptions.ca, + checkServerIdentity: () => undefined, // allow hostname mismatches + }; + agents.push(agent); + const res = await new Promise((resolve, reject) => + getFn(url, options, resolve).once('error', reject) + ); + let body = ''; + res.setEncoding('utf8'); + for await (const chunk of res) body += chunk; + return Object.assign(res, { body }); + }; + + beforeEach(async function () { + agents = []; + setup = new HTTPServerProxyTestSetup(); + await setup.listen(); + }); + + afterEach(async function () { + await setup.teardown(); + for (const agent of new Set(agents)) { + agent.destroy(); + } + }); + + context('socks5', function () { + it('can connect to a socks5 proxy without auth', async function () { + setup.socks5ProxyServer.useAuth(socks5AuthNone()); + + const res = await get( + 'http://example.com/hello', + createAgent({ proxy: `socks5://127.0.0.1:${setup.socks5ProxyPort}` }) + ); + expect(res.body).to.equal('OK /hello'); + expect(setup.getRequestedUrls()).to.deep.equal([ + 'http://example.com/hello', + ]); + }); + + it('can connect to a socks5 proxy with successful auth', async function () { + const authHandler = sinon.stub().yields(true); + setup.socks5ProxyServer.useAuth(socks5AuthUserPassword(authHandler)); + + const res = await get( + 'http://example.com/hello', + createAgent({ + proxy: `socks5://foo:bar@127.0.0.1:${setup.socks5ProxyPort}`, + }) + ); + expect(res.body).to.equal('OK /hello'); + expect(setup.getRequestedUrls()).to.deep.equal([ + 'http://example.com/hello', + ]); + expect(authHandler).to.have.been.calledOnceWith('foo', 'bar'); + }); + + it('fails to connect to a socks5 proxy with unsuccessful auth', async function () { + const authHandler = sinon.stub().yields(false); + setup.socks5ProxyServer.useAuth(socks5AuthUserPassword(authHandler)); + + try { + await get( + 'http://example.com/hello', + createAgent({ + proxy: `socks5://foo:bar@127.0.0.1:${setup.socks5ProxyPort}`, + }) + ); + expect.fail('missed exception'); + } catch (err: any) { + expect(err.message).to.equal('Socks5 Authentication failed'); + } + }); + }); + + context('http proxy', function () { + it('can connect to a socks5 proxy without auth', async function () { + const res = await get( + 'http://example.com/hello', + createAgent({ proxy: `http://127.0.0.1:${setup.httpProxyPort}` }) + ); + expect(res.body).to.equal('OK /hello'); + expect(setup.getRequestedUrls()).to.deep.equal([ + 'http://example.com/hello', + ]); + }); + + it('can connect to a socks5 proxy with successful auth', async function () { + setup.authHandler = sinon.stub().returns(true); + + const res = await get( + 'http://example.com/hello', + createAgent({ + proxy: `http://foo:bar@127.0.0.1:${setup.httpProxyPort}`, + }) + ); + expect(res.body).to.equal('OK /hello'); + expect(setup.getRequestedUrls()).to.deep.equal([ + 'http://example.com/hello', + ]); + expect(setup.authHandler).to.have.been.calledOnceWith('foo', 'bar'); + }); + + it('fails to connect to a socks5 proxy with unsuccessful auth', async function () { + setup.authHandler = sinon.stub().returns(false); + + const res = await get( + 'http://example.com/hello', + createAgent({ + proxy: `http://foo:bar@127.0.0.1:${setup.httpProxyPort}`, + }) + ); + expect(res.statusCode).to.equal(407); + }); + }); + + context('https/connect proxy', function () { + it('can connect to a https proxy without auth', async function () { + const res = await get( + 'https://example.com/hello', + createAgent({ proxy: `http://127.0.0.1:${setup.httpsProxyPort}` }) + ); + expect(res.body).to.equal('OK /hello'); + expect(setup.getRequestedUrls()).to.deep.equal([ + 'http://example.com/hello', + ]); + }); + + it('can connect to a socks5 proxy with successful auth', async function () { + setup.authHandler = sinon.stub().returns(true); + + const res = await get( + 'https://example.com/hello', + createAgent({ + proxy: `http://foo:bar@127.0.0.1:${setup.httpsProxyPort}`, + }) + ); + expect(res.body).to.equal('OK /hello'); + expect(setup.getRequestedUrls()).to.deep.equal([ + 'http://example.com/hello', + ]); + expect(setup.authHandler).to.have.been.calledOnceWith('foo', 'bar'); + }); + + it('fails to connect to a socks5 proxy with unsuccessful auth', async function () { + setup.authHandler = sinon.stub().returns(false); + + const res = await get( + 'https://example.com/hello', + createAgent({ + proxy: `http://foo:bar@127.0.0.1:${setup.httpsProxyPort}`, + }) + ); + expect(res.statusCode).to.equal(407); + }); + }); + + context('ssh proxy', function () { + it('can connect to a ssh proxy without auth', async function () { + const res = await get( + 'https://example.com/hello', + createAgent({ proxy: `ssh://someuser@127.0.0.1:${setup.sshProxyPort}` }) + ); + expect(res.body).to.equal('OK /hello'); + expect(setup.getRequestedUrls()).to.deep.equal([ + 'http://example.com/hello', + ]); + expect(setup.sshTunnelInfos).to.deep.equal([ + { destIP: 'example.com', destPort: 0, srcIP: '127.0.0.1', srcPort: 0 }, + ]); + }); + + it('can connect to a ssh proxy with successful auth', async function () { + setup.authHandler = sinon.stub().returns(true); + + const res = await get( + 'https://example.com/hello', + createAgent({ proxy: `ssh://foo:bar@127.0.0.1:${setup.sshProxyPort}` }) + ); + expect(res.body).to.equal('OK /hello'); + expect(setup.getRequestedUrls()).to.deep.equal([ + 'http://example.com/hello', + ]); + expect(setup.authHandler).to.have.been.calledOnceWith('foo', 'bar'); + }); + + it('fails to connect to a ssh proxy with unsuccessful auth', async function () { + setup.authHandler = sinon.stub().returns(false); + + try { + await get( + 'http://example.com/hello', + createAgent({ + proxy: `ssh://foo:bar@127.0.0.1:${setup.sshProxyPort}`, + }) + ); + expect.fail('missed exception'); + } catch (err: any) { + expect(err.message).to.equal( + 'All configured authentication methods failed' + ); + } + }); + + it('fails to connect to a ssh proxy with unavailable tunneling', async function () { + setup.authHandler = sinon.stub().returns(true); + setup.canTunnel = false; + + try { + await get( + 'http://example.com/hello', + createAgent({ + proxy: `ssh://foo:bar@127.0.0.1:${setup.sshProxyPort}`, + }) + ); + expect.fail('missed exception'); + } catch (err: any) { + expect(err.message).to.include('Channel open failure'); + } + }); + }); +}); diff --git a/packages/devtools-proxy-support/src/agent.ts b/packages/devtools-proxy-support/src/agent.ts index 66ee2153..40fb4b6b 100644 --- a/packages/devtools-proxy-support/src/agent.ts +++ b/packages/devtools-proxy-support/src/agent.ts @@ -30,6 +30,9 @@ export type AgentWithInitialize = Agent & { export function createAgent( proxyOptions: DevtoolsProxyOptions ): AgentWithInitialize { + // This could be made a bit more flexible by creating an Agent using AgentBase + // that will dynamically choose between SSHAgent and ProxyAgent. + // Right now, this is a bit simpler in terms of lifetime management for SSHAgent. if (proxyOptions.proxy && new URL(proxyOptions.proxy).protocol === 'ssh:') { return new SSHAgent(proxyOptions); } @@ -38,3 +41,19 @@ export function createAgent( getProxyForUrl, }); } + +export function useOrCreateAgent( + proxyOptions: DevtoolsProxyOptions | AgentWithInitialize, + target?: string +): AgentWithInitialize | undefined { + if ('createConnection' in proxyOptions) { + return proxyOptions as AgentWithInitialize; + } else { + if ( + target !== undefined && + !proxyForUrl(proxyOptions as DevtoolsProxyOptions)(target) + ) + return undefined; + return createAgent(proxyOptions as DevtoolsProxyOptions); + } +} diff --git a/packages/devtools-proxy-support/src/fetch.spec.ts b/packages/devtools-proxy-support/src/fetch.spec.ts new file mode 100644 index 00000000..17a2f87c --- /dev/null +++ b/packages/devtools-proxy-support/src/fetch.spec.ts @@ -0,0 +1,44 @@ +import { expect } from 'chai'; +import { createFetch } from './'; +import { HTTPServerProxyTestSetup } from '../test/helpers'; + +describe('createFetch', function () { + it(`consistency check: plain "import('node-fetch') fails"`, async function () { + let failed = false; + try { + await import('node-fetch'); + } catch (error) { + failed = true; + expect((error as Error).message).to.include('require() of ES Module'); + } + expect(failed).to.equal(true); + }); + + context('HTTP calls', function () { + let setup: HTTPServerProxyTestSetup; + + before(async function () { + setup = new HTTPServerProxyTestSetup(); + await setup.listen(); + }); + + after(async function () { + await setup.teardown(); + }); + + it('provides a node-fetch-like HTTP functionality', async function () { + const response = await createFetch({})( + `http://127.0.0.1:${setup.httpServerPort}/test` + ); + expect(await response.text()).to.equal('OK /test'); + }); + + it.only('makes use of proxy support when instructed to do so', async function () { + const response = await createFetch({ + proxy: `ssh://someuser@127.0.0.1:${setup.sshProxyPort}`, + })(`http://127.0.0.1:${setup.httpServerPort}/test`); + expect(await response.text()).to.equal('OK /test'); + expect(setup.sshTunnelInfos).to.deep.equal([]); + }); + }); +}); diff --git a/packages/devtools-proxy-support/src/fetch.ts b/packages/devtools-proxy-support/src/fetch.ts index c88f8679..06da62c5 100644 --- a/packages/devtools-proxy-support/src/fetch.ts +++ b/packages/devtools-proxy-support/src/fetch.ts @@ -1,13 +1,60 @@ -import type { RequestInit, Response } from 'node-fetch'; -import fetch from 'node-fetch'; -import { createAgent } from './agent'; +import type fetch from 'node-fetch'; +import type { RequestInit, RequestInfo, Request, Response } from 'node-fetch'; +import type { AgentWithInitialize } from './agent'; +import { useOrCreateAgent } from './agent'; import type { DevtoolsProxyOptions } from './proxy-options'; +declare const __webpack_require__: unknown; + +async function importNodeFetch(): Promise { + // Node-fetch is an ESM module from 3.x + // Importing ESM modules to CommonJS is possible with a dynamic import. + // However, once this is transpiled with TS, `await import()` changes to `require()`, which fails to load + // the package at runtime. + // The alternative, to transpile with "moduleResolution": "NodeNext", is not always feasible. + // Use this function to safely import the node-fetch package + let module: typeof fetch | { default: typeof fetch }; + try { + module = await import('node-fetch'); + } catch (err: unknown) { + if ( + err && + typeof err === 'object' && + 'code' in err && + err.code === 'ERR_REQUIRE_ESM' && + typeof __webpack_require__ === 'undefined' + ) { + // This means that the import() above was transpiled to require() + // and that that require() called failed because it saw actual on-disk ESM. + // In this case, it should be safe to use eval'ed import(). + module = await eval(`import('node-fetch')`); + } else { + throw err; + } + } + + return typeof module === 'function' ? module : module.default; +} +let cachedFetch: Promise | undefined; + +export type { Request, Response, RequestInfo, RequestInit }; export function createFetch( - proxyOptions: DevtoolsProxyOptions -): (url: string, fetchOptions?: RequestInit) => Promise { - const agent = createAgent(proxyOptions); - return async (url, fetchOptions) => { - return await fetch(url, { agent, ...fetchOptions }); - }; + proxyOptions: DevtoolsProxyOptions | AgentWithInitialize +): { agent: AgentWithInitialize | undefined } & (( + url: string, + fetchOptions?: RequestInit +) => Promise) { + const agent = useOrCreateAgent(proxyOptions); + let agentInitializedPromise; + + return Object.assign( + async (url: string, fetchOptions?: RequestInit) => { + const [fetch] = await Promise.all([ + (cachedFetch ??= importNodeFetch()), + (agentInitializedPromise ??= agent?.initialize?.()), + ]); + return await fetch(url, { agent, ...fetchOptions }); + }, + { agent } + ); } diff --git a/packages/devtools-proxy-support/src/index.spec.ts b/packages/devtools-proxy-support/src/index.spec.ts deleted file mode 100644 index e69de29b..00000000 diff --git a/packages/devtools-proxy-support/src/index.ts b/packages/devtools-proxy-support/src/index.ts index 839ea778..2f414686 100644 --- a/packages/devtools-proxy-support/src/index.ts +++ b/packages/devtools-proxy-support/src/index.ts @@ -7,5 +7,11 @@ export { } from './proxy-options'; export { Tunnel, TunnelOptions, setupSocks5Tunnel } from './socks5'; export { createAgent } from './agent'; -export { createFetch } from './fetch'; +export { + createFetch, + Request, + Response, + RequestInfo, + RequestInit, +} from './fetch'; export { ProxyEventMap, ProxyLogEmitter, hookLogger } from './logging'; diff --git a/packages/devtools-proxy-support/src/socks5.ts b/packages/devtools-proxy-support/src/socks5.ts index ab319fce..7c97879c 100644 --- a/packages/devtools-proxy-support/src/socks5.ts +++ b/packages/devtools-proxy-support/src/socks5.ts @@ -1,8 +1,7 @@ import { EventEmitter, once } from 'events'; import type { DevtoolsProxyOptions } from './proxy-options'; -import { proxyForUrl } from './proxy-options'; import type { AgentWithInitialize } from './agent'; -import { createAgent } from './agent'; +import { useOrCreateAgent } from './agent'; // The socksv5 module is not bundle-able by itself, so we get the // subpackages directly @@ -268,14 +267,8 @@ export async function setupSocks5Tunnel( tunnelOptions?: Partial, target = 'mongodb://' ): Promise { - let agent: AgentWithInitialize; - if ('createConnection' in proxyOptions) { - agent = proxyOptions as AgentWithInitialize; - } else { - if (!proxyForUrl(proxyOptions as DevtoolsProxyOptions)(target)) - return undefined; - agent = createAgent(proxyOptions as DevtoolsProxyOptions); - } + const agent = useOrCreateAgent(proxyOptions, target); + if (!agent) return undefined; const server = new Socks5Server(agent, { ...tunnelOptions }); await server.listen(); diff --git a/packages/devtools-proxy-support/test/fixtures/ca.crt b/packages/devtools-proxy-support/test/fixtures/ca.crt new file mode 100644 index 00000000..5a0bcb82 --- /dev/null +++ b/packages/devtools-proxy-support/test/fixtures/ca.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE9DCCAtwCCQDF4fcEj5YnPzANBgkqhkiG9w0BAQsFADA7MRAwDgYDVQQKDAdN +b25nb0RCMREwDwYDVQQLDAhEZXZUb29sczEUMBIGA1UEAwwLRGV2VG9vbHMgQ0Ew +IBcNMjAxMjE3MDg1NTA1WhgPMjI5NDEwMDEwODU1MDVaMDsxEDAOBgNVBAoMB01v +bmdvREIxETAPBgNVBAsMCERldlRvb2xzMRQwEgYDVQQDDAtEZXZUb29scyBDQTCC +AiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKhwAJUXj5cTCjxRyxVBCVd0 +eS/NEZ+HspSTs/ESKBAJ01l50VKfgKSeBjPaymYvSMyp/UHQVk0WUGmgkJSwtQjB +PgtshVbhAg9MGURFT8yCo2wVEUiRJsJJVyYPfkafRtXe1wgh+Ei85qqemIY3MZto +rre/oRO4/90NjCuATqmV6fR1Dw+Yr/gRBe0DQy3KDmKd6Nb2NPGw4KApe/pLUN4M +i44moYi/NbYNhRuL/uzdZxT+H21uuOe8/9TeG/rz5uxKuKSlTZNuQQX55BGDyG/r +yt7v0dVMIxVrL0bBLoYKbQfQYh7WnArlskiczXs5pTaGTKm560f+yVmIcCzbywiz +0zD2nUzoqmvFIbdkCGB/9sU4S81buI6qGQYHTsiFWjBA5U+G3Lu4k9GUA6wm5d0p +x9cqN9Wc7M2latsz40UGGF7KXeBoAJtQoqAUNxz8MErJGE5VS1626uU1gCGiphfP +Ng+pxWbIQJbEOKezkq+OMD45/G38HkHg0LQ9mAmI6hvUuoBuEUMSlD8o1yzKPaP1 +ZbCe737oo1iuorm+2A30E+NdZHpT21UoYOLwWuBA4/pxXoRtP2FpFpsoQrW2eRTO +2vZqJK7uIdfBrvTQrLuGX2T+Z22hitA1PzWWw426kOkBlxkICov0NCQrqOYE6QfS +M1ljzao82aepX4Uz95gvAgMBAAEwDQYJKoZIhvcNAQELBQADggIBAAGddnbgFTAn +fftl7dgKrAB4Cqe2uZt+q0aWoBsNq0e1IYv3e8Ww948fiXNxi/+x8LmtidG8tRiw ++mnM6HMjH5rubqbEALBkfLE5H1ruZN+g7yBZyNJ7rdqzTbv75p1xym5acafMV8yE +1/w6TU8NymAlEKGjDehWSLrb6xtgnIKzjc/VpCu/DqRUrfbNVwai68dIUGufcSoz +HNdL5YiLdwtMhfpBSsh1rHOqXvCPSvCFlSw5KRYUkmAusFgXhB+GuJyGz5rPO4ie +IRdZ+Wz7zOR1KOyaEEU7DdgN8TjrEMG22+L35p6KbQb6VFVW0qSH5+G0Y7d2lroo +ckeqail06vRdsgsLRkQH/oBEcuS1KLaZuTJwlCdLS9uosYxn4vd3v7z//FnQeQSs +O/Wz+mUJHWW+rCeum5jUDP9Z5158SDCMA1oNUF9FBgexFkR5F8MX+2qzXspLl17b +6Q1oFCQ47DVGSpGJ4j973rD4dzEFAe6Yct7R5uqw5zjslHS599nwE7TLNDWz2/2E +Umho59WF0oYUTaOsGERq7qAwEkCT4KVWxVUwb25Brcf9Z/D9I3lEU552G0BBDvYK +uFBrE/UzJHrWHKe4SgCeKbOvsBMlyoxizX2VWHO+vLFuc2YQs91eL0ZWv2AkkBII +QEQnAnxSNIFO1Y1NhfTdmlNCfHGo8kyc +-----END CERTIFICATE----- diff --git a/packages/devtools-proxy-support/test/fixtures/server.bundle.pem b/packages/devtools-proxy-support/test/fixtures/server.bundle.pem new file mode 100644 index 00000000..c3e66bb0 --- /dev/null +++ b/packages/devtools-proxy-support/test/fixtures/server.bundle.pem @@ -0,0 +1,161 @@ +Certificate: + Data: + Version: 1 (0x0) + Serial Number: + f3:49:92:0f:8b:55:bb:12 + Signature Algorithm: sha256WithRSAEncryption + Issuer: O=MongoDB, OU=DevTools, CN=DevTools CA + Validity + Not Before: Dec 22 12:54:19 2020 GMT + Not After : Oct 6 12:54:19 2294 GMT + Subject: O=MongoDB, OU=DevTools, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (4096 bit) + Modulus: + 00:b3:57:d2:f7:31:46:0b:76:ba:03:a6:65:53:cc: + 6e:8f:8f:48:82:e1:70:d9:31:7e:23:18:62:3d:f3: + e3:7c:12:8a:9a:10:42:6a:40:d2:48:f2:2e:2f:a1: + f3:92:35:4b:eb:e4:c1:56:36:10:77:0e:13:eb:3d: + cc:95:e1:27:2d:7a:45:ef:0a:f1:19:d5:ac:6c:d0: + 9b:61:d0:5f:00:0a:25:ce:20:ca:82:a7:1d:20:40: + d8:3b:3a:5d:dd:7b:f1:b2:46:92:16:7e:ee:97:e0: + 8f:6c:de:05:79:29:0b:9d:92:df:a9:81:e4:53:78: + 1f:26:44:8e:d2:a5:66:33:74:04:d7:16:f4:9f:9e: + e8:b4:d8:62:1b:2b:95:14:2d:46:35:f8:b8:b1:cb: + 52:21:df:49:29:04:82:2f:8d:66:e1:19:eb:11:6a: + 9b:cf:a4:21:f0:54:b3:16:39:a7:34:75:5c:f6:62: + 50:39:71:06:5b:3e:e1:ff:dc:25:7c:fc:05:3a:84: + 8b:71:05:1f:5f:15:78:37:e0:dc:1f:70:c3:87:ab: + 70:22:18:98:81:8e:c3:2c:99:96:ac:0f:28:ea:fd: + c6:cd:b4:5f:83:47:77:69:0e:42:34:8d:0a:7c:a9: + 71:c9:0a:0c:f4:cb:03:c2:64:dc:92:32:82:89:bb: + 73:4f:e2:f3:de:e2:c9:1b:85:71:3c:88:72:48:38: + 54:49:b6:6d:31:d7:71:a7:f3:75:bd:70:1b:4f:3b: + 93:d4:34:05:7f:1a:2d:f1:be:d5:ee:87:b1:a0:5f: + 2a:52:d8:6e:b9:8a:1c:b2:3d:80:a8:74:9b:06:ff: + c3:d8:9c:f4:50:0f:f3:aa:ba:c2:80:43:44:aa:b6: + d7:ae:7f:bf:5f:a4:74:89:87:07:80:ac:8d:a9:94: + 1c:ed:49:0d:5c:0e:75:da:42:71:2f:e4:f9:47:93: + ff:5a:b7:bd:55:6b:cd:d3:de:b6:b2:4e:04:08:23: + 31:70:56:fc:79:e2:71:46:d1:0c:9d:72:6a:f4:b1: + a4:9a:1f:99:17:8b:15:2a:ac:3d:18:1f:1e:06:24: + aa:a4:af:3a:e7:57:4a:a4:f4:67:f8:9b:68:00:3c: + 82:fa:48:89:01:92:a6:e6:62:28:7f:cf:b6:e7:0d: + c6:91:e3:9e:dd:08:4a:66:1f:ed:41:7b:65:ee:05: + 30:34:0a:45:96:58:92:06:6b:0c:61:f9:63:ba:c5: + 18:b6:91:4e:bc:70:6c:a2:85:a9:e1:08:1c:ac:df: + 9b:03:83:10:e5:4a:dc:72:e6:4a:fd:91:8f:de:4e: + d3:5e:8c:65:2e:96:de:06:e5:2a:a9:dc:53:7a:4f: + 90:c7:d1 + Exponent: 65537 (0x10001) + Signature Algorithm: sha256WithRSAEncryption + 0e:4a:aa:22:56:d9:61:20:fe:e6:3f:f0:aa:87:e5:3a:40:a3: + bf:e3:b5:92:3b:b8:ed:ad:0d:17:d4:ad:fd:16:6d:2c:f6:74: + 94:9c:45:6c:1e:b1:59:68:a2:cf:69:18:c3:76:39:fe:d9:44: + d7:12:6b:38:c6:c4:d3:b8:f0:b0:b6:81:80:46:b6:1a:67:da: + 22:8f:5e:cc:39:13:58:64:37:3a:a9:ac:ba:d6:67:c7:01:69: + d5:2d:f4:12:62:be:1c:d7:e3:a3:19:f5:90:dd:e5:ce:d2:92: + e7:73:2f:49:5e:d8:8f:7d:dc:57:ee:e3:f8:1d:1e:3d:27:85: + 82:25:dd:6f:03:4e:1f:f1:4f:6b:ad:31:df:0c:b3:3e:94:c9: + 5e:9b:a5:b7:ef:86:1f:a9:60:22:1b:bb:87:64:51:65:0d:c5: + 83:d7:8a:c3:ef:15:a0:21:fd:9f:97:83:c4:f3:5b:76:3f:6b: + 83:42:94:ac:72:fd:31:3e:e7:4d:fa:c7:a1:b3:27:b5:77:d5: + 98:60:25:fb:23:ef:c6:13:c1:c4:2c:4f:c0:d5:94:11:86:db: + 10:76:a5:ac:da:e2:a2:e3:ac:dd:3e:18:69:c1:2a:0d:64:71: + 97:7b:93:b8:02:99:f5:e4:fe:fa:9a:59:47:a7:9e:e5:48:08: + da:36:1e:11:50:02:0b:e5:ad:13:f5:f7:60:c2:4e:27:82:c5: + 10:a1:3e:7d:6c:84:51:64:b9:6f:b7:59:e1:ed:77:90:75:a8: + 48:c8:ce:90:25:da:01:74:c1:48:80:3b:7e:8c:0f:ce:43:d8: + d0:8e:1a:e6:e1:16:6e:08:e1:b0:e3:5d:c2:96:e3:1c:70:97: + 00:0c:4e:d9:e1:77:a0:88:a1:57:5c:65:35:73:77:bf:da:c3: + a3:0e:ec:a2:e1:1f:d1:de:c1:3f:65:47:09:82:87:db:65:3d: + f2:d8:a3:c1:eb:6e:6a:98:0f:69:34:46:29:11:fb:b1:0d:0e: + 25:80:c5:ae:2a:ad:d2:76:cf:bf:51:6c:c8:d4:30:73:ec:84: + 87:dd:37:b0:f4:28:b6:84:6d:26:15:50:f3:2e:2d:74:40:49: + 36:f0:19:d4:33:f3:d5:e3:e3:a5:04:28:51:6b:f3:ee:15:d2: + 7e:fb:87:5a:74:c6:71:57:f0:39:11:8e:1c:05:05:05:ca:40: + d0:6e:e0:05:41:fd:78:78:fe:ce:64:b5:25:02:fc:1b:39:19: + 0b:97:3e:02:38:b5:86:75:1e:b5:7e:5a:8d:fe:42:ed:59:58: + 69:47:10:3d:c1:6e:b7:b1:f2:75:50:fa:d2:81:71:1d:42:32: + ce:b3:a7:28:71:46:8c:d1 +-----BEGIN CERTIFICATE----- +MIIE8jCCAtoCCQDzSZIPi1W7EjANBgkqhkiG9w0BAQsFADA7MRAwDgYDVQQKDAdN +b25nb0RCMREwDwYDVQQLDAhEZXZUb29sczEUMBIGA1UEAwwLRGV2VG9vbHMgQ0Ew +IBcNMjAxMjIyMTI1NDE5WhgPMjI5NDEwMDYxMjU0MTlaMDkxEDAOBgNVBAoMB01v +bmdvREIxETAPBgNVBAsMCERldlRvb2xzMRIwEAYDVQQDDAlsb2NhbGhvc3QwggIi +MA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCzV9L3MUYLdroDpmVTzG6Pj0iC +4XDZMX4jGGI98+N8EoqaEEJqQNJI8i4vofOSNUvr5MFWNhB3DhPrPcyV4SctekXv +CvEZ1axs0Jth0F8ACiXOIMqCpx0gQNg7Ol3de/GyRpIWfu6X4I9s3gV5KQudkt+p +geRTeB8mRI7SpWYzdATXFvSfnui02GIbK5UULUY1+Lixy1Ih30kpBIIvjWbhGesR +apvPpCHwVLMWOac0dVz2YlA5cQZbPuH/3CV8/AU6hItxBR9fFXg34NwfcMOHq3Ai +GJiBjsMsmZasDyjq/cbNtF+DR3dpDkI0jQp8qXHJCgz0ywPCZNySMoKJu3NP4vPe +4skbhXE8iHJIOFRJtm0x13Gn83W9cBtPO5PUNAV/Gi3xvtXuh7GgXypS2G65ihyy +PYCodJsG/8PYnPRQD/OqusKAQ0Sqtteuf79fpHSJhweArI2plBztSQ1cDnXaQnEv +5PlHk/9at71Va83T3rayTgQIIzFwVvx54nFG0Qydcmr0saSaH5kXixUqrD0YHx4G +JKqkrzrnV0qk9Gf4m2gAPIL6SIkBkqbmYih/z7bnDcaR457dCEpmH+1Be2XuBTA0 +CkWWWJIGawxh+WO6xRi2kU68cGyihanhCBys35sDgxDlStxy5kr9kY/eTtNejGUu +lt4G5Sqp3FN6T5DH0QIDAQABMA0GCSqGSIb3DQEBCwUAA4ICAQAOSqoiVtlhIP7m +P/Cqh+U6QKO/47WSO7jtrQ0X1K39Fm0s9nSUnEVsHrFZaKLPaRjDdjn+2UTXEms4 +xsTTuPCwtoGARrYaZ9oij17MORNYZDc6qay61mfHAWnVLfQSYr4c1+OjGfWQ3eXO +0pLncy9JXtiPfdxX7uP4HR49J4WCJd1vA04f8U9rrTHfDLM+lMlem6W374YfqWAi +G7uHZFFlDcWD14rD7xWgIf2fl4PE81t2P2uDQpSscv0xPudN+sehsye1d9WYYCX7 +I+/GE8HELE/A1ZQRhtsQdqWs2uKi46zdPhhpwSoNZHGXe5O4Apn15P76mllHp57l +SAjaNh4RUAIL5a0T9fdgwk4ngsUQoT59bIRRZLlvt1nh7XeQdahIyM6QJdoBdMFI +gDt+jA/OQ9jQjhrm4RZuCOGw413CluMccJcADE7Z4XegiKFXXGU1c3e/2sOjDuyi +4R/R3sE/ZUcJgofbZT3y2KPB625qmA9pNEYpEfuxDQ4lgMWuKq3Sds+/UWzI1DBz +7ISH3Tew9Ci2hG0mFVDzLi10QEk28BnUM/PV4+OlBChRa/PuFdJ++4dadMZxV/A5 +EY4cBQUFykDQbuAFQf14eP7OZLUlAvwbORkLlz4COLWGdR61flqN/kLtWVhpRxA9 +wW63sfJ1UPrSgXEdQjLOs6cocUaM0Q== +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIIJJwIBAAKCAgEAs1fS9zFGC3a6A6ZlU8xuj49IguFw2TF+IxhiPfPjfBKKmhBC +akDSSPIuL6HzkjVL6+TBVjYQdw4T6z3MleEnLXpF7wrxGdWsbNCbYdBfAAolziDK +gqcdIEDYOzpd3XvxskaSFn7ul+CPbN4FeSkLnZLfqYHkU3gfJkSO0qVmM3QE1xb0 +n57otNhiGyuVFC1GNfi4sctSId9JKQSCL41m4RnrEWqbz6Qh8FSzFjmnNHVc9mJQ +OXEGWz7h/9wlfPwFOoSLcQUfXxV4N+DcH3DDh6twIhiYgY7DLJmWrA8o6v3GzbRf +g0d3aQ5CNI0KfKlxyQoM9MsDwmTckjKCibtzT+Lz3uLJG4VxPIhySDhUSbZtMddx +p/N1vXAbTzuT1DQFfxot8b7V7oexoF8qUthuuYocsj2AqHSbBv/D2Jz0UA/zqrrC +gENEqrbXrn+/X6R0iYcHgKyNqZQc7UkNXA512kJxL+T5R5P/Wre9VWvN0962sk4E +CCMxcFb8eeJxRtEMnXJq9LGkmh+ZF4sVKqw9GB8eBiSqpK8651dKpPRn+JtoADyC ++kiJAZKm5mIof8+25w3GkeOe3QhKZh/tQXtl7gUwNApFlliSBmsMYfljusUYtpFO +vHBsooWp4QgcrN+bA4MQ5UrccuZK/ZGP3k7TXoxlLpbeBuUqqdxTek+Qx9ECAwEA +AQKCAgAnYC/0j//2AmJC7OSwamEX6ficq8ywExrDr9XaDlMy6Ys+2jUKySKiW3Hi +iWdGAaeiqj38pLAmw5OCAbaKbb3BUWhLXJak+nH6Di8fYbWJ97BnOnHcD77oVvRi +xKTZPR5K6w2mHMZb2DUFTRXcjTN8rEOpHti3rtb/+ut0ulXfFCj/5I1fAm/LTPr1 +RjNcE7innu1GNALbZv/UHqZKtHwTDjK7RzHgfAPrlV0gkxXwAZigi+NbSUKBBCRw +nbApadfoCjPnY52IrdkQQ7FwwbxcbDqZcdcCEwstHqejLUzpk0Bz55dU5nsEONo2 +WYUk67sMzcUcdfpkUurOuoJIpTxivrShR7b96YXFo3oDSKL0YxxPrBQhzDieEa4d +BM1j9VyOXJxOANo/T9dM2nDx9/sEPKapZ/91OHk2qackDxtY3U4IHSo0Gcgz96jX +DAjLjRawhJPww/OyNslDs5Cqeeh0j5wohYtoNFqXASV+cfFlD8DAAYRuW5Q6cSIu +7OD1GAiNokSlDqyQtvLa4wj9hLVwjHdwF2Iz0rTcr0nTwbZ3n7sQuIrsusSgnu6g +J5Zw84CHBPpY3NNBoOEEMf/L41ge1YESmB3Z5y+2AAE0xayBwmlI42rj8E19nCKD +6VAJG3IhrXkAKkI1za+cKkJud4b+PieTm65FTCcDmAu+xVfCRQKCAQEA2C27QxkH +9i1SYypqk4yrf1eZKqyFwwqZZya+2BLrXoha5rjq8GD+2AFII7+fNDNx/FbxSIqm +bdvYX+TUFx8Ymz3HB9VzT9MutpEcOU7SBkLm6Bxh8oIeHFQJhYwfZb1KyFai6/oQ +sT7Fw3xkMLldn9WzsqpNuntnE8+Q11J+qa40My5qcrTg2DbQ11ripJE1QTVybzRJ +3MJPIb8t3r5G8S9mgQ/6ijnm8n+v9aZlE/Y8Yb60eojKv5Ycrln9RI12sSeQsUjC +q9h2xhYsaDfHnWH90DPn97D0lQi2b1YJPhfzc6UAV7YKTVIWvva9U+4aIidAQAMy +DXJ1jOkXR4CwKwKCAQEA1GEMgbRaXDEfw674lgbG5yKkLdB4BrVKPtwkTsBks60T +hwwC0YMvKhCVZ0avSLH6yjq2kqFn9E6mOG+GN6sNNSTPqY/j6BG3yFNDCN3VzLI8 +/eFvd0QQpUA+SglgMi2jGYT1YK+xAoySQSgqi/rp1WEh3VS2lW2bbgqgFHZ52qif +9WpGnXyJFzZn1OAK53hE0+j1Xmv2AAXHp/O0OVSgopOKvbcLz7hsB2jur23CYepc +QaZMw3tnKgnKd2Sns3lVH0QmAQl1WB2wB5x5OtNAjI3wRMhcUrSQgH0DdXYE3rXE +FJGi4sZ59rm1mKQcaI2qqMyxc9/rjgEeLTt5aVYt8wKCAQAvvYGyYq/AbOfZ4H8i +0Jj7CVRY+TqdBRU1k3fn7d2uxbwVYdb4eOMuvrG4u1OkSowspuKoG35/mmJhYv53 +kLV1ayIuF7Rcd43EYa1y9nYpTdMChWoYDSYrKV/k7znr1O7T6VYXeOUAz5ULA8h4 +ficv6hjCJxv9R81OsbMR6jTwsfjzKJf3dvyEoy5hsL+Gik7RdMUty8VYDE0/baHq +o+i00Lv5WpcAuaLIAOzR+goua12QVHY07UqhHrx8wIDPB9KjctJgZGohWy93tWPf +LBpYJlDQDvZ2W8zXsNHhKoXmAZIRAupddGU4CB3f1EuhYl7BFhv5Rvthvto44Szz +7HZ7AoIBAGXk5g44zH2c6AOHqCa6u6VbXm/IrD1zr1p3XkgWUHpQGKVbYuLIUNGa +wUOaizCSi6OJBd1V6T0ymtdK0pdUzvJnZeaRbaQDFCzdnbTcUd5yyYZ5KXSMSHww +VXJKC0gn0y/ENcIcqQ1zChyOu9MLIDNK9edt5GfO/ZqPKHyI1y/MZWpmgdi/Tjq/ +1JMypqilcEhHZanWnGEoZME00IwNCUDXDuK0tssDTXlEo/ew7mjx+y9YLU5An59w +sR4VQcx+8xMcLCulMRKEs9cDALrzbe3Bj7xAcOMRVJ8A2674X5/Gj3va385zsUJp +mTHR0vtqtK1l/+F7VlV8PdEvzrF2MncCggEAEYMiXzTs+TRtTRLBH+vsaa6hu5KE +jMST+4q7opl/joscp+LYlS4y2oUp+LBQX2RnujR1Rfp/TFd/mCwOx5v4n4bzFqUu +jQaxbx8afiP1lIhxT1+TmoXT1X+y3FssALULzZzHY5zwx/r3zLOk6FgbnmRshF4S +VWbFOIooO53ZIW0QhyhLrRft8aGzGv7j2Sz5oesclC/8N/HRyDCaN7Wr3ES0xbSE +cg7cMGwxUownhLq5xubwyve6mO0Hg1fEqt/VhkHzuFjSUCMz0GZcEfWC9iEBU97Z +cIe6NXM3rtfuevnfJtRZrsAJeJtPLDJtsxh2VkrmH/A/nYpJmKMY+nh2TA== +-----END RSA PRIVATE KEY----- diff --git a/packages/devtools-proxy-support/test/fixtures/sshd.key b/packages/devtools-proxy-support/test/fixtures/sshd.key new file mode 100644 index 00000000..4503961f --- /dev/null +++ b/packages/devtools-proxy-support/test/fixtures/sshd.key @@ -0,0 +1,15 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDus6d2ccviyVDeC0D+7he6yznH7tninsVOBvUdXDDiiSSMZKra +9+sPKrsrTpzez1PbIaR6N5EAi5Tmq1hIAtxroopm8BXqAEoIfdYTPUrnT6WiKDak +FZ8608N2g4QEcvSFuD0Jzm3yrqbmKno0aAJFB1kjfgDyyUkWKjL7el4hEQIDAQAB +AoGBALMFG+ASAmZIs5SD/i/zYuqdxezzboeuqa0WPLjTTnpnfnioexcT/j92vb8C +C1ZzSaG7vn5GtEIjbP3+nYwMUUAZOV/xhZJxbM5vZF5/yjw+rbWK7HKuM+fmks72 +mrAonBaFVudVxNRQvsPkZVwE+uAURgYEtiWJW3sPVWcdhjARAkEA/9XlGwqQRV41 +HkhgrlXCBCIz09bv3e2/ZSOFohMd7zCLjPOs3iijd/xfIpsuxJd9musCm2plX1/K +EzTNd8p87QJBAO7a8HnCMBAKmmL9+fRzXJKVMiCMx6WX8H6YaXWEzZnnw4PD7K10 +vgPPfQpFXZ4W5D23FOA2UepDRqs2VbQ21DUCQFhrGkVgP0BaMM2IgjF+XhGDqJnc +PQZcdruDrVm4da4G/xP125bkQKrlRBP3whAbs0NpWXtRKDvwJSzCIQj3qHkCQQCb +x9FlKABexfuRKqH2C7NJquLJlee0GZdxiYfmbJoHkb/TVVoseuJe69ladIktTTLJ +CXolDWh5iC00Bzj4U3YtAkBXuawsJOIbxaMBlKkm0Q1g+K+htoC+B62ZQkXvCzDl +QOpN8MM1Xb+/qxEVEVtnGprUqEVF4CQEy6IBAHzUn5s+ +-----END RSA PRIVATE KEY----- diff --git a/packages/devtools-proxy-support/test/helpers.ts b/packages/devtools-proxy-support/test/helpers.ts new file mode 100644 index 00000000..bad3dd8a --- /dev/null +++ b/packages/devtools-proxy-support/test/helpers.ts @@ -0,0 +1,194 @@ +import { once } from 'events'; +import { readFileSync } from 'fs'; +import type { + Server as HTTPServer, + IncomingMessage, + RequestListener, +} from 'http'; +import type { Server as HTTPSServer } from 'https'; +import { createServer as createHTTPSServer } from 'https'; +import type { AddressInfo, Server, Socket } from 'net'; +import path from 'path'; +import { createServer as createHTTPServer, get as httpGet } from 'http'; +import type { TcpipRequestInfo } from 'ssh2'; +import { Server as SSHServer } from 'ssh2'; +import DuplexPair from 'duplexpair'; + +import socks5Server from 'socksv5/lib/server'; +import { promisify } from 'util'; + +function parseHTTPAuthHeader(header: string | undefined): [string, string] { + if (!header?.startsWith('Basic ')) return ['', '']; + const [username = '', pw = ''] = Buffer.from(header.split(' ')[1], 'base64') + .toString() + .split(':'); + return [username, pw]; +} + +export class HTTPServerProxyTestSetup { + // Target servers: These actually handle requests. + readonly httpServer: HTTPServer; + readonly httpsServer: HTTPSServer; + // Proxy servers: + readonly socks5ProxyServer: Server & { useAuth: any }; + readonly httpProxyServer: HTTPServer; + readonly httpsProxyServer: HTTPServer; + readonly sshServer: SSHServer; + readonly sshTunnelInfos: TcpipRequestInfo[] = []; + canTunnel = true; + authHandler: undefined | ((username: string, password: string) => boolean); + + get httpServerPort(): number { + return (this.httpServer.address() as AddressInfo).port; + } + get httpsServerPort(): number { + return (this.httpsServer.address() as AddressInfo).port; + } + get socks5ProxyPort(): number { + return (this.socks5ProxyServer.address() as AddressInfo).port; + } + get httpProxyPort(): number { + return (this.httpProxyServer.address() as AddressInfo).port; + } + get httpsProxyPort(): number { + return (this.httpsProxyServer.address() as AddressInfo).port; + } + get sshProxyPort(): number { + return (this.sshServer.address() as AddressInfo).port; + } + + requests: IncomingMessage[]; + tlsOptions = Object.freeze({ + key: readFileSync(path.resolve(__dirname, 'fixtures', 'server.bundle.pem')), + cert: readFileSync( + path.resolve(__dirname, 'fixtures', 'server.bundle.pem') + ), + ca: readFileSync(path.resolve(__dirname, 'fixtures', 'ca.crt')), + sshdKey: readFileSync(path.resolve(__dirname, 'fixtures', 'sshd.key')), + }); + + constructor() { + this.requests = []; + const handler: RequestListener = (req, res) => { + this.requests.push(req); + res.writeHead(200); + res.end(`OK ${req.url ?? ''}`); + }; + this.httpServer = createHTTPServer(handler); + this.httpsServer = createHTTPSServer({ ...this.tlsOptions }, handler); + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion + this.socks5ProxyServer = (socks5Server as any).createServer( + ( + info: any, + accept: (intercept: true) => Socket + //deny: () => void + ): void => { + const socket = accept(true); + this.httpServer.emit('connection', socket); + } + ); + + this.httpProxyServer = createHTTPServer((req, res) => { + const [username, pw] = parseHTTPAuthHeader( + req.headers['proxy-authorization'] + ); + if (this.authHandler?.(username, pw) === false) { + res.writeHead(407); + res.end(); + return; + } + httpGet( + req.url!, + { + createConnection: () => { + const { socket1, socket2 } = new DuplexPair(); + this.httpServer.emit('connection', socket2); + return socket1; + }, + }, + (proxyRes) => proxyRes.pipe(res) + ); + }); + + this.httpsProxyServer = createHTTPServer(() => { + throw new Error('should not use normal req/res handler'); + }).on('connect', (req, socket, head) => { + const [username, pw] = parseHTTPAuthHeader( + req.headers['proxy-authorization'] + ); + if (this.authHandler?.(username, pw) === false) { + socket.end('HTTP/1.0 407 Proxy Authentication Required\r\n\r\n'); + return; + } + socket.unshift(head); + this.httpsServer.emit('connection', socket); + socket.write('HTTP/1.0 200 OK\r\n\r\n'); + }); + + this.sshServer = new SSHServer( + { + hostKeys: [this.tlsOptions.sshdKey], + }, + (client) => { + client + .on('authentication', (ctx) => { + if (ctx.method === 'none' && !this.authHandler) return ctx.accept(); + if ( + ctx.method === 'password' && + this.authHandler?.(ctx.username, ctx.password) + ) + return ctx.accept(); + return ctx.reject(); + }) + .on('ready', () => { + client.on('tcpip', (accept, reject, info) => { + if (!this.canTunnel) { + return reject(); + } + this.sshTunnelInfos.push(info); + this.httpServer.emit('connection', accept()); + }); + }); + } + ); + } + + async listen(): Promise { + await Promise.all( + [ + this.httpServer, + this.httpsServer, + this.socks5ProxyServer, + this.httpProxyServer, + this.httpsProxyServer, + this.sshServer, + ].map((server) => promisify(server.listen.bind(server))(0)) + ); + } + + getRequestedUrls() { + return this.requests.map((r) => + Object.assign(new URL(`http://_`), { + pathname: r.url, + host: r.headers.host, + }).toString() + ); + } + + async teardown() { + const closePromises: Promise[] = []; + for (const server of [ + this.httpServer, + this.httpsServer, + this.socks5ProxyServer, + this.httpProxyServer, + this.httpsProxyServer, + ]) { + server.close(); + closePromises.push(once(server, 'close')); + } + this.sshServer.close(); // Doesn't emit 'close' + await Promise.all(closePromises); + } +} diff --git a/packages/devtools-proxy-support/test/socksv5.d.ts b/packages/devtools-proxy-support/test/socksv5.d.ts new file mode 100644 index 00000000..9c9cc3ab --- /dev/null +++ b/packages/devtools-proxy-support/test/socksv5.d.ts @@ -0,0 +1,12 @@ +declare module 'socksv5/lib/server' { + const mod: any; + export = mod; +} +declare module 'socksv5/lib/auth/None' { + const mod: any; + export = mod; +} +declare module 'socksv5/lib/auth/UserPassword' { + const mod: any; + export = mod; +} From f50e351b0db3cb1127ab45736d03b57215da1ab8 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Mon, 22 Jul 2024 22:32:52 +0200 Subject: [PATCH 07/30] WIP --- packages/devtools-proxy-support/src/ssh.ts | 20 ++++++--- .../test/fixtures/sshd.key | 42 ++++++++++++------- .../test/fixtures/sshd.key.pub | 1 + 3 files changed, 42 insertions(+), 21 deletions(-) create mode 100644 packages/devtools-proxy-support/test/fixtures/sshd.key.pub diff --git a/packages/devtools-proxy-support/src/ssh.ts b/packages/devtools-proxy-support/src/ssh.ts index d3c37f37..fbfc58fa 100644 --- a/packages/devtools-proxy-support/src/ssh.ts +++ b/packages/devtools-proxy-support/src/ssh.ts @@ -1,3 +1,4 @@ +import type { AgentConnectOpts } from 'agent-base'; import { Agent as AgentBase } from 'agent-base'; import type { DevtoolsProxyOptions } from './proxy-options'; import type { AgentWithInitialize } from './agent'; @@ -104,16 +105,23 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { this.logger.emit('ssh:established-connection'); } - override async connect(req: ClientRequest): Promise { - return await this._connect(req); + override async connect( + req: ClientRequest, + connectOpts: AgentConnectOpts + ): Promise { + return await this._connect(req, connectOpts); } - private async _connect(req: ClientRequest, retriesLeft = 1): Promise { + private async _connect( + req: ClientRequest, + connectOpts: AgentConnectOpts, + retriesLeft = 1 + ): Promise { let host = ''; try { // Using the `host` header matches what proxy-agent does - host = req.getHeader('host') as string; - const url = new URL(req.path, `tcp://${host}`); + host = connectOpts.host || (req.getHeader('host') as string); + const url = new URL(req.path, `tcp://${host}:${connectOpts.port}`); await this.initialize(); @@ -130,7 +138,7 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { this.connected = false; if (retriesLeft > 0) { await this.initialize(); - return await this._connect(req, retriesLeft - 1); + return await this._connect(req, connectOpts, retriesLeft - 1); } } throw err; diff --git a/packages/devtools-proxy-support/test/fixtures/sshd.key b/packages/devtools-proxy-support/test/fixtures/sshd.key index 4503961f..2862d6af 100644 --- a/packages/devtools-proxy-support/test/fixtures/sshd.key +++ b/packages/devtools-proxy-support/test/fixtures/sshd.key @@ -1,15 +1,27 @@ ------BEGIN RSA PRIVATE KEY----- -MIICXQIBAAKBgQDus6d2ccviyVDeC0D+7he6yznH7tninsVOBvUdXDDiiSSMZKra -9+sPKrsrTpzez1PbIaR6N5EAi5Tmq1hIAtxroopm8BXqAEoIfdYTPUrnT6WiKDak -FZ8608N2g4QEcvSFuD0Jzm3yrqbmKno0aAJFB1kjfgDyyUkWKjL7el4hEQIDAQAB -AoGBALMFG+ASAmZIs5SD/i/zYuqdxezzboeuqa0WPLjTTnpnfnioexcT/j92vb8C -C1ZzSaG7vn5GtEIjbP3+nYwMUUAZOV/xhZJxbM5vZF5/yjw+rbWK7HKuM+fmks72 -mrAonBaFVudVxNRQvsPkZVwE+uAURgYEtiWJW3sPVWcdhjARAkEA/9XlGwqQRV41 -HkhgrlXCBCIz09bv3e2/ZSOFohMd7zCLjPOs3iijd/xfIpsuxJd9musCm2plX1/K -EzTNd8p87QJBAO7a8HnCMBAKmmL9+fRzXJKVMiCMx6WX8H6YaXWEzZnnw4PD7K10 -vgPPfQpFXZ4W5D23FOA2UepDRqs2VbQ21DUCQFhrGkVgP0BaMM2IgjF+XhGDqJnc -PQZcdruDrVm4da4G/xP125bkQKrlRBP3whAbs0NpWXtRKDvwJSzCIQj3qHkCQQCb -x9FlKABexfuRKqH2C7NJquLJlee0GZdxiYfmbJoHkb/TVVoseuJe69ladIktTTLJ -CXolDWh5iC00Bzj4U3YtAkBXuawsJOIbxaMBlKkm0Q1g+K+htoC+B62ZQkXvCzDl -QOpN8MM1Xb+/qxEVEVtnGprUqEVF4CQEy6IBAHzUn5s+ ------END RSA PRIVATE KEY----- +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn +NhAAAAAwEAAQAAAQEA1se5+kem8bHLbCO8Hi2Dyb5vCb8MYQPibfksAaZBTMOA+31skw8r +0Vf4US/HuwdTzJVN+fFHP9+okS6wmWX2qIs8oX9wRhKBJVY11XFeb2YKQRAQUTZDujBFHg +Emhrse9ooSJYXlF4/XqTtG5myChb+tdpP5dh7JLOw/GUOhuaTjTrWKZ4/duLgVRrB1UHR4 +SY/Brpx3HIQqcYoznUJbkReW21h65BO1P1Pu4brV9dp4EEQPfLbm2hx0yICGw2vvPLe7k3 +10QNa81brQsl+67eLX0PDt9C63ozVkfFR1DdxEvjlzj5ETrgzYDzEPiciY51iaIrZFOkol +LHc3yaThAQAAA9gViGq9FYhqvQAAAAdzc2gtcnNhAAABAQDWx7n6R6bxsctsI7weLYPJvm +8JvwxhA+Jt+SwBpkFMw4D7fWyTDyvRV/hRL8e7B1PMlU358Uc/36iRLrCZZfaoizyhf3BG +EoElVjXVcV5vZgpBEBBRNkO6MEUeASaGux72ihIlheUXj9epO0bmbIKFv612k/l2Hsks7D +8ZQ6G5pONOtYpnj924uBVGsHVQdHhJj8GunHcchCpxijOdQluRF5bbWHrkE7U/U+7hutX1 +2ngQRA98tubaHHTIgIbDa+88t7uTfXRA1rzVutCyX7rt4tfQ8O30LrejNWR8VHUN3ES+OX +OPkROuDNgPMQ+JyJjnWJoitkU6SiUsdzfJpOEBAAAAAwEAAQAAAQAnU6raCQSofMWip2hq +nirjZdsvDaxWlz9+o4FLTAXo6GNVqUGYK876JgFx3C3WMSFG9I+ylFtXdryG2OW9MM5ZTi +Vr7MnCCuFa1M5GptuDyktWXnqeZDFChlQhilRjlx+0RNPNyxaHme8DLbdtubCsjMxWSgID +ft+XOirTlW6nbxOLEx3k1TG4xKA6ds92U2VaSycRpUn76TbO62R09UZi+V1Ysvld6ubbFp +cJX1+EaV3+lOSwV0EHcFale+Dlsh8cNmNzsAjrsAXC0swFTDdRd3eZBkNyIVuHe8hVCdvt +Of1+UGQ3t7zlyoXVdGVqLNq7fqS2u1oKjrAfORoDHpGBAAAAgGk2Ptv/0Umn181+cKSOiP +nWJexpl8rOB5k9IGMvfz+6uc/ZQXotqp2mdG+s9IrIYfaEVJfTbrAWceLhh2TjDPd7xvew +QQxkQsQkXGCXMtWN5d+s7moYS/lalinQT1LxqlejMEZ8vEK3eRrKBwoux1y8p8PkxJiu6J +FLwLeqZtBlAAAAgQD97wDcqvKk5izf+87L7aACJzhuNFXxI9g3FzNCU4atEWn/zLItV754 +T7pLXW5sNFwakwkF8px4GAXaY/FtueluoJuizdTyXGyPi0ZEF3o8QowP2CP+p6Dw5OWa6a +pkusZ67YpO7UxNjkvOcjQXLmW8umSNwrXvBsY4jfc3qDNcMwAAAIEA2Icoimjr7h0pDbed +3avQdSjQz8qRBvhDGm7FzcgpAfMN1tewKFLl3I1Zw5pZMbJJENxmqH04GXPkzyzeJqgqVf +qKsig6q6BMczQWBknTIo8Iw4YofMX3TFuCpmXnJE/jCtUqptmmo1xaY7nIg6bFvfXMcwBt +NJ/fg9/W1+8VmfsAAAAcYW5uYS5oZW5uaW5nc2VuQE0tRFdIVzRSNlc1QwECAwQFBgc= +-----END OPENSSH PRIVATE KEY----- diff --git a/packages/devtools-proxy-support/test/fixtures/sshd.key.pub b/packages/devtools-proxy-support/test/fixtures/sshd.key.pub new file mode 100644 index 00000000..988a76bc --- /dev/null +++ b/packages/devtools-proxy-support/test/fixtures/sshd.key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDWx7n6R6bxsctsI7weLYPJvm8JvwxhA+Jt+SwBpkFMw4D7fWyTDyvRV/hRL8e7B1PMlU358Uc/36iRLrCZZfaoizyhf3BGEoElVjXVcV5vZgpBEBBRNkO6MEUeASaGux72ihIlheUXj9epO0bmbIKFv612k/l2Hsks7D8ZQ6G5pONOtYpnj924uBVGsHVQdHhJj8GunHcchCpxijOdQluRF5bbWHrkE7U/U+7hutX12ngQRA98tubaHHTIgIbDa+88t7uTfXRA1rzVutCyX7rt4tfQ8O30LrejNWR8VHUN3ES+OXOPkROuDNgPMQ+JyJjnWJoitkU6SiUsdzfJpOEB anna.henningsen@M-DWHW4R6W5C From 4195519b3d7a3950c171cf734b09f32e164a1218 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Mon, 22 Jul 2024 23:41:59 +0200 Subject: [PATCH 08/30] WIP --- .../devtools-proxy-support/src/agent.spec.ts | 32 +++++----- packages/devtools-proxy-support/src/agent.ts | 1 + .../devtools-proxy-support/src/fetch.spec.ts | 60 ++++++++++++++++--- .../src/proxy-options.ts | 9 ++- packages/devtools-proxy-support/src/ssh.ts | 18 +++++- .../devtools-proxy-support/test/helpers.ts | 19 +++++- 6 files changed, 109 insertions(+), 30 deletions(-) diff --git a/packages/devtools-proxy-support/src/agent.spec.ts b/packages/devtools-proxy-support/src/agent.spec.ts index 67550686..ac47a240 100644 --- a/packages/devtools-proxy-support/src/agent.spec.ts +++ b/packages/devtools-proxy-support/src/agent.spec.ts @@ -5,8 +5,6 @@ import { get as httpsGet } from 'https'; import { expect } from 'chai'; import sinon from 'sinon'; import { HTTPServerProxyTestSetup } from '../test/helpers'; -import socks5AuthNone from 'socksv5/lib/auth/None'; -import socks5AuthUserPassword from 'socksv5/lib/auth/UserPassword'; describe('createAgent', function () { let setup: HTTPServerProxyTestSetup; @@ -47,7 +45,7 @@ describe('createAgent', function () { context('socks5', function () { it('can connect to a socks5 proxy without auth', async function () { - setup.socks5ProxyServer.useAuth(socks5AuthNone()); + setup.socks5AuthNone(); const res = await get( 'http://example.com/hello', @@ -61,7 +59,7 @@ describe('createAgent', function () { it('can connect to a socks5 proxy with successful auth', async function () { const authHandler = sinon.stub().yields(true); - setup.socks5ProxyServer.useAuth(socks5AuthUserPassword(authHandler)); + setup.socks5AuthUsernamePassword(authHandler); const res = await get( 'http://example.com/hello', @@ -78,7 +76,7 @@ describe('createAgent', function () { it('fails to connect to a socks5 proxy with unsuccessful auth', async function () { const authHandler = sinon.stub().yields(false); - setup.socks5ProxyServer.useAuth(socks5AuthUserPassword(authHandler)); + setup.socks5AuthUsernamePassword(authHandler); try { await get( @@ -95,7 +93,7 @@ describe('createAgent', function () { }); context('http proxy', function () { - it('can connect to a socks5 proxy without auth', async function () { + it('can connect to a http proxy without auth', async function () { const res = await get( 'http://example.com/hello', createAgent({ proxy: `http://127.0.0.1:${setup.httpProxyPort}` }) @@ -106,7 +104,7 @@ describe('createAgent', function () { ]); }); - it('can connect to a socks5 proxy with successful auth', async function () { + it('can connect to a http proxy with successful auth', async function () { setup.authHandler = sinon.stub().returns(true); const res = await get( @@ -122,7 +120,7 @@ describe('createAgent', function () { expect(setup.authHandler).to.have.been.calledOnceWith('foo', 'bar'); }); - it('fails to connect to a socks5 proxy with unsuccessful auth', async function () { + it('fails to connect to a http proxy with unsuccessful auth', async function () { setup.authHandler = sinon.stub().returns(false); const res = await get( @@ -147,7 +145,7 @@ describe('createAgent', function () { ]); }); - it('can connect to a socks5 proxy with successful auth', async function () { + it('can connect to a https proxy with successful auth', async function () { setup.authHandler = sinon.stub().returns(true); const res = await get( @@ -163,7 +161,7 @@ describe('createAgent', function () { expect(setup.authHandler).to.have.been.calledOnceWith('foo', 'bar'); }); - it('fails to connect to a socks5 proxy with unsuccessful auth', async function () { + it('fails to connect to a https proxy with unsuccessful auth', async function () { setup.authHandler = sinon.stub().returns(false); const res = await get( @@ -177,9 +175,9 @@ describe('createAgent', function () { }); context('ssh proxy', function () { - it('can connect to a ssh proxy without auth', async function () { + it('can connect to an ssh proxy without auth', async function () { const res = await get( - 'https://example.com/hello', + 'http://example.com/hello', createAgent({ proxy: `ssh://someuser@127.0.0.1:${setup.sshProxyPort}` }) ); expect(res.body).to.equal('OK /hello'); @@ -187,15 +185,15 @@ describe('createAgent', function () { 'http://example.com/hello', ]); expect(setup.sshTunnelInfos).to.deep.equal([ - { destIP: 'example.com', destPort: 0, srcIP: '127.0.0.1', srcPort: 0 }, + { destIP: 'example.com', destPort: 80, srcIP: '127.0.0.1', srcPort: 0 }, ]); }); - it('can connect to a ssh proxy with successful auth', async function () { + it('can connect to an ssh proxy with successful auth', async function () { setup.authHandler = sinon.stub().returns(true); const res = await get( - 'https://example.com/hello', + 'http://example.com/hello', createAgent({ proxy: `ssh://foo:bar@127.0.0.1:${setup.sshProxyPort}` }) ); expect(res.body).to.equal('OK /hello'); @@ -205,7 +203,7 @@ describe('createAgent', function () { expect(setup.authHandler).to.have.been.calledOnceWith('foo', 'bar'); }); - it('fails to connect to a ssh proxy with unsuccessful auth', async function () { + it('fails to connect to an ssh proxy with unsuccessful auth', async function () { setup.authHandler = sinon.stub().returns(false); try { @@ -223,7 +221,7 @@ describe('createAgent', function () { } }); - it('fails to connect to a ssh proxy with unavailable tunneling', async function () { + it('fails to connect to an ssh proxy with unavailable tunneling', async function () { setup.authHandler = sinon.stub().returns(true); setup.canTunnel = false; diff --git a/packages/devtools-proxy-support/src/agent.ts b/packages/devtools-proxy-support/src/agent.ts index 40fb4b6b..a09af314 100644 --- a/packages/devtools-proxy-support/src/agent.ts +++ b/packages/devtools-proxy-support/src/agent.ts @@ -39,6 +39,7 @@ export function createAgent( const getProxyForUrl = proxyForUrl(proxyOptions); return new ProxyAgent({ getProxyForUrl, + ...proxyOptions, }); } diff --git a/packages/devtools-proxy-support/src/fetch.spec.ts b/packages/devtools-proxy-support/src/fetch.spec.ts index 17a2f87c..1cd6dfa6 100644 --- a/packages/devtools-proxy-support/src/fetch.spec.ts +++ b/packages/devtools-proxy-support/src/fetch.spec.ts @@ -3,7 +3,7 @@ import { createFetch } from './'; import { HTTPServerProxyTestSetup } from '../test/helpers'; describe('createFetch', function () { - it(`consistency check: plain "import('node-fetch') fails"`, async function () { + it("consistency check: plain `import('node-fetch')` fails", async function () { let failed = false; try { await import('node-fetch'); @@ -17,12 +17,12 @@ describe('createFetch', function () { context('HTTP calls', function () { let setup: HTTPServerProxyTestSetup; - before(async function () { + beforeEach(async function () { setup = new HTTPServerProxyTestSetup(); await setup.listen(); }); - after(async function () { + afterEach(async function () { await setup.teardown(); }); @@ -33,12 +33,58 @@ describe('createFetch', function () { expect(await response.text()).to.equal('OK /test'); }); - it.only('makes use of proxy support when instructed to do so', async function () { - const response = await createFetch({ + it('makes use of SSH proxy support when instructed to do so', async function () { + const fetch = createFetch({ proxy: `ssh://someuser@127.0.0.1:${setup.sshProxyPort}`, - })(`http://127.0.0.1:${setup.httpServerPort}/test`); + }); + const response = await fetch( + `http://localhost:${setup.httpServerPort}/test` + ); + expect(await response.text()).to.equal('OK /test'); + expect(setup.sshTunnelInfos).to.deep.equal([ + { + destIP: 'localhost', + destPort: setup.httpServerPort, + srcIP: '127.0.0.1', + srcPort: 0, + }, + ]); + fetch.agent?.destroy?.(); + }); + + it('makes use of HTTP proxy support when instructed to do so', async function () { + const fetch = createFetch({ + proxy: `http://127.0.0.1:${setup.httpProxyPort}`, + }); + const response = await fetch( + `http://localhost:${setup.httpServerPort}/test` + ); + expect(await response.text()).to.equal('OK /test'); + fetch.agent?.destroy?.(); + }); + + it('makes use of HTTPS proxy support when instructed to do so', async function () { + const fetch = createFetch({ + proxy: `http://127.0.0.1:${setup.httpsProxyPort}`, + ca: setup.tlsOptions.ca, + }); + const response = await fetch( + `https://localhost:${setup.httpsServerPort}/test` + ); + expect(await response.text()).to.equal('OK /test'); + fetch.agent?.destroy?.(); + }); + + it('makes use of Socks5 proxy support when instructed to do so', async function () { + setup.socks5AuthNone(); + const fetch = createFetch({ + proxy: `socks5://127.0.0.1:${setup.socks5ProxyPort}`, + }); + const response = await fetch( + `http://localhost:${setup.httpServerPort}/test` + ); expect(await response.text()).to.equal('OK /test'); - expect(setup.sshTunnelInfos).to.deep.equal([]); + fetch.agent?.destroy?.(); }); }); }); diff --git a/packages/devtools-proxy-support/src/proxy-options.ts b/packages/devtools-proxy-support/src/proxy-options.ts index f1dd008f..5f300a6c 100644 --- a/packages/devtools-proxy-support/src/proxy-options.ts +++ b/packages/devtools-proxy-support/src/proxy-options.ts @@ -1,7 +1,9 @@ +import type { ConnectionOptions } from 'tls'; + // Should be an opaque type, but TS does not support those. export type DevtoolsProxyOptionsSecrets = string; -export interface DevtoolsProxyOptions { +export type DevtoolsProxyOptions = { // Can be an ssh://, socks5://, http://, https:// or pac<...>:// URL // Everything besides ssh:// gets forwarded to the `proxy-agent` npm package proxy?: string; @@ -14,7 +16,10 @@ export interface DevtoolsProxyOptions { identityKeyFile?: string; identityKeyPassphrase?: string; }; -} +} & Pick< + ConnectionOptions, + 'ca' | 'cert' | 'crl' | 'key' | 'passphrase' | 'pfx' +>; // https://www.electronjs.org/docs/latest/api/structures/proxy-config interface ElectronProxyConfig { diff --git a/packages/devtools-proxy-support/src/ssh.ts b/packages/devtools-proxy-support/src/ssh.ts index fbfc58fa..5696eacc 100644 --- a/packages/devtools-proxy-support/src/ssh.ts +++ b/packages/devtools-proxy-support/src/ssh.ts @@ -10,6 +10,8 @@ import EventEmitter, { once } from 'events'; import { promises as fs } from 'fs'; import { promisify } from 'util'; import type { ProxyLogEmitter } from './logging'; +import { connect as tlsConnect } from 'tls'; +import type { Socket } from 'net'; export class SSHAgent extends AgentBase implements AgentWithInitialize { public logger: ProxyLogEmitter; @@ -68,6 +70,7 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { ? await fs.readFile(this.proxyOptions.sshOptions.identityKeyFile) : undefined, passphrase: this.proxyOptions.sshOptions?.identityKeyPassphrase, + // debug: console.log.bind(null, '[client]') }; this.logger.emit('ssh:establishing-conection', { @@ -125,7 +128,20 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { await this.initialize(); - return await this.forwardOut('127.0.0.1', 0, url.hostname, +url.port); + let sock: Duplex & Partial> = + await this.forwardOut('127.0.0.1', 0, url.hostname, +url.port); + (sock as any).setTimeout ??= function () { + // noop, required for node-fetch + return this; + }; + if (connectOpts.secureEndpoint) { + sock = tlsConnect({ + ...this.proxyOptions, + ...connectOpts, + socket: sock, + }); + } + return sock; } catch (err: unknown) { const retryableError = (err as Error).message === 'Not connected'; this.logger.emit('ssh:failed-forward', { diff --git a/packages/devtools-proxy-support/test/helpers.ts b/packages/devtools-proxy-support/test/helpers.ts index bad3dd8a..02f49b67 100644 --- a/packages/devtools-proxy-support/test/helpers.ts +++ b/packages/devtools-proxy-support/test/helpers.ts @@ -13,9 +13,11 @@ import { createServer as createHTTPServer, get as httpGet } from 'http'; import type { TcpipRequestInfo } from 'ssh2'; import { Server as SSHServer } from 'ssh2'; import DuplexPair from 'duplexpair'; +import { promisify } from 'util'; import socks5Server from 'socksv5/lib/server'; -import { promisify } from 'util'; +import socks5AuthNone from 'socksv5/lib/auth/None'; +import socks5AuthUserPassword from 'socksv5/lib/auth/UserPassword'; function parseHTTPAuthHeader(header: string | undefined): [string, string] { if (!header?.startsWith('Basic ')) return ['', '']; @@ -129,6 +131,7 @@ export class HTTPServerProxyTestSetup { this.sshServer = new SSHServer( { hostKeys: [this.tlsOptions.sshdKey], + // debug: console.log.bind(null, '[server]') }, (client) => { client @@ -167,7 +170,17 @@ export class HTTPServerProxyTestSetup { ); } - getRequestedUrls() { + socks5AuthNone(): void { + this.socks5ProxyServer.useAuth(socks5AuthNone()); + } + + socks5AuthUsernamePassword( + cb: (user: string, pass: string, cb: (success: boolean) => void) => void + ): void { + this.socks5ProxyServer.useAuth(socks5AuthUserPassword(cb)); + } + + getRequestedUrls(): string[] { return this.requests.map((r) => Object.assign(new URL(`http://_`), { pathname: r.url, @@ -176,7 +189,7 @@ export class HTTPServerProxyTestSetup { ); } - async teardown() { + async teardown(): Promise { const closePromises: Promise[] = []; for (const server of [ this.httpServer, From 8eadc41c2d161081813a87aff0c29215ddec8b4b Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 23 Jul 2024 13:53:10 +0200 Subject: [PATCH 09/30] WIP --- package-lock.json | 532 +++++++++++++++++- packages/devtools-proxy-support/package.json | 4 +- .../devtools-proxy-support/src/agent.spec.ts | 12 +- .../devtools-proxy-support/src/fetch.spec.ts | 34 +- .../src/proxy-options.spec.ts | 355 ++++++++++++ .../src/proxy-options.ts | 139 +++-- .../test/electron-test-server.js | 27 + .../devtools-proxy-support/test/helpers.ts | 15 +- 8 files changed, 1051 insertions(+), 67 deletions(-) create mode 100644 packages/devtools-proxy-support/src/proxy-options.spec.ts create mode 100644 packages/devtools-proxy-support/test/electron-test-server.js diff --git a/package-lock.json b/package-lock.json index af455079..6290836b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2074,6 +2074,59 @@ "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", "dev": true }, + "node_modules/@electron/get": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@electron/get/-/get-2.0.3.tgz", + "integrity": "sha512-Qkzpg2s9GnVV2I2BjRksUi43U5e6+zaQMcjoJy0C+C5oxaKl+fmckGDQFtRpZpZV0NQekuZZ+tGz7EA9TVnQtQ==", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "env-paths": "^2.2.0", + "fs-extra": "^8.1.0", + "got": "^11.8.5", + "progress": "^2.0.3", + "semver": "^6.2.0", + "sumchecker": "^3.0.1" + }, + "engines": { + "node": ">=12" + }, + "optionalDependencies": { + "global-agent": "^3.0.0" + } + }, + "node_modules/@electron/get/node_modules/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/@electron/get/node_modules/jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", + "dev": true, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/@electron/get/node_modules/universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "dev": true, + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/@emotion/babel-plugin": { "version": "11.11.0", "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz", @@ -7307,6 +7360,16 @@ "integrity": "sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==", "dev": true }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "dev": true, + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.62.0.tgz", @@ -8742,6 +8805,13 @@ "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "dev": true, + "optional": true + }, "node_modules/bplist-parser": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/bplist-parser/-/bplist-parser-0.2.0.tgz", @@ -10787,6 +10857,13 @@ "node": ">=8" } }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "dev": true, + "optional": true + }, "node_modules/dicer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.3.0.tgz", @@ -11065,11 +11142,38 @@ "node": ">=0.10.0" } }, + "node_modules/electron": { + "version": "31.2.1", + "resolved": "https://registry.npmjs.org/electron/-/electron-31.2.1.tgz", + "integrity": "sha512-g3CLKjl4yuXt6VWm/KpgEjYYhFiCl19RgUn8lOC8zV/56ZXAS3+mqV4wWzicE/7vSYXs6GRO7vkYRwrwhX3Gaw==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "@electron/get": "^2.0.0", + "@types/node": "^20.9.0", + "extract-zip": "^2.0.1" + }, + "bin": { + "electron": "cli.js" + }, + "engines": { + "node": ">= 12.20.55" + } + }, "node_modules/electron-to-chromium": { "version": "1.4.454", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.454.tgz", "integrity": "sha512-pmf1rbAStw8UEQ0sr2cdJtWl48ZMuPD9Sto8HVQOq9vx9j2WgDEN6lYoaqFvqEHYOmGA9oRGn7LqWI9ta0YugQ==" }, + "node_modules/electron/node_modules/@types/node": { + "version": "20.14.11", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.11.tgz", + "integrity": "sha512-kprQpL8MMeszbz6ojB5/tU8PLN4kesnN8Gjzw349rDlNgsSzg90lAVj3llK99Dh7JON+t9AuscPPFW6mPbTnSA==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, "node_modules/email-validator": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/email-validator/-/email-validator-2.0.4.tgz", @@ -12411,6 +12515,41 @@ "node": ">=0.6.0" } }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/extract-zip/node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dev": true, + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", @@ -13608,6 +13747,37 @@ "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", "dev": true }, + "node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "dev": true, + "optional": true, + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/global-agent/node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "dev": true, + "optional": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/global-jsdom": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/global-jsdom/-/global-jsdom-9.1.0.tgz", @@ -17659,6 +17829,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "dev": true, + "optional": true, + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -21948,6 +22131,31 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "dev": true, + "optional": true, + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/roarr/node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "dev": true, + "optional": true + }, "node_modules/rrweb-cssom": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz", @@ -22320,6 +22528,35 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, + "node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "dev": true, + "optional": true, + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/serialize-error/node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "dev": true, + "optional": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/serialize-javascript": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", @@ -23175,6 +23412,18 @@ "integrity": "sha512-Orov6g6BB1sDfYgzWfTHDOxamtX1bE/zo104Dh9e6fqJ3PooipYyfJ0pUmrZO2wAvO8YbEyeFrkV91XTsGMSrw==", "dev": true }, + "node_modules/sumchecker": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/sumchecker/-/sumchecker-3.0.1.tgz", + "integrity": "sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg==", + "dev": true, + "dependencies": { + "debug": "^4.1.0" + }, + "engines": { + "node": ">= 8.0" + } + }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -24778,6 +25027,46 @@ "node": ">=0.4" } }, + "node_modules/xvfb-maybe": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/xvfb-maybe/-/xvfb-maybe-0.2.1.tgz", + "integrity": "sha512-9IyRz3l6Qyhl6LvnGRF5jMPB4oBEepQnuzvVAFTynP6ACLLSevqigICJ9d/+ofl29m2daeaVBChnPYUnaeJ7yA==", + "dev": true, + "dependencies": { + "debug": "^2.2.0", + "which": "^1.2.4" + }, + "bin": { + "xvfb-maybe": "src/xvfb-maybe.js" + } + }, + "node_modules/xvfb-maybe/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/xvfb-maybe/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "node_modules/xvfb-maybe/node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -25673,13 +25962,15 @@ "chai": "^4.3.6", "depcheck": "^1.4.1", "duplexpair": "^1.0.2", + "electron": "^31.2.1", "eslint": "^7.25.0", "gen-esm-wrapper": "^1.1.0", "mocha": "^8.4.0", "nyc": "^15.1.0", "prettier": "^2.3.2", "sinon": "^9.2.3", - "typescript": "^5.0.4" + "typescript": "^5.0.4", + "xvfb-maybe": "^0.2.1" } }, "packages/devtools-proxy-support/node_modules/agent-base": { @@ -28453,6 +28744,50 @@ } } }, + "@electron/get": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@electron/get/-/get-2.0.3.tgz", + "integrity": "sha512-Qkzpg2s9GnVV2I2BjRksUi43U5e6+zaQMcjoJy0C+C5oxaKl+fmckGDQFtRpZpZV0NQekuZZ+tGz7EA9TVnQtQ==", + "dev": true, + "requires": { + "debug": "^4.1.1", + "env-paths": "^2.2.0", + "fs-extra": "^8.1.0", + "global-agent": "^3.0.0", + "got": "^11.8.5", + "progress": "^2.0.3", + "semver": "^6.2.0", + "sumchecker": "^3.0.1" + }, + "dependencies": { + "fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dev": true, + "requires": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, + "jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6" + } + }, + "universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "dev": true + } + } + }, "@emotion/babel-plugin": { "version": "11.11.0", "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.11.0.tgz", @@ -32127,6 +32462,7 @@ "chai": "^4.3.6", "depcheck": "^1.4.1", "duplexpair": "^1.0.2", + "electron": "^31.2.1", "eslint": "^7.25.0", "gen-esm-wrapper": "^1.1.0", "mocha": "^8.4.0", @@ -32137,7 +32473,8 @@ "sinon": "^9.2.3", "socksv5": "^0.0.6", "ssh2": "^1.15.0", - "typescript": "^5.0.4" + "typescript": "^5.0.4", + "xvfb-maybe": "^0.2.1" }, "dependencies": { "agent-base": { @@ -34279,6 +34616,16 @@ "integrity": "sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==", "dev": true }, + "@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "dev": true, + "optional": true, + "requires": { + "@types/node": "*" + } + }, "@typescript-eslint/eslint-plugin": { "version": "5.62.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-5.62.0.tgz", @@ -35399,6 +35746,13 @@ "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" }, + "boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "dev": true, + "optional": true + }, "bplist-parser": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/bplist-parser/-/bplist-parser-0.2.0.tgz", @@ -36900,6 +37254,13 @@ "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", "optional": true }, + "detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "dev": true, + "optional": true + }, "dicer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.3.0.tgz", @@ -37101,6 +37462,28 @@ "jake": "^10.8.5" } }, + "electron": { + "version": "31.2.1", + "resolved": "https://registry.npmjs.org/electron/-/electron-31.2.1.tgz", + "integrity": "sha512-g3CLKjl4yuXt6VWm/KpgEjYYhFiCl19RgUn8lOC8zV/56ZXAS3+mqV4wWzicE/7vSYXs6GRO7vkYRwrwhX3Gaw==", + "dev": true, + "requires": { + "@electron/get": "^2.0.0", + "@types/node": "^20.9.0", + "extract-zip": "^2.0.1" + }, + "dependencies": { + "@types/node": { + "version": "20.14.11", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.11.tgz", + "integrity": "sha512-kprQpL8MMeszbz6ojB5/tU8PLN4kesnN8Gjzw349rDlNgsSzg90lAVj3llK99Dh7JON+t9AuscPPFW6mPbTnSA==", + "dev": true, + "requires": { + "undici-types": "~5.26.4" + } + } + } + }, "electron-to-chromium": { "version": "1.4.454", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.454.tgz", @@ -38125,6 +38508,29 @@ } } }, + "extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "dev": true, + "requires": { + "@types/yauzl": "^2.9.1", + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "dependencies": { + "get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dev": true, + "requires": { + "pump": "^3.0.0" + } + } + } + }, "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", @@ -39035,6 +39441,30 @@ "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==", "dev": true }, + "global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "dev": true, + "optional": true, + "requires": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "dependencies": { + "semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "dev": true, + "optional": true + } + } + }, "global-jsdom": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/global-jsdom/-/global-jsdom-9.1.0.tgz", @@ -42124,6 +42554,16 @@ "integrity": "sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==", "dev": true }, + "matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "dev": true, + "optional": true, + "requires": { + "escape-string-regexp": "^4.0.0" + } + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -45506,6 +45946,30 @@ "glob": "^7.1.3" } }, + "roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "dev": true, + "optional": true, + "requires": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "dependencies": { + "sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "dev": true, + "optional": true + } + } + }, "rrweb-cssom": { "version": "0.6.0", "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz", @@ -45800,6 +46264,25 @@ } } }, + "serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "dev": true, + "optional": true, + "requires": { + "type-fest": "^0.13.1" + }, + "dependencies": { + "type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "dev": true, + "optional": true + } + } + }, "serialize-javascript": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", @@ -46467,6 +46950,15 @@ "integrity": "sha512-Orov6g6BB1sDfYgzWfTHDOxamtX1bE/zo104Dh9e6fqJ3PooipYyfJ0pUmrZO2wAvO8YbEyeFrkV91XTsGMSrw==", "dev": true }, + "sumchecker": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/sumchecker/-/sumchecker-3.0.1.tgz", + "integrity": "sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg==", + "dev": true, + "requires": { + "debug": "^4.1.0" + } + }, "supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -47678,6 +48170,42 @@ "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" }, + "xvfb-maybe": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/xvfb-maybe/-/xvfb-maybe-0.2.1.tgz", + "integrity": "sha512-9IyRz3l6Qyhl6LvnGRF5jMPB4oBEepQnuzvVAFTynP6ACLLSevqigICJ9d/+ofl29m2daeaVBChnPYUnaeJ7yA==", + "dev": true, + "requires": { + "debug": "^2.2.0", + "which": "^1.2.4" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "requires": { + "isexe": "^2.0.0" + } + } + } + }, "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/packages/devtools-proxy-support/package.json b/packages/devtools-proxy-support/package.json index fdfd3849..74ac5778 100644 --- a/packages/devtools-proxy-support/package.json +++ b/packages/devtools-proxy-support/package.json @@ -64,12 +64,14 @@ "chai": "^4.3.6", "depcheck": "^1.4.1", "duplexpair": "^1.0.2", + "electron": "^31.2.1", "eslint": "^7.25.0", "gen-esm-wrapper": "^1.1.0", "mocha": "^8.4.0", "nyc": "^15.1.0", "prettier": "^2.3.2", "sinon": "^9.2.3", - "typescript": "^5.0.4" + "typescript": "^5.0.4", + "xvfb-maybe": "^0.2.1" } } diff --git a/packages/devtools-proxy-support/src/agent.spec.ts b/packages/devtools-proxy-support/src/agent.spec.ts index ac47a240..bae5635f 100644 --- a/packages/devtools-proxy-support/src/agent.spec.ts +++ b/packages/devtools-proxy-support/src/agent.spec.ts @@ -93,7 +93,7 @@ describe('createAgent', function () { }); context('http proxy', function () { - it('can connect to a http proxy without auth', async function () { + it('can connect to an http proxy without auth', async function () { const res = await get( 'http://example.com/hello', createAgent({ proxy: `http://127.0.0.1:${setup.httpProxyPort}` }) @@ -104,7 +104,7 @@ describe('createAgent', function () { ]); }); - it('can connect to a http proxy with successful auth', async function () { + it('can connect to an http proxy with successful auth', async function () { setup.authHandler = sinon.stub().returns(true); const res = await get( @@ -120,7 +120,7 @@ describe('createAgent', function () { expect(setup.authHandler).to.have.been.calledOnceWith('foo', 'bar'); }); - it('fails to connect to a http proxy with unsuccessful auth', async function () { + it('fails to connect to an http proxy with unsuccessful auth', async function () { setup.authHandler = sinon.stub().returns(false); const res = await get( @@ -134,7 +134,7 @@ describe('createAgent', function () { }); context('https/connect proxy', function () { - it('can connect to a https proxy without auth', async function () { + it('can connect to an https proxy without auth', async function () { const res = await get( 'https://example.com/hello', createAgent({ proxy: `http://127.0.0.1:${setup.httpsProxyPort}` }) @@ -145,7 +145,7 @@ describe('createAgent', function () { ]); }); - it('can connect to a https proxy with successful auth', async function () { + it('can connect to an https proxy with successful auth', async function () { setup.authHandler = sinon.stub().returns(true); const res = await get( @@ -161,7 +161,7 @@ describe('createAgent', function () { expect(setup.authHandler).to.have.been.calledOnceWith('foo', 'bar'); }); - it('fails to connect to a https proxy with unsuccessful auth', async function () { + it('fails to connect to an https proxy with unsuccessful auth', async function () { setup.authHandler = sinon.stub().returns(false); const res = await get( diff --git a/packages/devtools-proxy-support/src/fetch.spec.ts b/packages/devtools-proxy-support/src/fetch.spec.ts index 1cd6dfa6..588b0ab9 100644 --- a/packages/devtools-proxy-support/src/fetch.spec.ts +++ b/packages/devtools-proxy-support/src/fetch.spec.ts @@ -15,6 +15,7 @@ describe('createFetch', function () { }); context('HTTP calls', function () { + let fetch: ReturnType; let setup: HTTPServerProxyTestSetup; beforeEach(async function () { @@ -24,6 +25,7 @@ describe('createFetch', function () { afterEach(async function () { await setup.teardown(); + fetch?.agent?.destroy?.(); }); it('provides a node-fetch-like HTTP functionality', async function () { @@ -34,7 +36,7 @@ describe('createFetch', function () { }); it('makes use of SSH proxy support when instructed to do so', async function () { - const fetch = createFetch({ + fetch = createFetch({ proxy: `ssh://someuser@127.0.0.1:${setup.sshProxyPort}`, }); const response = await fetch( @@ -49,22 +51,20 @@ describe('createFetch', function () { srcPort: 0, }, ]); - fetch.agent?.destroy?.(); }); it('makes use of HTTP proxy support when instructed to do so', async function () { - const fetch = createFetch({ + fetch = createFetch({ proxy: `http://127.0.0.1:${setup.httpProxyPort}`, }); const response = await fetch( `http://localhost:${setup.httpServerPort}/test` ); expect(await response.text()).to.equal('OK /test'); - fetch.agent?.destroy?.(); }); it('makes use of HTTPS proxy support when instructed to do so', async function () { - const fetch = createFetch({ + fetch = createFetch({ proxy: `http://127.0.0.1:${setup.httpsProxyPort}`, ca: setup.tlsOptions.ca, }); @@ -72,19 +72,37 @@ describe('createFetch', function () { `https://localhost:${setup.httpsServerPort}/test` ); expect(await response.text()).to.equal('OK /test'); - fetch.agent?.destroy?.(); }); it('makes use of Socks5 proxy support when instructed to do so', async function () { setup.socks5AuthNone(); - const fetch = createFetch({ + fetch = createFetch({ proxy: `socks5://127.0.0.1:${setup.socks5ProxyPort}`, }); const response = await fetch( `http://localhost:${setup.httpServerPort}/test` ); expect(await response.text()).to.equal('OK /test'); - fetch.agent?.destroy?.(); + }); + + it('makes use of PAC-based proxy support when instructed to do so', async function () { + setup.socks5AuthNone(); + fetch = createFetch({ + proxy: `pac+http://127.0.0.1:${setup.httpServerPort}/pac`, + }); + const response = await fetch( + `http://localhost:${setup.httpsServerPort}/test` + ); + expect(await response.text()).to.equal('OK /test'); + + try { + await fetch(`http://pac-invalidproxy/test`); + expect.fail('missed exception'); + } catch (err) { + expect(err.message).to.include( + 'Failed to establish a socket connection to proxies: ["SOCKS5 127.0.0.1:1"]' + ); + } }); }); }); diff --git a/packages/devtools-proxy-support/src/proxy-options.spec.ts b/packages/devtools-proxy-support/src/proxy-options.spec.ts new file mode 100644 index 00000000..ceba47b1 --- /dev/null +++ b/packages/devtools-proxy-support/src/proxy-options.spec.ts @@ -0,0 +1,355 @@ +import { expect } from 'chai'; +import type { DevtoolsProxyOptions } from './proxy-options'; +import { + extractProxySecrets, + mergeProxySecrets, + proxyConfForEnvVars, + proxyForUrl, + redactUrl, + translateToElectronProxyConfig, +} from './proxy-options'; +import type { ChildProcess } from 'child_process'; +import { spawn } from 'child_process'; +import path from 'path'; +import { once } from 'events'; +import type { AddressInfo, Server, Socket } from 'net'; +import { createServer } from 'net'; +import { HTTPServerProxyTestSetup } from '../test/helpers'; + +describe('proxy options handling', function () { + describe('proxyConfForEnvVars', function () { + it('should return a map of proxies and noProxy', function () { + const env = { + HTTP_PROXY: 'http://proxy.example.com', + HTTPS_PROXY: 'https://proxy.example.com', + NO_PROXY: 'localhost,127.0.0.1', + }; + + const { map, noProxy } = proxyConfForEnvVars(env); + + expect([...map]).to.deep.equal([ + ['http', 'http://proxy.example.com'], + ['https', 'https://proxy.example.com'], + ]); + expect(noProxy).to.equal('localhost,127.0.0.1'); + }); + }); + + describe('proxyForUrl', function () { + it('should return a proxy function for a specified proxy URL', function () { + const getProxy = proxyForUrl({ proxy: 'http://proxy.example.com' }); + + expect(getProxy('http://target.com')).to.equal( + 'http://proxy.example.com' + ); + }); + + it('should respect noProxyHosts', function () { + const getProxy = proxyForUrl({ + proxy: 'http://proxy.example.com', + noProxyHosts: 'localhost', + }); + + expect(getProxy('http://localhost')).to.equal(''); + expect(getProxy('http://example.com')).to.equal( + 'http://proxy.example.com' + ); + }); + + it('should use environment variables as a fallback', function () { + const getProxy = proxyForUrl({ + useEnvironmentVariableProxies: true, + env: { + HTTP_PROXY: 'socks5://env-proxy.example.com', + NO_PROXY: 'localhost', + }, + }); + + expect(getProxy('http://localhost')).to.equal(''); + expect(getProxy('http://localhost:12345')).to.equal(''); + expect(getProxy('http://example.com')).to.equal( + 'socks5://env-proxy.example.com' + ); + }); + }); + + describe('electron options transformation', function () { + context('unit tests', function () { + it('rejects ssh proxies', function () { + expect(() => + translateToElectronProxyConfig({ proxy: 'ssh://proxy.example.com' }) + ).to.throw( + /Using ssh:\/\/ proxies for generic browser proxy usage is not supported/ + ); + }); + it('rejects authenticated proxies', function () { + expect(() => + translateToElectronProxyConfig({ + proxy: 'http://foo:bar@proxy.example.com', + }) + ).to.throw( + /Using authenticated proxies for generic browser proxy usage is not supported/ + ); + }); + it('rejects unsupported proxy protocols', function () { + expect(() => + translateToElectronProxyConfig({ proxy: 'meow://proxy.example.com' }) + ).to.throw(/Unsupported proxy protocol/); + }); + it('translates a pac file proxy to the corresponding electron config', function () { + expect( + translateToElectronProxyConfig({ + proxy: 'pac+http://example.com/pac', + }) + ).to.deep.equal({ + mode: 'pac_script', + pacScript: 'http://example.com/pac', + proxyBypassRules: undefined, + }); + }); + it('translates a http proxy to the corresponding electron config', function () { + expect( + translateToElectronProxyConfig({ + proxy: 'http://example.com/', + noProxyHosts: 'localhost,example.com', + }) + ).to.deep.equal({ + mode: 'fixed_servers', + proxyBypassRules: 'localhost,example.com', + proxyRules: 'http://example.com', + }); + }); + it('translates an env var proxy config to the corresponding electron config', function () { + expect( + translateToElectronProxyConfig({ + useEnvironmentVariableProxies: true, + noProxyHosts: 'localhost,example.com', + env: { + HTTP_PROXY: 'socks5://env-proxy.example.com', + NO_PROXY: 'zombo.com', + }, + }) + ).to.deep.equal({ + mode: 'fixed_servers', + proxyBypassRules: 'localhost,example.com,zombo.com', + proxyRules: 'http=socks5://env-proxy.example.com', + }); + }); + it('translates an empty config to an empty config', function () { + expect(translateToElectronProxyConfig({})).to.deep.equal({}); + expect( + translateToElectronProxyConfig({ + useEnvironmentVariableProxies: true, + env: {}, + }) + ).to.deep.equal({}); + }); + }); + context('integration tests', function () { + let childProcess: ChildProcess; + let exitPromise: Promise; + let server: Server; + let socket: Socket; + let runJS: (js: string) => Promise; + let testResolveProxy: ( + proxyOptions: DevtoolsProxyOptions, + url: string + ) => Promise; + let setup: HTTPServerProxyTestSetup; + + before(async function () { + setup = new HTTPServerProxyTestSetup(); + await setup.listen(); + + server = createServer(); + server.listen(0); + await once(server, 'listening'); + const connEvent = once(server, 'connection'); + childProcess = spawn( + 'npx', + [ + 'xvfb-maybe', + // eslint-disable-next-line @typescript-eslint/no-var-requires + require('electron') as unknown as string, + path.resolve(__dirname, '..', 'test', 'electron-test-server.js'), + ], + { + env: { + ...process.env, + TEST_SERVER_PORT: String((server.address() as AddressInfo).port), + }, + stdio: 'inherit', + } + ); + exitPromise = once(childProcess, 'exit').catch(() => { + /* ignore */ + }); + await once(childProcess, 'spawn'); + + [socket] = await connEvent; + socket.setEncoding('utf8'); + let buffer = ''; + socket.on('data', (chunk) => (buffer += chunk)); + + runJS = async (js: string) => { + socket.write(JSON.stringify(js) + '\0'); + while (!buffer.includes('\0')) { + await once(socket, 'data'); + } + const response = buffer.substring(0, buffer.indexOf('\0')); + buffer = buffer.substring(buffer.indexOf('\0') + 1); + return JSON.parse(response); + }; + + testResolveProxy = async (proxyOptions, url) => { + return await runJS(`app.setProxy(${JSON.stringify( + translateToElectronProxyConfig(proxyOptions) + )}).then(_ => { + return app.resolveProxy(${JSON.stringify(url)}); + })`); + }; + }); + + after(async function () { + childProcess.kill(); + socket.destroy(); + server.close(); + await Promise.all([ + once(socket, 'close'), + once(server, 'close'), + exitPromise, + setup.teardown(), + ]); + }); + + it('correctly handles explicit proxies', async function () { + expect( + await testResolveProxy( + { + proxy: 'http://example.com:12345', + }, + 'http://example.net' + ) + ).to.equal('PROXY example.com:12345'); + expect( + await testResolveProxy( + { + proxy: 'http://example.com:12345', + noProxyHosts: 'localhost', + }, + 'http://example.net' + ) + ).to.equal('PROXY example.com:12345'); + expect( + await testResolveProxy( + { + proxy: 'http://example.com:12345', + noProxyHosts: 'localhost', + }, + 'http://localhost' + ) + ).to.equal('DIRECT'); + expect( + await testResolveProxy( + { + proxy: 'http://example.com:12345', + noProxyHosts: 'localhost', + }, + 'http://localhost:1234' + ) + ).to.equal('DIRECT'); + expect( + await testResolveProxy( + { + proxy: 'socks5://example.com:12345', + }, + 'http://example.net' + ) + ).to.equal('SOCKS5 example.com:12345'); + }); + + it('correctly handles pac-script-specified proxies', async function () { + expect( + await testResolveProxy( + { + proxy: `pac+http://127.0.0.1:${setup.httpServerPort}/pac`, + }, + 'http://example.com' + ) + ).to.equal(`SOCKS5 127.0.0.1:${setup.socks5ProxyPort}`); + expect( + await testResolveProxy( + { + proxy: `pac+http://127.0.0.1:${setup.httpServerPort}/pac`, + }, + 'http://pac-invalidproxy/test' + ) + ).to.equal(`SOCKS5 127.0.0.1:1`); + }); + + it('correctly handles environment-specified proxies', async function () { + const config: DevtoolsProxyOptions = { + env: { + HTTP_PROXY: 'https://http-proxy.example.net', + HTTPS_PROXY: 'http://https-proxy.example.net', + NO_PROXY: 'example.net:1234,example.com', + }, + noProxyHosts: 'example.net:4567', + useEnvironmentVariableProxies: true, + }; + expect(await testResolveProxy(config, 'http://localhost')).to.equal( + 'DIRECT' + ); + expect(await testResolveProxy(config, 'http://example.net')).to.equal( + 'HTTPS http-proxy.example.net:443' + ); + expect(await testResolveProxy(config, 'https://example.net')).to.equal( + 'PROXY https-proxy.example.net:80' + ); + expect( + await testResolveProxy(config, 'https://example.net:1234') + ).to.equal('DIRECT'); + expect( + await testResolveProxy(config, 'https://example.net:4567') + ).to.equal('DIRECT'); + expect( + await testResolveProxy(config, 'https://example.net:9801') + ).to.equal('PROXY https-proxy.example.net:80'); + expect(await testResolveProxy(config, 'https://example.com')).to.equal( + 'DIRECT' + ); + }); + }); + }); + + describe('secrets management', function () { + it('can extract and re-merge secrets for proxy options', function () { + const options: DevtoolsProxyOptions = { + proxy: 'ssh://username:password@host.example.net/', + sshOptions: { + identityKeyFile: '/file.key', + identityKeyPassphrase: 'secret', + }, + }; + expect(extractProxySecrets(options)).to.deep.equal({ + proxyOptions: { + proxy: 'ssh://username@host.example.net/', + sshOptions: { + identityKeyFile: '/file.key', + identityKeyPassphrase: undefined, + }, + }, + secrets: '{"password":"password","sshIdentityKeyPassphrase":"secret"}', + }); + expect(mergeProxySecrets(extractProxySecrets(options))).to.deep.equal( + options + ); + }); + + it('can redact URL passwords', function () { + expect(redactUrl('ssh://username:password@host.example.net/')).to.equal( + 'ssh://username:(credential)@host.example.net/' + ); + }); + }); +}); diff --git a/packages/devtools-proxy-support/src/proxy-options.ts b/packages/devtools-proxy-support/src/proxy-options.ts index 5f300a6c..3a2fd762 100644 --- a/packages/devtools-proxy-support/src/proxy-options.ts +++ b/packages/devtools-proxy-support/src/proxy-options.ts @@ -3,7 +3,7 @@ import type { ConnectionOptions } from 'tls'; // Should be an opaque type, but TS does not support those. export type DevtoolsProxyOptionsSecrets = string; -export type DevtoolsProxyOptions = { +export interface DevtoolsProxyOptions { // Can be an ssh://, socks5://, http://, https:// or pac<...>:// URL // Everything besides ssh:// gets forwarded to the `proxy-agent` npm package proxy?: string; @@ -16,10 +16,15 @@ export type DevtoolsProxyOptions = { identityKeyFile?: string; identityKeyPassphrase?: string; }; -} & Pick< - ConnectionOptions, - 'ca' | 'cert' | 'crl' | 'key' | 'passphrase' | 'pfx' ->; + + // Not being honored by the translate-to-electron functionality + ca?: ConnectionOptions['ca']; + + // Mostly intended for testing, defaults to `process.env`. + // This should usually not be stored, and secrets will not be + // redacted from this option. + env?: Record; +} // https://www.electronjs.org/docs/latest/api/structures/proxy-config interface ElectronProxyConfig { @@ -29,9 +34,10 @@ interface ElectronProxyConfig { proxyBypassRules?: string; } -function proxyConfForEnvVars( - env: Record = process.env -): { map: Map; noProxy: string } { +export function proxyConfForEnvVars(env: Record): { + map: Map; + noProxy: string; +} { const map = new Map(); let noProxy = ''; for (const [_key, value] of Object.entries(env)) { @@ -80,7 +86,9 @@ export function proxyForUrl( } if (proxyOptions.useEnvironmentVariableProxies) { - const { map, noProxy } = proxyConfForEnvVars(); + const { map, noProxy } = proxyConfForEnvVars( + proxyOptions.env ?? process.env + ); return (target: string) => { const url = new URL(target); const protocol = url.protocol.replace(/:$/, ''); @@ -98,60 +106,92 @@ export function proxyForUrl( return () => ''; } +function validateElectronProxyURL(url: URL | string): string { + url = new URL(url.toString()); + if (url.protocol === 'ssh:') { + throw new Error( + `Using ssh:// proxies for generic browser proxy usage is not supported (translating '${redactUrl( + url + )}')` + ); + } + if (url.username || url.password) { + throw new Error( + `Using authenticated proxies for generic browser proxy usage is not supported (translating '${redactUrl( + url + )}')` + ); + } + if ( + url.protocol !== 'http:' && + url.protocol !== 'https:' && + url.protocol !== 'socks5:' + ) { + throw new Error( + `Unsupported proxy protocol (translating '${redactUrl(url)}')` + ); + } + if (url.search || url.hash) { + throw new Error( + `Unsupported URL extensions in proxy specification (translating '${redactUrl( + url + )}')` + ); + } + if (url.pathname === '') return url.toString(); + if (url.pathname !== '/') { + throw new Error( + `Unsupported URL pathname in proxy specification (translating '${redactUrl( + url + )}')` + ); + } + return url.toString().replace(/\/$/, ''); +} + export function translateToElectronProxyConfig( proxyOptions: DevtoolsProxyOptions ): ElectronProxyConfig { if (proxyOptions.proxy) { - const url = new URL(proxyOptions.proxy); - if (url.protocol === 'ssh:') { - throw new Error( - `Using ssh:// proxies for generic browser proxy usage is not supported (translating '${redactUrl( - url - )}')` - ); - } - if (url.username || url.password) { - throw new Error( - `Using authenticated proxies for generic browser proxy usage is not supported (translating '${redactUrl( - url - )}')` - ); - } - if (url.protocol.startsWith('pac+')) { - url.protocol = url.protocol.replace('pac+', ''); + let url = proxyOptions.proxy; + if (new URL(url).protocol.startsWith('pac+')) { + url = url.replace('pac+', ''); return { mode: 'pac_script', pacScript: url.toString(), proxyBypassRules: proxyOptions.noProxyHosts, }; } - if ( - url.protocol !== 'http:' && - url.protocol !== 'https:' && - url.protocol !== 'socks5:' - ) { - throw new Error( - `Unsupported proxy protocol (translating '${redactUrl(url)}')` - ); - } return { mode: 'fixed_servers', - proxyRules: url.toString(), + proxyRules: validateElectronProxyURL(url), proxyBypassRules: proxyOptions.noProxyHosts, }; } if (proxyOptions.useEnvironmentVariableProxies) { - const proxyRules: string[] = []; - const proxyBypassRules = [proxyOptions.noProxyHosts]; - const { map, noProxy } = proxyConfForEnvVars(); - for (const [key, value] of map) proxyBypassRules.push(`${key}=${value}`); - proxyBypassRules.push(noProxy); + const proxyRulesList: string[] = []; + const proxyBypassRulesList = [proxyOptions.noProxyHosts]; + const { map, noProxy } = proxyConfForEnvVars( + proxyOptions.env ?? process.env + ); + for (const [key, value] of map) + proxyRulesList.push(`${key}=${validateElectronProxyURL(value)}`); + proxyBypassRulesList.push(noProxy); + + const proxyRules = proxyRulesList.join(';'); + const proxyBypassRules = + proxyBypassRulesList.filter(Boolean).join(',') || undefined; + + if (!proxyRules) { + if (!proxyBypassRules) return {}; + else return { proxyBypassRules }; + } return { mode: 'fixed_servers', - proxyBypassRules: proxyBypassRules.filter(Boolean).join(',') || undefined, - proxyRules: proxyRules.join(';'), + proxyBypassRules, + proxyRules, }; } @@ -164,7 +204,9 @@ interface DevtoolsProxyOptionsSecretsInternal { sshIdentityKeyPassphrase?: string; } -// These mirror our secrets extraction/merging logic in Compass +// These mirror our secrets extraction/merging logic in Compass. +// They do *not* extract secrets from env vars, since the `.env` +// property is generally intended for testing/not for storage. export function extractProxySecrets( proxyOptions: Readonly ): { @@ -174,8 +216,8 @@ export function extractProxySecrets( const secrets: DevtoolsProxyOptionsSecretsInternal = {}; if (proxyOptions.proxy) { const proxyUrl = new URL(proxyOptions.proxy); - ({ username: secrets.username, password: secrets.password } = proxyUrl); - proxyUrl.username = proxyUrl.password = ''; + secrets.password = proxyUrl.password; + proxyUrl.password = ''; proxyOptions = { ...proxyOptions, proxy: proxyUrl.toString() }; } if (proxyOptions.sshOptions) { @@ -209,7 +251,6 @@ export function mergeProxySecrets({ proxyOptions.proxy ) { const proxyUrl = new URL(proxyOptions.proxy); - proxyUrl.username = parsedSecrets.username || ''; proxyUrl.password = parsedSecrets.password || ''; proxyOptions = { ...proxyOptions, proxy: proxyUrl.toString() }; } @@ -225,8 +266,8 @@ export function mergeProxySecrets({ return proxyOptions; } -function redactUrl(urlOrString: URL | string): string { +export function redactUrl(urlOrString: URL | string): string { const url = new URL(urlOrString.toString()); - url.username = url.password = '(credential)'; + if (url.password) url.password = '(credential)'; return url.toString(); } diff --git a/packages/devtools-proxy-support/test/electron-test-server.js b/packages/devtools-proxy-support/test/electron-test-server.js new file mode 100644 index 00000000..d9303049 --- /dev/null +++ b/packages/devtools-proxy-support/test/electron-test-server.js @@ -0,0 +1,27 @@ +'use strict'; +const { app } = require('electron'); +const { once } = require('events'); +const { connect } = require('net'); +(async function () { + try { + await app.whenReady(); + const socket = connect(+process.env.TEST_SERVER_PORT); + await once(socket, 'connect'); + socket.setEncoding('utf8'); + let buffer = ''; + for await (const chunk of socket) { + buffer += chunk; + while (buffer.includes('\0')) { + const readyToExecute = buffer.substring(0, buffer.indexOf('\0')); + buffer = buffer.substring(buffer.indexOf('\0') + 1); + const result = JSON.stringify(await eval(JSON.parse(readyToExecute))); + socket.write(result + '\0'); + } + } + process.exit(0); + } catch (err) { + // eslint-disable-next-line no-console + console.error(err); + process.exit(1); + } +})(); diff --git a/packages/devtools-proxy-support/test/helpers.ts b/packages/devtools-proxy-support/test/helpers.ts index 02f49b67..1995cce7 100644 --- a/packages/devtools-proxy-support/test/helpers.ts +++ b/packages/devtools-proxy-support/test/helpers.ts @@ -74,7 +74,11 @@ export class HTTPServerProxyTestSetup { const handler: RequestListener = (req, res) => { this.requests.push(req); res.writeHead(200); - res.end(`OK ${req.url ?? ''}`); + if (req.url === '/pac') { + res.end(this.pacFile()); + } else { + res.end(`OK ${req.url ?? ''}`); + } }; this.httpServer = createHTTPServer(handler); this.httpsServer = createHTTPSServer({ ...this.tlsOptions }, handler); @@ -204,4 +208,13 @@ export class HTTPServerProxyTestSetup { this.sshServer.close(); // Doesn't emit 'close' await Promise.all(closePromises); } + + pacFile() { + return `function FindProxyForURL(url, host) { + if (host === 'pac-invalidproxy') { + return 'SOCKS5 127.0.0.1:1'; + } + return 'SOCKS5 127.0.0.1:${this.socks5ProxyPort}'; + }`; + } } From 8e29562baad32d3c2f243147886d47ae3c124acb Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 23 Jul 2024 16:59:08 +0200 Subject: [PATCH 10/30] WIP --- .../devtools-proxy-support/src/agent.spec.ts | 2 +- .../src/proxy-options.ts | 1 + .../devtools-proxy-support/src/socks5.spec.ts | 100 +++++++++++++++ packages/devtools-proxy-support/src/socks5.ts | 73 +++++++---- .../devtools-proxy-support/src/ssh.spec.ts | 115 ++++++++++++++++++ packages/devtools-proxy-support/src/ssh.ts | 9 +- .../devtools-proxy-support/test/helpers.ts | 39 +++--- 7 files changed, 297 insertions(+), 42 deletions(-) create mode 100644 packages/devtools-proxy-support/src/socks5.spec.ts create mode 100644 packages/devtools-proxy-support/src/ssh.spec.ts diff --git a/packages/devtools-proxy-support/src/agent.spec.ts b/packages/devtools-proxy-support/src/agent.spec.ts index bae5635f..fe1a33ed 100644 --- a/packages/devtools-proxy-support/src/agent.spec.ts +++ b/packages/devtools-proxy-support/src/agent.spec.ts @@ -223,7 +223,7 @@ describe('createAgent', function () { it('fails to connect to an ssh proxy with unavailable tunneling', async function () { setup.authHandler = sinon.stub().returns(true); - setup.canTunnel = false; + setup.canTunnel = sinon.stub().returns(false); try { await get( diff --git a/packages/devtools-proxy-support/src/proxy-options.ts b/packages/devtools-proxy-support/src/proxy-options.ts index 3a2fd762..f6c84e2c 100644 --- a/packages/devtools-proxy-support/src/proxy-options.ts +++ b/packages/devtools-proxy-support/src/proxy-options.ts @@ -18,6 +18,7 @@ export interface DevtoolsProxyOptions { }; // Not being honored by the translate-to-electron functionality + // TODO(COMPASS-8077): Integrate system CA here ca?: ConnectionOptions['ca']; // Mostly intended for testing, defaults to `process.env`. diff --git a/packages/devtools-proxy-support/src/socks5.spec.ts b/packages/devtools-proxy-support/src/socks5.spec.ts new file mode 100644 index 00000000..ba9dd32d --- /dev/null +++ b/packages/devtools-proxy-support/src/socks5.spec.ts @@ -0,0 +1,100 @@ +import sinon from 'sinon'; +import { HTTPServerProxyTestSetup } from '../test/helpers'; +import type { Tunnel } from './socks5'; +import { setupSocks5Tunnel } from './socks5'; +import { expect } from 'chai'; +import { createFetch } from './fetch'; + +describe('setupSocks5Tunnel', function () { + let setup: HTTPServerProxyTestSetup; + let tunnel: Tunnel | undefined; + + beforeEach(async function () { + setup = new HTTPServerProxyTestSetup(); + await setup.listen(); + tunnel = undefined; + }); + + afterEach(async function () { + await setup.teardown(); + await tunnel?.close(); + }); + + it('can be used to create a Socks5 server that forwards requests to another proxy', async function () { + setup.authHandler = sinon.stub().returns(true); + + tunnel = await setupSocks5Tunnel( + { + proxy: `http://foo:bar@127.0.0.1:${setup.httpProxyPort}`, + }, + { + proxyUsername: 'baz', + proxyPassword: 'quux', + } + ); + if (!tunnel) { + // regular conditional instead of assertion so that TS can follow it + expect.fail('failed to create Socks5 tunnel'); + } + + const fetch = createFetch({ + proxy: `socks5://baz:quux@127.0.0.1:${tunnel.config.proxyPort}`, + }); + const response = await fetch('http://example.com/hello'); + expect(await response.text()).to.equal('OK /hello'); + + try { + await fetch('http://localhost:1/hello'); + expect.fail('missed exception'); + } catch (err) { + expect(err.message).to.include( + 'request to http://localhost:1/hello failed' + ); + } + }); + + it('rejects mismatching auth', async function () { + tunnel = await setupSocks5Tunnel( + { + useEnvironmentVariableProxies: true, + env: {}, + }, + { + proxyUsername: 'baz', + proxyPassword: 'quux', + } + ); + if (!tunnel) { + // regular conditional instead of assertion so that TS can follow it + expect.fail('failed to create Socks5 tunnel'); + } + + const fetch = createFetch({ + proxy: `socks5://baz:wrongpassword@127.0.0.1:${tunnel.config.proxyPort}`, + }); + + try { + await fetch('http://localhost:1234/hello'); + expect.fail('missed exception'); + } catch (err) { + expect(err.message).to.include('Socks5 Authentication failed'); + } + }); + + it('reports an error when it fails to listen', async function () { + try { + await setupSocks5Tunnel( + { + useEnvironmentVariableProxies: true, + env: {}, + }, + { + proxyHost: 'example.net', + } + ); + expect.fail('missed exception'); + } catch (err) { + expect(err.code).to.equal('EADDRNOTAVAIL'); + } + }); +}); diff --git a/packages/devtools-proxy-support/src/socks5.ts b/packages/devtools-proxy-support/src/socks5.ts index 7c97879c..07401d9e 100644 --- a/packages/devtools-proxy-support/src/socks5.ts +++ b/packages/devtools-proxy-support/src/socks5.ts @@ -46,17 +46,19 @@ export interface Tunnel { } function createFakeHttpClientRequest(dstAddr: string, dstPort: number) { - return { - host: dstAddr, + const headers: Record = { + host: `${isIPv6(dstAddr) ? `[${dstAddr}]` : dstAddr}:${dstPort}`, + upgrade: 'websocket', // hack to make proxy-agent prefer CONNECT over HTTP proxying + }; + return Object.assign(new EventEmitter() as ClientRequest, { + host: headers.host, protocol: 'http', method: 'GET', path: '/', - getHeader(name) { - return name === 'host' - ? `${isIPv6(dstAddr) ? `[${dstAddr}]` : dstAddr}:${dstPort}` - : undefined; + getHeader(name: string) { + return headers[name]; }, - } as ClientRequest; + }); } class Socks5Server extends EventEmitter implements Tunnel { @@ -126,14 +128,14 @@ class Socks5Server extends EventEmitter implements Tunnel { const listeningPromise = this.serverListen(proxyPort, proxyHost); try { - await Promise.all([listeningPromise, this.ensureAgentInitialized()]); + await Promise.all([ + listeningPromise, + once(this, 'listening'), + this.ensureAgentInitialized(), + ]); this.agentInitialized = true; } catch (err: unknown) { - try { - await listeningPromise; - } finally { - await this.close(); - } + await this.close(); throw err; } } @@ -192,19 +194,36 @@ class Socks5Server extends EventEmitter implements Tunnel { this.closeOpenConnections(), ]); - if (maybeError) { + if ( + maybeError && + !('code' in maybeError && maybeError.code === 'ERR_SERVER_NOT_RUNNING') + ) { throw maybeError; } } private async forwardOut(dstAddr: string, dstPort: number): Promise { - const channel = await promisify(this.agent.createSocket.bind(this.agent))( - createFakeHttpClientRequest(dstAddr, dstPort), - { - host: dstAddr, - port: dstPort, - } - ); + const channel = await new Promise((resolve, reject) => { + const req = createFakeHttpClientRequest(dstAddr, dstPort); + req.onSocket = (sock) => { + if (sock) resolve(sock); + }; + this.agent.createSocket( + req, + { + host: dstAddr, + port: dstPort, + }, + (err, sock) => { + // Ideally, we would always be using this callback for retrieving the `sock` + // instance. However, agent-base does not call the callback at all if + // the agent resolved to another agent (as is the case for e.g. `ProxyAgent`). + if (err) reject(err); + else if (sock) resolve(sock); + } + ); + }); + if (!channel) throw new Error(`Could not create channel to ${dstAddr}:${dstPort}`); return channel; @@ -231,15 +250,19 @@ class Socks5Server extends EventEmitter implements Tunnel { socket = accept(true); this.connections.add(socket); - - socket.on('error', (err: ErrorWithOrigin) => { + const forwardingErrorHandler = (err: ErrorWithOrigin) => { + if (!socket?.writableEnded) socket?.end(); + if (!channel?.writableEnded) channel?.end(); err.origin ??= 'connection'; this.logger.emit('socks5:forwarding-error', { ...logMetadata, error: String((err as Error).stack), }); this.emit('forwardingError', err); - }); + }; + + channel.on('error', forwardingErrorHandler); + socket.on('error', forwardingErrorHandler); socket.once('close', () => { this.logger.emit('socks5:forwarded-socket-closed', { ...logMetadata }); @@ -265,7 +288,7 @@ class Socks5Server extends EventEmitter implements Tunnel { export async function setupSocks5Tunnel( proxyOptions: DevtoolsProxyOptions | AgentWithInitialize, tunnelOptions?: Partial, - target = 'mongodb://' + target?: string | undefined ): Promise { const agent = useOrCreateAgent(proxyOptions, target); if (!agent) return undefined; diff --git a/packages/devtools-proxy-support/src/ssh.spec.ts b/packages/devtools-proxy-support/src/ssh.spec.ts new file mode 100644 index 00000000..8cb93c79 --- /dev/null +++ b/packages/devtools-proxy-support/src/ssh.spec.ts @@ -0,0 +1,115 @@ +import { HTTPServerProxyTestSetup } from '../test/helpers'; +import { SSHAgent } from './ssh'; +import { createFetch } from './fetch'; +import { expect } from 'chai'; +import sinon from 'sinon'; + +describe('SSHAgent', function () { + let setup: HTTPServerProxyTestSetup; + let agent: SSHAgent | undefined; + + beforeEach(async function () { + setup = new HTTPServerProxyTestSetup(); + await setup.listen(); + agent = undefined; + }); + + afterEach(async function () { + await setup.teardown(); + agent?.destroy(); + }); + + it('allows establishing connections through an SSH server', async function () { + agent = new SSHAgent({ + proxy: `ssh://someuser@127.0.0.1:${setup.sshProxyPort}/`, + }); + const fetch = createFetch(agent); + const response = await fetch('http://example.com/hello'); + expect(await response.text()).to.equal('OK /hello'); + }); + + it('re-uses a single SSH connection if it can', async function () { + setup.authHandler = sinon.stub().returns(true); + agent = new SSHAgent({ + proxy: `ssh://foo:bar@127.0.0.1:${setup.sshProxyPort}/`, + }); + const fetch = createFetch(agent); + await Promise.all([ + fetch('http://example.com/hello'), + fetch('http://example.com/hello'), + ]); + expect(setup.authHandler).to.have.been.calledOnceWith('foo', 'bar'); + }); + + it('allows explicitly initializing the connection', async function () { + setup.authHandler = sinon.stub().returns(true); + agent = new SSHAgent({ + proxy: `ssh://foo:bar@127.0.0.1:${setup.sshProxyPort}/`, + }); + await agent.initialize(); + await createFetch(agent)('http://example.com/hello'); + expect(setup.authHandler).to.have.been.calledOnceWith('foo', 'bar'); + }); + + it('automatically reconnects if a connection was broken', async function () { + setup.authHandler = sinon.stub().returns(true); + agent = new SSHAgent({ + proxy: `ssh://foo:bar@127.0.0.1:${setup.sshProxyPort}/`, + }); + await agent.initialize(); + const fetch = createFetch(agent); + await fetch('http://example.com/hello'); + await agent.interruptForTesting(); + await fetch('http://example.com/hello'); + expect(setup.authHandler).to.have.been.calledTwice; + }); + + it('does not reconnect if the agent was intentionally closed', async function () { + setup.authHandler = sinon.stub().returns(true); + agent = new SSHAgent({ + proxy: `ssh://foo:bar@127.0.0.1:${setup.sshProxyPort}/`, + }); + await agent.initialize(); + const fetch = createFetch(agent); + await fetch('http://example.com/hello'); + agent.destroy(); + try { + await fetch('http://example.com/hello'); + expect.fail('missed exception'); + } catch (err) { + expect(err.message).to.include( + 'request to http://example.com/hello failed, reason: Disconnected' + ); + } + expect(setup.authHandler).to.have.been.calledOnce; + }); + + it('automatically retries the forwarding operation once (connection lost)', async function () { + setup.authHandler = sinon.stub().returns(true); + agent = new SSHAgent({ + proxy: `ssh://foo:bar@127.0.0.1:${setup.sshProxyPort}/`, + }); + await agent.initialize(); + await agent.interruptForTesting(); + const fetch = createFetch(agent); + await fetch('http://example.com/hello'); + expect(setup.authHandler).to.have.been.calledTwice; + }); + + it('automatically retries the forwarding operation once (tunnel failure)', async function () { + setup.authHandler = sinon.stub().returns(true); + setup.canTunnel = sinon + .stub() + .onFirstCall() + .returns(false) + .onSecondCall() + .returns(true); + agent = new SSHAgent({ + proxy: `ssh://foo:bar@127.0.0.1:${setup.sshProxyPort}/`, + }); + const fetch = createFetch(agent); + await fetch('http://example.com/hello'); + expect(setup.authHandler).to.have.been.calledTwice; + expect(setup.canTunnel).to.have.been.calledTwice; + }); +}); diff --git a/packages/devtools-proxy-support/src/ssh.ts b/packages/devtools-proxy-support/src/ssh.ts index 5696eacc..c290a51f 100644 --- a/packages/devtools-proxy-support/src/ssh.ts +++ b/packages/devtools-proxy-support/src/ssh.ts @@ -143,7 +143,9 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { } return sock; } catch (err: unknown) { - const retryableError = (err as Error).message === 'Not connected'; + const retryableError = /Not connected|Channel open failure/.test( + (err as Error).message + ); this.logger.emit('ssh:failed-forward', { host, error: String((err as Error).stack), @@ -165,4 +167,9 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { this.closed = true; this.sshClient.end(); } + + async interruptForTesting(): Promise { + this.sshClient.end(); + await once(this.sshClient, 'close'); + } } diff --git a/packages/devtools-proxy-support/test/helpers.ts b/packages/devtools-proxy-support/test/helpers.ts index 1995cce7..9302fff7 100644 --- a/packages/devtools-proxy-support/test/helpers.ts +++ b/packages/devtools-proxy-support/test/helpers.ts @@ -18,6 +18,7 @@ import { promisify } from 'util'; import socks5Server from 'socksv5/lib/server'; import socks5AuthNone from 'socksv5/lib/auth/None'; import socks5AuthUserPassword from 'socksv5/lib/auth/UserPassword'; +import type { Duplex } from 'stream'; function parseHTTPAuthHeader(header: string | undefined): [string, string] { if (!header?.startsWith('Basic ')) return ['', '']; @@ -37,7 +38,7 @@ export class HTTPServerProxyTestSetup { readonly httpsProxyServer: HTTPServer; readonly sshServer: SSHServer; readonly sshTunnelInfos: TcpipRequestInfo[] = []; - canTunnel = true; + canTunnel: () => boolean = () => true; authHandler: undefined | ((username: string, password: string) => boolean); get httpServerPort(): number { @@ -95,6 +96,25 @@ export class HTTPServerProxyTestSetup { } ); + const onconnect = + (server: HTTPServer) => + (req: IncomingMessage, socket: Duplex, head: Buffer) => { + const [username, pw] = parseHTTPAuthHeader( + req.headers['proxy-authorization'] + ); + if (this.authHandler?.(username, pw) === false) { + socket.end('HTTP/1.0 407 Proxy Authentication Required\r\n\r\n'); + return; + } + if (req.url === '127.0.0.1:1') { + socket.end('HTTP/1.0 502 Bad Gateway\r\n\r\n'); + return; + } + socket.unshift(head); + server.emit('connection', socket); + socket.write('HTTP/1.0 200 OK\r\n\r\n'); + }; + this.httpProxyServer = createHTTPServer((req, res) => { const [username, pw] = parseHTTPAuthHeader( req.headers['proxy-authorization'] @@ -115,22 +135,11 @@ export class HTTPServerProxyTestSetup { }, (proxyRes) => proxyRes.pipe(res) ); - }); + }).on('connect', onconnect(this.httpServer)); this.httpsProxyServer = createHTTPServer(() => { throw new Error('should not use normal req/res handler'); - }).on('connect', (req, socket, head) => { - const [username, pw] = parseHTTPAuthHeader( - req.headers['proxy-authorization'] - ); - if (this.authHandler?.(username, pw) === false) { - socket.end('HTTP/1.0 407 Proxy Authentication Required\r\n\r\n'); - return; - } - socket.unshift(head); - this.httpsServer.emit('connection', socket); - socket.write('HTTP/1.0 200 OK\r\n\r\n'); - }); + }).on('connect', onconnect(this.httpsServer)); this.sshServer = new SSHServer( { @@ -150,7 +159,7 @@ export class HTTPServerProxyTestSetup { }) .on('ready', () => { client.on('tcpip', (accept, reject, info) => { - if (!this.canTunnel) { + if (!this.canTunnel()) { return reject(); } this.sshTunnelInfos.push(info); From e8e6d5e2cfc7b59ca267c8bda1ae3309413147ba Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 23 Jul 2024 17:42:56 +0200 Subject: [PATCH 11/30] WIP --- .depalignrc.json | 8 ++++++++ packages/devtools-proxy-support/.depcheckrc | 1 + 2 files changed, 9 insertions(+) create mode 100644 .depalignrc.json diff --git a/.depalignrc.json b/.depalignrc.json new file mode 100644 index 00000000..2e4a6b0e --- /dev/null +++ b/.depalignrc.json @@ -0,0 +1,8 @@ +{ + "ignore": { + "node-fetch": [ + "^3.3.2", + "^2.6.11" + ] + } +} diff --git a/packages/devtools-proxy-support/.depcheckrc b/packages/devtools-proxy-support/.depcheckrc index 48bf9af6..7719f626 100644 --- a/packages/devtools-proxy-support/.depcheckrc +++ b/packages/devtools-proxy-support/.depcheckrc @@ -4,5 +4,6 @@ ignores: - '@types/chai' - '@types/sinon-chai' - 'sinon' + - 'xvfb-maybe' ignore-patterns: - 'dist' From 841776f949967de633e7d33dd982d000be255025 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 23 Jul 2024 17:57:22 +0200 Subject: [PATCH 12/30] WIP --- .../src/proxy-options.spec.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/devtools-proxy-support/src/proxy-options.spec.ts b/packages/devtools-proxy-support/src/proxy-options.spec.ts index ceba47b1..bbdd2126 100644 --- a/packages/devtools-proxy-support/src/proxy-options.spec.ts +++ b/packages/devtools-proxy-support/src/proxy-options.spec.ts @@ -158,6 +158,9 @@ describe('proxy options handling', function () { let setup: HTTPServerProxyTestSetup; before(async function () { + if (process.platform === 'win32' && process.env.CI) { + return this.skip(); + } setup = new HTTPServerProxyTestSetup(); await setup.listen(); @@ -211,14 +214,14 @@ describe('proxy options handling', function () { }); after(async function () { - childProcess.kill(); - socket.destroy(); - server.close(); + childProcess?.kill?.(); + socket?.destroy?.(); + server?.close?.(); await Promise.all([ - once(socket, 'close'), - once(server, 'close'), + socket && once(socket, 'close'), + server && once(server, 'close'), exitPromise, - setup.teardown(), + setup?.teardown?.(), ]); }); From 3454f2f4bb0f2876b976bffa51f15edc77fd49d4 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 23 Jul 2024 18:35:17 +0200 Subject: [PATCH 13/30] WIP --- .../devtools-proxy-support/src/agent.spec.ts | 5 +++-- packages/devtools-proxy-support/src/agent.ts | 5 ++++- packages/devtools-proxy-support/src/fetch.ts | 2 ++ packages/devtools-proxy-support/src/logging.ts | 5 +++++ .../src/proxy-options.spec.ts | 13 +++++++++++-- .../devtools-proxy-support/src/proxy-options.ts | 17 +++++++++++++++++ packages/devtools-proxy-support/src/socks5.ts | 2 ++ packages/devtools-proxy-support/src/ssh.ts | 4 +++- .../test/electron-test-server.js | 14 ++++++++++++-- 9 files changed, 59 insertions(+), 8 deletions(-) diff --git a/packages/devtools-proxy-support/src/agent.spec.ts b/packages/devtools-proxy-support/src/agent.spec.ts index fe1a33ed..5bb2423a 100644 --- a/packages/devtools-proxy-support/src/agent.spec.ts +++ b/packages/devtools-proxy-support/src/agent.spec.ts @@ -14,7 +14,8 @@ describe('createAgent', function () { url: string, agent: Agent ): Promise => { - const getFn = new URL(url).protocol === 'https:' ? httpsGet : httpGet; + const nodeJSBuiltinGet = + new URL(url).protocol === 'https:' ? httpsGet : httpGet; const options = { agent, ca: setup.tlsOptions.ca, @@ -22,7 +23,7 @@ describe('createAgent', function () { }; agents.push(agent); const res = await new Promise((resolve, reject) => - getFn(url, options, resolve).once('error', reject) + nodeJSBuiltinGet(url, options, resolve).once('error', reject) ); let body = ''; res.setEncoding('utf8'); diff --git a/packages/devtools-proxy-support/src/agent.ts b/packages/devtools-proxy-support/src/agent.ts index a09af314..e3a6158e 100644 --- a/packages/devtools-proxy-support/src/agent.ts +++ b/packages/devtools-proxy-support/src/agent.ts @@ -10,6 +10,9 @@ import { SSHAgent } from './ssh'; import type { ProxyLogEmitter } from './logging'; import type { EventEmitter } from 'events'; +// Helper type that represents an https.Agent (= connection factory) +// with some custom properties that TS does not know about and/or +// that we add for our own purposes. export type AgentWithInitialize = Agent & { // This is genuinely custom for our usage (to allow establishing an SSH tunnel // first before starting to push connections through it) @@ -24,7 +27,7 @@ export type AgentWithInitialize = Agent & { cb: (err: Error | null, s?: Duplex) => void ): void; - // http.Agent is an EventEmitter + // http.Agent is an EventEmitter, just missing from @types/node } & Partial; export function createAgent( diff --git a/packages/devtools-proxy-support/src/fetch.ts b/packages/devtools-proxy-support/src/fetch.ts index 06da62c5..29dff5fb 100644 --- a/packages/devtools-proxy-support/src/fetch.ts +++ b/packages/devtools-proxy-support/src/fetch.ts @@ -6,6 +6,8 @@ import type { DevtoolsProxyOptions } from './proxy-options'; declare const __webpack_require__: unknown; +// The original version of this code was largely taken from +// https://github.com/mongodb-js/mongosh/blob/8e6962432397154941f593c847d8f774bfd49f1c/packages/import-node-fetch/src/index.ts async function importNodeFetch(): Promise { // Node-fetch is an ESM module from 3.x // Importing ESM modules to CommonJS is possible with a dynamic import. diff --git a/packages/devtools-proxy-support/src/logging.ts b/packages/devtools-proxy-support/src/logging.ts index 4974ca9c..dc7a7266 100644 --- a/packages/devtools-proxy-support/src/logging.ts +++ b/packages/devtools-proxy-support/src/logging.ts @@ -1,3 +1,8 @@ +// Most mongoLogId() calls here come from code that was +// previously part of the MongoDB Compass monorepo, hence the specific +// values used here; in particular, +// https://github.com/mongodb-js/compass/tree/55a5a608713d7316d158dc66febeb6b114d8b40d/packages/ssh-tunnel/src + interface BaseSocks5RequestMetadata { srcAddr: string; srcPort: number; diff --git a/packages/devtools-proxy-support/src/proxy-options.spec.ts b/packages/devtools-proxy-support/src/proxy-options.spec.ts index bbdd2126..1d9695e3 100644 --- a/packages/devtools-proxy-support/src/proxy-options.spec.ts +++ b/packages/devtools-proxy-support/src/proxy-options.spec.ts @@ -18,7 +18,7 @@ import { HTTPServerProxyTestSetup } from '../test/helpers'; describe('proxy options handling', function () { describe('proxyConfForEnvVars', function () { - it('should return a map of proxies and noProxy', function () { + it('should return a map listing configured proxies and no-proxy hosts', function () { const env = { HTTP_PROXY: 'http://proxy.example.com', HTTPS_PROXY: 'https://proxy.example.com', @@ -145,7 +145,12 @@ describe('proxy options handling', function () { ).to.deep.equal({}); }); }); + context('integration tests', function () { + // These are some integration tests that leverage the fact that + // Electron can be used as a script runner to verify that our generated + // Electron proxyh configuration actually behaves the way it is intended to. + let childProcess: ChildProcess; let exitPromise: Promise; let server: Server; @@ -164,6 +169,8 @@ describe('proxy options handling', function () { setup = new HTTPServerProxyTestSetup(); await setup.listen(); + // Use a TCP connection for communication with the electron process; + // see electron-test-server.js for details. server = createServer(); server.listen(0); await once(server, 'listening'); @@ -185,7 +192,7 @@ describe('proxy options handling', function () { } ); exitPromise = once(childProcess, 'exit').catch(() => { - /* ignore */ + // ignore unhandledRejection warning/error }); await once(childProcess, 'spawn'); @@ -205,6 +212,8 @@ describe('proxy options handling', function () { }; testResolveProxy = async (proxyOptions, url) => { + // https://www.electronjs.org/docs/latest/api/app#appsetproxyconfig + // https://www.electronjs.org/docs/latest/api/app#appresolveproxyurl return await runJS(`app.setProxy(${JSON.stringify( translateToElectronProxyConfig(proxyOptions) )}).then(_ => { diff --git a/packages/devtools-proxy-support/src/proxy-options.ts b/packages/devtools-proxy-support/src/proxy-options.ts index f6c84e2c..cc268eb0 100644 --- a/packages/devtools-proxy-support/src/proxy-options.ts +++ b/packages/devtools-proxy-support/src/proxy-options.ts @@ -35,6 +35,7 @@ interface ElectronProxyConfig { proxyBypassRules?: string; } +// Reads through all environment variables and gathers proxy settings from them export function proxyConfForEnvVars(env: Record): { map: Map; noProxy: string; @@ -52,6 +53,7 @@ export function proxyConfForEnvVars(env: Record): { return { map, noProxy }; } +// Whether a given URL should be proxied or not, according to `noProxy` function shouldProxy(noProxy: string, url: URL): boolean { if (!noProxy) return true; if (noProxy === '*') return false; @@ -72,6 +74,8 @@ function shouldProxy(noProxy: string, url: URL): boolean { return true; } +// Create a function which returns the proxy URL for a given target URL, +// based on the proxy config passed to it. export function proxyForUrl( proxyOptions: DevtoolsProxyOptions ): (url: string) => string { @@ -109,6 +113,17 @@ export function proxyForUrl( function validateElectronProxyURL(url: URL | string): string { url = new URL(url.toString()); + // ssh and authenticated proxies are not supported by Electron/Chromium currently. + // (See https://issues.chromium.org/issues/40829748, https://bugs.chromium.org/p/chromium/issues/detail?id=1309413 + // and related tickets for history.) + // If we do want to support them at some point, a possible way to achieve this would be + // to use a local in-application Socks5 proxy server which reaches out to the + // actual target proxy (or directly connects, depending on the configuration), + // and then passing the local proxy's information to Electron. + // A core downside with this approach, however, is that because the proxy cannot be + // authenticated, it would be available to any local application, which is problematic + // when running on multi-user machine where the proxy would then be available to + // arbitraty users. if (url.protocol === 'ssh:') { throw new Error( `Using ssh:// proxies for generic browser proxy usage is not supported (translating '${redactUrl( @@ -150,11 +165,13 @@ function validateElectronProxyURL(url: URL | string): string { return url.toString().replace(/\/$/, ''); } +// Convert our own `DevtoolsProxyOptions` to the format used by Electron. export function translateToElectronProxyConfig( proxyOptions: DevtoolsProxyOptions ): ElectronProxyConfig { if (proxyOptions.proxy) { let url = proxyOptions.proxy; + // https://en.wikipedia.org/wiki/Proxy_auto-config if (new URL(url).protocol.startsWith('pac+')) { url = url.replace('pac+', ''); return { diff --git a/packages/devtools-proxy-support/src/socks5.ts b/packages/devtools-proxy-support/src/socks5.ts index 07401d9e..a10d98a2 100644 --- a/packages/devtools-proxy-support/src/socks5.ts +++ b/packages/devtools-proxy-support/src/socks5.ts @@ -61,6 +61,8 @@ function createFakeHttpClientRequest(dstAddr: string, dstPort: number) { }); } +// The original version of this code was largely taken from +// https://github.com/mongodb-js/compass/tree/55a5a608713d7316d158dc66febeb6b114d8b40d/packages/ssh-tunnel/src class Socks5Server extends EventEmitter implements Tunnel { public logger: ProxyLogEmitter = new EventEmitter(); private readonly agent: AgentWithInitialize; diff --git a/packages/devtools-proxy-support/src/ssh.ts b/packages/devtools-proxy-support/src/ssh.ts index c290a51f..c698accb 100644 --- a/packages/devtools-proxy-support/src/ssh.ts +++ b/packages/devtools-proxy-support/src/ssh.ts @@ -13,6 +13,8 @@ import type { ProxyLogEmitter } from './logging'; import { connect as tlsConnect } from 'tls'; import type { Socket } from 'net'; +// The original version of this code was largely taken from +// https://github.com/mongodb-js/compass/tree/55a5a608713d7316d158dc66febeb6b114d8b40d/packages/ssh-tunnel/src export class SSHAgent extends AgentBase implements AgentWithInitialize { public logger: ProxyLogEmitter; private readonly proxyOptions: Readonly; @@ -31,7 +33,7 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { constructor(options: DevtoolsProxyOptions, logger?: ProxyLogEmitter) { super(); (this as AgentWithInitialize).on?.('error', () => { - //Errors should not crash the process + // Errors here should not crash the process }); this.logger = logger ?? new EventEmitter(); this.proxyOptions = options; diff --git a/packages/devtools-proxy-support/test/electron-test-server.js b/packages/devtools-proxy-support/test/electron-test-server.js index d9303049..dca409ec 100644 --- a/packages/devtools-proxy-support/test/electron-test-server.js +++ b/packages/devtools-proxy-support/test/electron-test-server.js @@ -2,6 +2,11 @@ const { app } = require('electron'); const { once } = require('events'); const { connect } = require('net'); + +// Essentially a REPL that we use for running Electron code. +// Communicates with a parent process via TCP and runs the code +// it receives over that socket. Input and output are NUL-delimited +// chunks of UTF-8 strings. (async function () { try { await app.whenReady(); @@ -14,8 +19,13 @@ const { connect } = require('net'); while (buffer.includes('\0')) { const readyToExecute = buffer.substring(0, buffer.indexOf('\0')); buffer = buffer.substring(buffer.indexOf('\0') + 1); - const result = JSON.stringify(await eval(JSON.parse(readyToExecute))); - socket.write(result + '\0'); + let result; + try { + result = await eval(JSON.parse(readyToExecute)); + } catch (err) { + result = { message: err.message, stack: err.stack, ...err }; + } + socket.write(JSON.stringify(result) + '\0'); } } process.exit(0); From a76bdc682cd6f87ab2ef8c8c2fe743ad200d3e5c Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 24 Jul 2024 14:35:07 +0200 Subject: [PATCH 14/30] WIP --- packages/devtools-proxy-support/src/agent.ts | 12 +++-- packages/devtools-proxy-support/src/index.ts | 2 +- .../devtools-proxy-support/src/logging.ts | 6 +++ .../src/proxy-options.ts | 22 ++++++++++ .../devtools-proxy-support/src/socks5.spec.ts | 44 ++++++++++++++++++- packages/devtools-proxy-support/src/socks5.ts | 25 +++++++++++ packages/devtools-proxy-support/src/ssh.ts | 2 +- 7 files changed, 106 insertions(+), 7 deletions(-) diff --git a/packages/devtools-proxy-support/src/agent.ts b/packages/devtools-proxy-support/src/agent.ts index e3a6158e..65cc9b34 100644 --- a/packages/devtools-proxy-support/src/agent.ts +++ b/packages/devtools-proxy-support/src/agent.ts @@ -18,6 +18,7 @@ export type AgentWithInitialize = Agent & { // first before starting to push connections through it) initialize?(): Promise; logger?: ProxyLogEmitter; + readonly proxyOptions?: Readonly; // This is just part of the regular Agent interface, used by Node.js itself, // but missing from @types/node @@ -40,10 +41,13 @@ export function createAgent( return new SSHAgent(proxyOptions); } const getProxyForUrl = proxyForUrl(proxyOptions); - return new ProxyAgent({ - getProxyForUrl, - ...proxyOptions, - }); + return Object.assign( + new ProxyAgent({ + getProxyForUrl, + ...proxyOptions, + }), + { proxyOptions } + ); } export function useOrCreateAgent( diff --git a/packages/devtools-proxy-support/src/index.ts b/packages/devtools-proxy-support/src/index.ts index 2f414686..1ec2233d 100644 --- a/packages/devtools-proxy-support/src/index.ts +++ b/packages/devtools-proxy-support/src/index.ts @@ -6,7 +6,7 @@ export { mergeProxySecrets, } from './proxy-options'; export { Tunnel, TunnelOptions, setupSocks5Tunnel } from './socks5'; -export { createAgent } from './agent'; +export { createAgent, useOrCreateAgent, AgentWithInitialize } from './agent'; export { createFetch, Request, diff --git a/packages/devtools-proxy-support/src/logging.ts b/packages/devtools-proxy-support/src/logging.ts index dc7a7266..22d7127c 100644 --- a/packages/devtools-proxy-support/src/logging.ts +++ b/packages/devtools-proxy-support/src/logging.ts @@ -74,12 +74,18 @@ interface MongoLogWriter { mongoLogId(this: void, id: number): unknown; } +let alreadyHooked: WeakMap>; let idCounter = 0; export function hookLogger( emitter: ProxyLogEmitter, logCtx: string, log: MongoLogWriter ): void { + // This helps avoid unintentionally attaching the same logging twice in devtools-connet + if (alreadyHooked.get(emitter)?.has(log)) return; + if (!alreadyHooked.has(emitter)) alreadyHooked.set(emitter, new WeakSet()); + alreadyHooked.get(emitter)!.add(log); + logCtx = `${logCtx}-${idCounter++}`; const { mongoLogId } = log; diff --git a/packages/devtools-proxy-support/src/proxy-options.ts b/packages/devtools-proxy-support/src/proxy-options.ts index cc268eb0..72c8683e 100644 --- a/packages/devtools-proxy-support/src/proxy-options.ts +++ b/packages/devtools-proxy-support/src/proxy-options.ts @@ -1,4 +1,5 @@ import type { ConnectionOptions } from 'tls'; +import type { TunnelOptions } from './socks5'; // Should be an opaque type, but TS does not support those. export type DevtoolsProxyOptionsSecrets = string; @@ -216,6 +217,27 @@ export function translateToElectronProxyConfig( return {}; } +// Return the Socks5 tunnel configuration, if proxyOptions always resolves to one. +// This is used by setupSocks5Tunnel() to avoid creating a local Socks5 tunnel +// that would just forward to another Socks5 tunnel anyway. +export function getSocks5OnlyProxyOptions( + proxyOptions: DevtoolsProxyOptions, + target?: string +): TunnelOptions | undefined { + let proxyUrl: string | undefined; + if (target !== undefined) proxyUrl = proxyForUrl(proxyOptions)(target); + else if (!proxyOptions.noProxyHosts) proxyUrl = proxyOptions.proxy; + if (!proxyUrl) return undefined; + const url = new URL(proxyUrl); + if (url.protocol !== 'socks5:') return undefined; + return { + proxyHost: decodeURIComponent(url.hostname), + proxyPort: +(url.port || 1080), + proxyUsername: decodeURIComponent(url.username) || undefined, + proxyPassword: decodeURIComponent(url.password) || undefined, + }; +} + interface DevtoolsProxyOptionsSecretsInternal { username?: string; password?: string; diff --git a/packages/devtools-proxy-support/src/socks5.spec.ts b/packages/devtools-proxy-support/src/socks5.spec.ts index ba9dd32d..0ed56b33 100644 --- a/packages/devtools-proxy-support/src/socks5.spec.ts +++ b/packages/devtools-proxy-support/src/socks5.spec.ts @@ -1,9 +1,10 @@ import sinon from 'sinon'; import { HTTPServerProxyTestSetup } from '../test/helpers'; -import type { Tunnel } from './socks5'; +import type { Tunnel, TunnelOptions } from './socks5'; import { setupSocks5Tunnel } from './socks5'; import { expect } from 'chai'; import { createFetch } from './fetch'; +import type { DevtoolsProxyOptions } from './proxy-options'; describe('setupSocks5Tunnel', function () { let setup: HTTPServerProxyTestSetup; @@ -97,4 +98,45 @@ describe('setupSocks5Tunnel', function () { expect(err.code).to.equal('EADDRNOTAVAIL'); } }); + + it('does not start an actual server if the proxy config already specifies socks5', async function () { + async function existingTunnelConfig( + options: DevtoolsProxyOptions, + target?: string + ): Promise { + const tunnel = await setupSocks5Tunnel(options, undefined, target); + expect(tunnel?.constructor.name).to.equal('ExistingTunnel'); + return JSON.parse(JSON.stringify(tunnel?.config)); // filter out undefined values + } + + expect( + await existingTunnelConfig({ proxy: 'socks5://example.com:123' }) + ).to.deep.equal({ proxyHost: 'example.com', proxyPort: 123 }); + expect( + await existingTunnelConfig({ proxy: 'socks5://example.com' }) + ).to.deep.equal({ proxyHost: 'example.com', proxyPort: 1080 }); + expect( + await existingTunnelConfig({ proxy: 'socks5://foo:bar@example.com' }) + ).to.deep.equal({ + proxyHost: 'example.com', + proxyPort: 1080, + proxyUsername: 'foo', + proxyPassword: 'bar', + }); + expect( + await existingTunnelConfig( + { proxy: 'socks5://example.com:123' }, + 'mongodb://' + ) + ).to.deep.equal({ proxyHost: 'example.com', proxyPort: 123 }); + expect( + await existingTunnelConfig( + { + useEnvironmentVariableProxies: true, + env: { MONGODB_PROXY: 'socks5://example.com:123' }, + }, + 'mongodb://' + ) + ).to.deep.equal({ proxyHost: 'example.com', proxyPort: 123 }); + }); }); diff --git a/packages/devtools-proxy-support/src/socks5.ts b/packages/devtools-proxy-support/src/socks5.ts index a10d98a2..60767209 100644 --- a/packages/devtools-proxy-support/src/socks5.ts +++ b/packages/devtools-proxy-support/src/socks5.ts @@ -1,4 +1,5 @@ import { EventEmitter, once } from 'events'; +import { getSocks5OnlyProxyOptions } from './proxy-options'; import type { DevtoolsProxyOptions } from './proxy-options'; import type { AgentWithInitialize } from './agent'; import { useOrCreateAgent } from './agent'; @@ -287,11 +288,35 @@ class Socks5Server extends EventEmitter implements Tunnel { } } +class ExistingTunnel extends EventEmitter { + logger = new EventEmitter(); + readonly config: TunnelOptions; + + constructor(config: TunnelOptions) { + super(); + this.config = config; + } + + async close() { + // nothing to do if we didn't start a server + } +} + export async function setupSocks5Tunnel( proxyOptions: DevtoolsProxyOptions | AgentWithInitialize, tunnelOptions?: Partial, target?: string | undefined ): Promise { + const socks5OnlyProxyOptions = getSocks5OnlyProxyOptions( + ('proxyOptions' in proxyOptions + ? proxyOptions.proxyOptions + : proxyOptions) as DevtoolsProxyOptions, + target + ); + if (socks5OnlyProxyOptions) { + return new ExistingTunnel(socks5OnlyProxyOptions); + } + const agent = useOrCreateAgent(proxyOptions, target); if (!agent) return undefined; diff --git a/packages/devtools-proxy-support/src/ssh.ts b/packages/devtools-proxy-support/src/ssh.ts index c698accb..74261b50 100644 --- a/packages/devtools-proxy-support/src/ssh.ts +++ b/packages/devtools-proxy-support/src/ssh.ts @@ -17,7 +17,7 @@ import type { Socket } from 'net'; // https://github.com/mongodb-js/compass/tree/55a5a608713d7316d158dc66febeb6b114d8b40d/packages/ssh-tunnel/src export class SSHAgent extends AgentBase implements AgentWithInitialize { public logger: ProxyLogEmitter; - private readonly proxyOptions: Readonly; + public readonly proxyOptions: Readonly; private readonly url: URL; private sshClient: SshClient; private connected = false; From 456a70c2eed56598cc55d3513269072372a7bccb Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 24 Jul 2024 17:03:47 +0200 Subject: [PATCH 15/30] WIP --- packages/devtools-connect/package.json | 1 + packages/devtools-connect/src/connect.ts | 280 ++++++++++++------ packages/devtools-connect/src/index.ts | 2 + packages/devtools-connect/src/log-hook.ts | 2 + packages/devtools-connect/src/types.ts | 5 +- packages/devtools-proxy-support/src/index.ts | 2 +- .../devtools-proxy-support/src/logging.ts | 4 +- .../src/proxy-options.ts | 2 +- .../devtools-proxy-support/src/socks5.spec.ts | 43 ++- packages/devtools-proxy-support/src/socks5.ts | 103 ++++--- packages/devtools-proxy-support/src/ssh.ts | 2 +- 11 files changed, 319 insertions(+), 127 deletions(-) diff --git a/packages/devtools-connect/package.json b/packages/devtools-connect/package.json index 0b0a4eb2..303d2782 100644 --- a/packages/devtools-connect/package.json +++ b/packages/devtools-connect/package.json @@ -48,6 +48,7 @@ "license": "Apache-2.0", "dependencies": { "@mongodb-js/oidc-http-server-pages": "1.1.2", + "@mongodb-js/devtools-proxy-support": "^0.1.0", "lodash.merge": "^4.6.2", "mongodb-connection-string-url": "^3.0.0", "socks": "^2.7.3", diff --git a/packages/devtools-connect/src/connect.ts b/packages/devtools-connect/src/connect.ts index ad3d100b..93a7edd1 100644 --- a/packages/devtools-connect/src/connect.ts +++ b/packages/devtools-connect/src/connect.ts @@ -8,14 +8,11 @@ import type { ServerHeartbeatSucceededEvent, TopologyDescription, } from 'mongodb'; -import type { - ConnectDnsResolutionDetail, - ConnectEventArgs, - ConnectEventMap, -} from './types'; +import type { ConnectDnsResolutionDetail } from './types'; import { systemCertsAsync } from 'system-ca'; import type { Options as SystemCAOptions } from 'system-ca'; import type { + HttpOptions as OIDCHTTPOptions, MongoDBOIDCPlugin, MongoDBOIDCPluginOptions, } from '@mongodb-js/oidc-plugin'; @@ -26,7 +23,15 @@ import { StateShareClient, StateShareServer } from './ipc-rpc-state-share'; import ConnectionString, { CommaAndColonSeparatedRecord, } from 'mongodb-connection-string-url'; -import EventEmitter from 'events'; +import { EventEmitter } from 'events'; +import { + createSocks5Tunnel, + DevtoolsProxyOptions, + AgentWithInitialize, + useOrCreateAgent, + Tunnel, +} from '@mongodb-js/devtools-proxy-support'; +export type { DevtoolsProxyOptions, AgentWithInitialize }; function isAtlas(str: string): boolean { try { @@ -267,6 +272,30 @@ function detectAndLogMissingOptionalDependencies(logger: ConnectLogEmitter) { } } +// Override 'from.emit' so that all events also end up being emitted on 'to' +function copyEventEmitterEvents( + from: { + emit: ( + event: K, + ...args: M[K] extends (...args: infer P) => any ? P : never + ) => void; + }, + to: { + emit: ( + event: K, + ...args: M[K] extends (...args: infer P) => any ? P : never + ) => void; + } +) { + from.emit = function ( + event: K, + ...args: M[K] extends (...args: infer P) => any ? P : never + ) { + to.emit(event, ...args); + return EventEmitter.prototype.emit.call(this, event, ...args); + }; +} + // Wrapper for all state that a devtools application may want to share // between MongoClient instances. Currently, this is only the OIDC state. // There are two ways of sharing this state: @@ -303,13 +332,7 @@ export class DevtoolsConnectionState { // (and not other OIDCPlugin instances that might be running on the same logger). const proxyingLogger = new EventEmitter(); proxyingLogger.setMaxListeners(Infinity); - proxyingLogger.emit = function ( - event: K, - ...args: ConnectEventArgs - ) { - logger.emit(event, ...args); - return EventEmitter.prototype.emit.call(this, event, ...args); - }; + copyEventEmitterEvents(proxyingLogger, logger); this.oidcPlugin = createMongoDBOIDCPlugin({ ...options.oidc, logger: proxyingLogger, @@ -318,7 +341,7 @@ export class DevtoolsConnectionState { options ), ...(systemCA - ? addCAToOIDCPluginHttpOptions(options.oidc, systemCA) + ? addToOIDCPluginHttpOptions(options.oidc, { ca: systemCA }) : {}), }); } @@ -370,6 +393,16 @@ export interface DevtoolsConnectOptions extends MongoClientOptions { * extends beyond the lifetime(s) of the respective dependent state instance(s). */ parentHandle?: string; + /** + * Proxy options or an existing proxy Agent instance that can be shared. These are applied to + * both database cluster traffic and, optionally, OIDC HTTP traffic. + */ + proxy?: DevtoolsProxyOptions | AgentWithInitialize; + /** + * Whether the proxy specified in `.proxy` should be applied to OIDC HTTP traffic as well. + * An explicitly specified `agent` in the options for the OIDC plugin will always take precedence. + */ + applyProxyToOIDC?: boolean; } /** @@ -386,82 +419,159 @@ export async function connectMongoClient( client: MongoClient; state: DevtoolsConnectionState; }> { + const cleanupOnClientClose: (() => void | Promise)[] = []; + const runClose = async () => { + let item: (() => void | Promise) | undefined; + while ((item = cleanupOnClientClose.shift())) await item(); + }; detectAndLogMissingOptionalDependencies(logger); - let systemCA: string | undefined; - if (clientOptions.useSystemCA) { - const systemCAOpts: SystemCAOptions = { includeNodeCertificates: true }; - const ca = await systemCertsAsync(systemCAOpts); - logger.emit('devtools-connect:used-system-ca', { - caCount: ca.length, - asyncFallbackError: systemCAOpts.asyncFallbackError, - }); - systemCA = ca.join('\n'); - } + try { + let systemCA: string | undefined; + // TODO(COMPASS-8077): Remove this option and enable it by default + if (clientOptions.useSystemCA) { + const systemCAOpts: SystemCAOptions = { includeNodeCertificates: true }; + const ca = await systemCertsAsync(systemCAOpts); + logger.emit('devtools-connect:used-system-ca', { + caCount: ca.length, + asyncFallbackError: systemCAOpts.asyncFallbackError, + }); + systemCA = ca.join('\n'); + } - // If PROVIDER_NAME was specified to the MongoClient options, adding callbacks would conflict - // with that; we should omit them so that e.g. mongosh users can leverage the non-human OIDC - // auth flows by specifying PROVIDER_NAME. - const shouldAddOidcCallbacks = isHumanOidcFlow(uri, clientOptions); - const state = - clientOptions.parentState ?? - new DevtoolsConnectionState(clientOptions, logger, systemCA); - const mongoClientOptions: MongoClientOptions & - Partial = merge( - {}, - clientOptions, - shouldAddOidcCallbacks ? state.oidcPlugin.mongoClientOptions : {}, - systemCA ? { ca: systemCA } : {} - ); + // Create a proxy agent, if requested. `useOrCreateAgent()` takes a target argument + // that can be used to select a proxy for a specific procotol or host; + // here we specify 'mongodb://' if we only intend to use the proxy for database + // connectivity. + const proxyAgent = + clientOptions.proxy && + useOrCreateAgent( + 'createConnection' in clientOptions.proxy + ? clientOptions.proxy + : { + ...(clientOptions.proxy as DevtoolsProxyOptions), + // TODO(COMPASS-8077): Always use explicit CA from either system CA or + // tlsCAFile option, including one potentially coming from the command line + ...(systemCA ? { ca: systemCA } : {}), + }, + clientOptions.applyProxyToOIDC ? undefined : 'mongodb://' + ); + cleanupOnClientClose.push(() => proxyAgent?.destroy()); - // Adopt dns result order changes with Node v18 that affected the VSCode extension VSCODE-458. - // Refs https://github.com/microsoft/vscode/issues/189805 - mongoClientOptions.lookup = (hostname, options, callback) => { - return dns.lookup(hostname, { verbatim: false, ...options }, callback); - }; + if (clientOptions.applyProxyToOIDC) { + clientOptions.oidc = { + ...clientOptions.oidc, + ...addToOIDCPluginHttpOptions(clientOptions.oidc, { + agent: proxyAgent, + }), + }; + } + + let tunnel: Tunnel | undefined; + if (proxyAgent && !hasProxyHostOption(uri, clientOptions)) { + tunnel = createSocks5Tunnel(proxyAgent, 'generate-credentials'); + cleanupOnClientClose.push(() => tunnel?.close()); + } + for (const proxyLogger of new Set([tunnel?.logger, proxyAgent?.logger])) { + if (proxyLogger) { + copyEventEmitterEvents(proxyLogger, logger); + } + } + if (tunnel) { + // Should happen after attaching loggers + await tunnel?.listen(); + clientOptions = { + ...clientOptions, + ...tunnel?.config, + }; + } - delete mongoClientOptions.useSystemCA; - delete mongoClientOptions.productDocsLink; - delete mongoClientOptions.productName; - delete mongoClientOptions.oidc; - delete mongoClientOptions.parentState; - delete mongoClientOptions.parentHandle; + // If PROVIDER_NAME was specified to the MongoClient options, adding callbacks would conflict + // with that; we should omit them so that e.g. mongosh users can leverage the non-human OIDC + // auth flows by specifying PROVIDER_NAME. + const shouldAddOidcCallbacks = isHumanOidcFlow(uri, clientOptions); + const state = + clientOptions.parentState ?? + new DevtoolsConnectionState(clientOptions, logger, systemCA); + const mongoClientOptions: MongoClientOptions & + Partial = merge( + {}, + clientOptions, + shouldAddOidcCallbacks ? state.oidcPlugin.mongoClientOptions : {}, + systemCA ? { ca: systemCA } : {} + ); + + // Adopt dns result order changes with Node v18 that affected the VSCode extension VSCODE-458. + // Refs https://github.com/microsoft/vscode/issues/189805 + mongoClientOptions.lookup = (hostname, options, callback) => { + return dns.lookup(hostname, { verbatim: false, ...options }, callback); + }; + + delete mongoClientOptions.useSystemCA; + delete mongoClientOptions.productDocsLink; + delete mongoClientOptions.productName; + delete mongoClientOptions.oidc; + delete mongoClientOptions.parentState; + delete mongoClientOptions.parentHandle; + delete mongoClientOptions.proxy; + delete mongoClientOptions.applyProxyToOIDC; - if ( - mongoClientOptions.autoEncryption !== undefined && - !mongoClientOptions.autoEncryption.bypassAutoEncryption && - !mongoClientOptions.autoEncryption.bypassQueryAnalysis - ) { - // connect first without autoEncryption and serverApi options. - const optionsWithoutFLE = { ...mongoClientOptions }; - delete optionsWithoutFLE.autoEncryption; - delete optionsWithoutFLE.serverApi; - const client = new MongoClientClass(uri, optionsWithoutFLE); - closeMongoClientWhenAuthFails(state, client); - await connectWithFailFast(uri, client, logger); - const buildInfo = await client - .db('admin') - .admin() - .command({ buildInfo: 1 }); - await client.close(); if ( - !buildInfo.modules?.includes('enterprise') && - !buildInfo.gitVersion?.match(/enterprise/) + mongoClientOptions.autoEncryption !== undefined && + !mongoClientOptions.autoEncryption.bypassAutoEncryption && + !mongoClientOptions.autoEncryption.bypassQueryAnalysis ) { - throw new MongoAutoencryptionUnavailable(); + // connect first without autoEncryption and serverApi options. + const optionsWithoutFLE = { ...mongoClientOptions }; + delete optionsWithoutFLE.autoEncryption; + delete optionsWithoutFLE.serverApi; + const client = new MongoClientClass(uri, optionsWithoutFLE); + closeMongoClientWhenAuthFails(state, client); + await connectWithFailFast(uri, client, logger); + const buildInfo = await client + .db('admin') + .admin() + .command({ buildInfo: 1 }); + await client.close(); + if ( + !buildInfo.modules?.includes('enterprise') && + !buildInfo.gitVersion?.match(/enterprise/) + ) { + throw new MongoAutoencryptionUnavailable(); + } + } + uri = await resolveMongodbSrv(uri, logger); + const client = new MongoClientClass(uri, mongoClientOptions); + client.once('close', runClose); + closeMongoClientWhenAuthFails(state, client); + await connectWithFailFast(uri, client, logger); + if ((client as any).autoEncrypter) { + // Enable Devtools-specific CSFLE result decoration. + (client as any).autoEncrypter[ + Symbol.for('@@mdb.decorateDecryptionResult') + ] = true; } + return { client, state }; + } catch (err: unknown) { + await runClose(); + throw err; } - uri = await resolveMongodbSrv(uri, logger); - const client = new MongoClientClass(uri, mongoClientOptions); - closeMongoClientWhenAuthFails(state, client); - await connectWithFailFast(uri, client, logger); - if ((client as any).autoEncrypter) { - // Enable Devtools-specific CSFLE result decoration. - (client as any).autoEncrypter[ - Symbol.for('@@mdb.decorateDecryptionResult') - ] = true; +} + +function hasProxyHostOption( + uri: string, + clientOptions: MongoClientOptions +): boolean { + if (clientOptions.proxyHost || clientOptions.proxyPort) return true; + let cs: ConnectionString; + try { + cs = new ConnectionString(uri, { looseValidation: true }); + } catch { + return false; } - return { client, state }; + + const sp = cs.typedSearchParams(); + return sp.has('proxyHost') || sp.has('proxyPort'); } export function isHumanOidcFlow( @@ -530,16 +640,20 @@ function closeMongoClientWhenAuthFails( ); } -function addCAToOIDCPluginHttpOptions( +function addToOIDCPluginHttpOptions( existingOIDCPluginOptions: MongoDBOIDCPluginOptions | undefined, - ca: string + addedOptions: Partial ): Pick { const existingCustomOptions = existingOIDCPluginOptions?.customHttpOptions; if (typeof existingCustomOptions === 'function') { return { customHttpOptions: (url, options, ...restArgs) => - existingCustomOptions(url, { ...options, ca }, ...restArgs), + existingCustomOptions( + url, + { ...options, ...addedOptions }, + ...restArgs + ), }; } - return { customHttpOptions: { ...existingCustomOptions, ca } }; + return { customHttpOptions: { ...existingCustomOptions, ...addedOptions } }; } diff --git a/packages/devtools-connect/src/index.ts b/packages/devtools-connect/src/index.ts index d298d0b5..489987e5 100644 --- a/packages/devtools-connect/src/index.ts +++ b/packages/devtools-connect/src/index.ts @@ -3,6 +3,8 @@ export { connectMongoClient } from './connect'; export type { DevtoolsConnectOptions, DevtoolsConnectionState, + DevtoolsProxyOptions, + AgentWithInitialize, } from './connect'; export { hookLogger } from './log-hook'; export { oidcServerRequestHandler } from './oidc/handler'; diff --git a/packages/devtools-connect/src/log-hook.ts b/packages/devtools-connect/src/log-hook.ts index ffbfdb24..2c35a974 100644 --- a/packages/devtools-connect/src/log-hook.ts +++ b/packages/devtools-connect/src/log-hook.ts @@ -11,6 +11,7 @@ import type { } from './types'; import { hookLoggerToMongoLogWriter as oidcHookLogger } from '@mongodb-js/oidc-plugin'; +import { hookLogger as proxyHookLogger } from '@mongodb-js/devtools-proxy-support'; interface MongoLogWriter { info(c: string, id: unknown, ctx: string, msg: string, attr?: any): void; @@ -26,6 +27,7 @@ export function hookLogger( redactURICredentials: (uri: string) => string ): void { oidcHookLogger(emitter, log, contextPrefix); + proxyHookLogger(emitter, log, contextPrefix); const { mongoLogId } = log; emitter.on( diff --git a/packages/devtools-connect/src/types.ts b/packages/devtools-connect/src/types.ts index 72b9fb78..42bde5b6 100644 --- a/packages/devtools-connect/src/types.ts +++ b/packages/devtools-connect/src/types.ts @@ -1,3 +1,4 @@ +import { ProxyEventMap } from '@mongodb-js/devtools-proxy-support'; import type { MongoDBOIDCLogEventsMap } from '@mongodb-js/oidc-plugin'; export interface ConnectAttemptInitializedEvent { @@ -55,7 +56,9 @@ export interface ConnectUsedSystemCAEvent { asyncFallbackError: Error | undefined; } -export interface ConnectEventMap extends MongoDBOIDCLogEventsMap { +export interface ConnectEventMap + extends MongoDBOIDCLogEventsMap, + ProxyEventMap { /** Signals that a connection attempt is about to be performed. */ 'devtools-connect:connect-attempt-initialized': ( ev: ConnectAttemptInitializedEvent diff --git a/packages/devtools-proxy-support/src/index.ts b/packages/devtools-proxy-support/src/index.ts index 1ec2233d..e46d074e 100644 --- a/packages/devtools-proxy-support/src/index.ts +++ b/packages/devtools-proxy-support/src/index.ts @@ -5,7 +5,7 @@ export { extractProxySecrets, mergeProxySecrets, } from './proxy-options'; -export { Tunnel, TunnelOptions, setupSocks5Tunnel } from './socks5'; +export { Tunnel, TunnelOptions, createSocks5Tunnel } from './socks5'; export { createAgent, useOrCreateAgent, AgentWithInitialize } from './agent'; export { createFetch, diff --git a/packages/devtools-proxy-support/src/logging.ts b/packages/devtools-proxy-support/src/logging.ts index 22d7127c..d5daf33f 100644 --- a/packages/devtools-proxy-support/src/logging.ts +++ b/packages/devtools-proxy-support/src/logging.ts @@ -78,8 +78,8 @@ let alreadyHooked: WeakMap>; let idCounter = 0; export function hookLogger( emitter: ProxyLogEmitter, - logCtx: string, - log: MongoLogWriter + log: MongoLogWriter, + logCtx: string ): void { // This helps avoid unintentionally attaching the same logging twice in devtools-connet if (alreadyHooked.get(emitter)?.has(log)) return; diff --git a/packages/devtools-proxy-support/src/proxy-options.ts b/packages/devtools-proxy-support/src/proxy-options.ts index 72c8683e..0234f65a 100644 --- a/packages/devtools-proxy-support/src/proxy-options.ts +++ b/packages/devtools-proxy-support/src/proxy-options.ts @@ -218,7 +218,7 @@ export function translateToElectronProxyConfig( } // Return the Socks5 tunnel configuration, if proxyOptions always resolves to one. -// This is used by setupSocks5Tunnel() to avoid creating a local Socks5 tunnel +// This is used by createSocks5Tunnel() to avoid creating a local Socks5 tunnel // that would just forward to another Socks5 tunnel anyway. export function getSocks5OnlyProxyOptions( proxyOptions: DevtoolsProxyOptions, diff --git a/packages/devtools-proxy-support/src/socks5.spec.ts b/packages/devtools-proxy-support/src/socks5.spec.ts index 0ed56b33..bb922af6 100644 --- a/packages/devtools-proxy-support/src/socks5.spec.ts +++ b/packages/devtools-proxy-support/src/socks5.spec.ts @@ -1,14 +1,21 @@ import sinon from 'sinon'; import { HTTPServerProxyTestSetup } from '../test/helpers'; import type { Tunnel, TunnelOptions } from './socks5'; -import { setupSocks5Tunnel } from './socks5'; +import { createSocks5Tunnel } from './socks5'; import { expect } from 'chai'; import { createFetch } from './fetch'; import type { DevtoolsProxyOptions } from './proxy-options'; -describe('setupSocks5Tunnel', function () { +describe('createSocks5Tunnel', function () { let setup: HTTPServerProxyTestSetup; let tunnel: Tunnel | undefined; + const setupSocks5Tunnel = async ( + ...args: Parameters + ): Promise> => { + const tunnel = createSocks5Tunnel(...args); + await tunnel?.listen(); + return tunnel; + }; beforeEach(async function () { setup = new HTTPServerProxyTestSetup(); @@ -139,4 +146,36 @@ describe('setupSocks5Tunnel', function () { ) ).to.deep.equal({ proxyHost: 'example.com', proxyPort: 123 }); }); + + it('can generate access credentials on demand', async function () { + tunnel = await setupSocks5Tunnel( + { + proxy: `http://foo:bar@127.0.0.1:${setup.httpProxyPort}`, + }, + 'generate-credentials' + ); + if (!tunnel) { + // regular conditional instead of assertion so that TS can follow it + expect.fail('failed to create Socks5 tunnel'); + } + + const fetch = createFetch({ + proxy: `socks5://${encodeURIComponent( + tunnel.config.proxyUsername! + )}:${encodeURIComponent(tunnel.config.proxyPassword!)}@127.0.0.1:${ + tunnel.config.proxyPort + }`, + }); + const response = await fetch('http://example.com/hello'); + expect(await response.text()).to.equal('OK /hello'); + + try { + await createFetch({ + proxy: `socks5://AzureDiamond:hunter2@127.0.0.1:${tunnel.config.proxyPort}`, + })('http://localhost:1234/hello'); + expect.fail('missed exception'); + } catch (err) { + expect(err.message).to.include('Socks5 Authentication failed'); + } + }); }); diff --git a/packages/devtools-proxy-support/src/socks5.ts b/packages/devtools-proxy-support/src/socks5.ts index 60767209..41d39c79 100644 --- a/packages/devtools-proxy-support/src/socks5.ts +++ b/packages/devtools-proxy-support/src/socks5.ts @@ -15,6 +15,9 @@ import type { Socket } from 'net'; import type { Duplex } from 'stream'; import type { ClientRequest } from 'http'; import type { ProxyLogEmitter } from './logging'; +import crypto from 'crypto'; + +const randomBytes = promisify(crypto.randomBytes); export interface TunnelOptions { // These can safely be assigned to driver MongoClientOptinos @@ -41,6 +44,7 @@ export interface Tunnel { on(ev: 'forwardingError', cb: (err: Error) => void): void; on(ev: 'error', cb: (err: Error) => void): void; + listen(): Promise; close(): Promise; readonly config: Readonly; @@ -51,60 +55,50 @@ function createFakeHttpClientRequest(dstAddr: string, dstPort: number) { host: `${isIPv6(dstAddr) ? `[${dstAddr}]` : dstAddr}:${dstPort}`, upgrade: 'websocket', // hack to make proxy-agent prefer CONNECT over HTTP proxying }; - return Object.assign(new EventEmitter() as ClientRequest, { - host: headers.host, - protocol: 'http', - method: 'GET', - path: '/', - getHeader(name: string) { - return headers[name]; - }, - }); + return Object.assign( + new EventEmitter().setMaxListeners(Infinity) as ClientRequest, + { + host: headers.host, + protocol: 'http', + method: 'GET', + path: '/', + getHeader(name: string) { + return headers[name]; + }, + } + ); } // The original version of this code was largely taken from // https://github.com/mongodb-js/compass/tree/55a5a608713d7316d158dc66febeb6b114d8b40d/packages/ssh-tunnel/src class Socks5Server extends EventEmitter implements Tunnel { - public logger: ProxyLogEmitter = new EventEmitter(); + public logger: ProxyLogEmitter = new EventEmitter().setMaxListeners(Infinity); private readonly agent: AgentWithInitialize; private server: any; private serverListen: (port?: number, host?: string) => Promise; private serverClose: () => Promise; private connections: Set = new Set(); private rawConfig: TunnelOptions; - private closed = false; + private generateCredentials: boolean; + private closed = true; private agentInitialized = false; private agentInitPromise?: Promise; constructor( agent: AgentWithInitialize, - tunnelOptions: Partial + tunnelOptions: Partial, + generateCredentials: boolean ) { super(); + this.setMaxListeners(Infinity); this.agent = agent; + this.generateCredentials = generateCredentials; if (agent.logger) this.logger = agent.logger; agent.on?.('error', (err: Error) => this.emit('forwardingError', err)); this.rawConfig = getTunnelOptions(tunnelOptions); this.server = socks5Server.createServer(this.socks5Request.bind(this)); - if (this.rawConfig.proxyUsername) { - this.server.useAuth( - socks5AuthUserPassword( - (user: string, pass: string, cb: (success: boolean) => void) => { - const success = - this.rawConfig.proxyUsername === user && - this.rawConfig.proxyPassword === pass; - this.logger.emit('socks5:authentication-complete', { success }); - queueMicrotask(() => cb(success)); - } - ) - ); - } else { - this.logger.emit('socks5:skip-auth-setup'); - this.server.useAuth(socks5AuthNone()); - } - this.serverListen = promisify(this.server.listen.bind(this.server)); this.serverClose = promisify(this.server.close.bind(this.server)); @@ -125,6 +119,33 @@ class Socks5Server extends EventEmitter implements Tunnel { } async listen(): Promise { + this.closed = false; + if (this.generateCredentials) { + const credentialsSource = await randomBytes(64); + this.rawConfig = { + ...this.rawConfig, + proxyUsername: credentialsSource.slice(0, 32).toString('base64url'), + proxyPassword: credentialsSource.slice(32).toString('base64url'), + }; + } + + if (this.rawConfig.proxyUsername) { + this.server.useAuth( + socks5AuthUserPassword( + (user: string, pass: string, cb: (success: boolean) => void) => { + const success = + this.rawConfig.proxyUsername === user && + this.rawConfig.proxyPassword === pass; + this.logger.emit('socks5:authentication-complete', { success }); + queueMicrotask(() => cb(success)); + } + ) + ); + } else { + this.logger.emit('socks5:skip-auth-setup'); + this.server.useAuth(socks5AuthNone()); + } + const { proxyHost, proxyPort } = this.rawConfig; this.logger.emit('socks5:start-listening', { proxyHost, proxyPort }); @@ -185,6 +206,7 @@ class Socks5Server extends EventEmitter implements Tunnel { } async close(): Promise { + if (this.closed) return; this.closed = true; this.logger.emit('socks5:closing-tunnel'); @@ -289,24 +311,29 @@ class Socks5Server extends EventEmitter implements Tunnel { } class ExistingTunnel extends EventEmitter { - logger = new EventEmitter(); + logger = new EventEmitter().setMaxListeners(Infinity); readonly config: TunnelOptions; constructor(config: TunnelOptions) { super(); + this.setMaxListeners(Infinity); this.config = config; } + async listen() { + // nothing to do if we didn't start a server + } + async close() { // nothing to do if we didn't start a server } } -export async function setupSocks5Tunnel( +export function createSocks5Tunnel( proxyOptions: DevtoolsProxyOptions | AgentWithInitialize, - tunnelOptions?: Partial, + tunnelOptions?: Partial | 'generate-credentials', target?: string | undefined -): Promise { +): Tunnel | undefined { const socks5OnlyProxyOptions = getSocks5OnlyProxyOptions( ('proxyOptions' in proxyOptions ? proxyOptions.proxyOptions @@ -320,7 +347,11 @@ export async function setupSocks5Tunnel( const agent = useOrCreateAgent(proxyOptions, target); if (!agent) return undefined; - const server = new Socks5Server(agent, { ...tunnelOptions }); - await server.listen(); - return server; + let generateCredentials = false; + if (tunnelOptions === 'generate-credentials') { + tunnelOptions = {}; + generateCredentials = true; + } + + return new Socks5Server(agent, { ...tunnelOptions }, generateCredentials); } diff --git a/packages/devtools-proxy-support/src/ssh.ts b/packages/devtools-proxy-support/src/ssh.ts index 74261b50..b9b74871 100644 --- a/packages/devtools-proxy-support/src/ssh.ts +++ b/packages/devtools-proxy-support/src/ssh.ts @@ -35,7 +35,7 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { (this as AgentWithInitialize).on?.('error', () => { // Errors here should not crash the process }); - this.logger = logger ?? new EventEmitter(); + this.logger = logger ?? new EventEmitter().setMaxListeners(Infinity); this.proxyOptions = options; this.url = new URL(options.proxy ?? ''); this.sshClient = new SshClient(); From 36e66f50c6c18c95a79e4150a25a75a2404475a6 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 24 Jul 2024 17:16:09 +0200 Subject: [PATCH 16/30] WIP --- package-lock.json | 2 ++ packages/devtools-proxy-support/src/logging.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 8b326062..ee1af327 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25221,6 +25221,7 @@ "version": "3.0.5", "license": "Apache-2.0", "dependencies": { + "@mongodb-js/devtools-proxy-support": "^0.1.0", "@mongodb-js/oidc-http-server-pages": "1.1.2", "lodash.merge": "^4.6.2", "mongodb-connection-string-url": "^3.0.0", @@ -31968,6 +31969,7 @@ "@mongodb-js/devtools-connect": { "version": "file:packages/devtools-connect", "requires": { + "@mongodb-js/devtools-proxy-support": "^0.1.0", "@mongodb-js/oidc-http-server-pages": "1.1.2", "@mongodb-js/oidc-plugin": "^1.0.0", "@mongodb-js/saslprep": "^1.1.8", diff --git a/packages/devtools-proxy-support/src/logging.ts b/packages/devtools-proxy-support/src/logging.ts index d5daf33f..2046fa23 100644 --- a/packages/devtools-proxy-support/src/logging.ts +++ b/packages/devtools-proxy-support/src/logging.ts @@ -74,7 +74,7 @@ interface MongoLogWriter { mongoLogId(this: void, id: number): unknown; } -let alreadyHooked: WeakMap>; +const alreadyHooked = new WeakMap>(); let idCounter = 0; export function hookLogger( emitter: ProxyLogEmitter, From 11ff6758d8201e19791d6d2ae51af03fdf6c11e4 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 24 Jul 2024 17:27:02 +0200 Subject: [PATCH 17/30] WIP --- packages/devtools-proxy-support/src/logging.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/packages/devtools-proxy-support/src/logging.ts b/packages/devtools-proxy-support/src/logging.ts index 2046fa23..0b148a98 100644 --- a/packages/devtools-proxy-support/src/logging.ts +++ b/packages/devtools-proxy-support/src/logging.ts @@ -74,18 +74,12 @@ interface MongoLogWriter { mongoLogId(this: void, id: number): unknown; } -const alreadyHooked = new WeakMap>(); let idCounter = 0; export function hookLogger( emitter: ProxyLogEmitter, log: MongoLogWriter, logCtx: string ): void { - // This helps avoid unintentionally attaching the same logging twice in devtools-connet - if (alreadyHooked.get(emitter)?.has(log)) return; - if (!alreadyHooked.has(emitter)) alreadyHooked.set(emitter, new WeakSet()); - alreadyHooked.get(emitter)!.add(log); - logCtx = `${logCtx}-${idCounter++}`; const { mongoLogId } = log; From 37092f4b219783846d10ca5de7986210d3e0816e Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 25 Jul 2024 14:05:29 +0200 Subject: [PATCH 18/30] WIP --- packages/devtools-connect/src/connect.ts | 6 +- packages/devtools-proxy-support/src/agent.ts | 81 +++++++++++++++---- .../src/proxy-options.spec.ts | 28 ++++++- .../src/proxy-options.ts | 51 ++++++------ .../devtools-proxy-support/src/socks5.spec.ts | 23 ++++++ packages/devtools-proxy-support/src/socks5.ts | 33 +++++++- 6 files changed, 173 insertions(+), 49 deletions(-) diff --git a/packages/devtools-connect/src/connect.ts b/packages/devtools-connect/src/connect.ts index 93a7edd1..f7f01286 100644 --- a/packages/devtools-connect/src/connect.ts +++ b/packages/devtools-connect/src/connect.ts @@ -469,7 +469,11 @@ export async function connectMongoClient( let tunnel: Tunnel | undefined; if (proxyAgent && !hasProxyHostOption(uri, clientOptions)) { - tunnel = createSocks5Tunnel(proxyAgent, 'generate-credentials'); + tunnel = createSocks5Tunnel( + proxyAgent, + 'generate-credentials', + 'mongodb://' + ); cleanupOnClientClose.push(() => tunnel?.close()); } for (const proxyLogger of new Set([tunnel?.logger, proxyAgent?.logger])) { diff --git a/packages/devtools-proxy-support/src/agent.ts b/packages/devtools-proxy-support/src/agent.ts index 65cc9b34..2fe1fe70 100644 --- a/packages/devtools-proxy-support/src/agent.ts +++ b/packages/devtools-proxy-support/src/agent.ts @@ -2,13 +2,14 @@ import { ProxyAgent } from 'proxy-agent'; import type { Agent } from 'https'; import type { DevtoolsProxyOptions } from './proxy-options'; import { proxyForUrl } from './proxy-options'; -import type { ClientRequest } from 'http'; +import type { ClientRequest, Agent as HTTPAgent } from 'http'; import type { TcpNetConnectOpts } from 'net'; import type { ConnectionOptions } from 'tls'; import type { Duplex } from 'stream'; import { SSHAgent } from './ssh'; import type { ProxyLogEmitter } from './logging'; import type { EventEmitter } from 'events'; +import type { AgentConnectOpts } from 'agent-base'; // Helper type that represents an https.Agent (= connection factory) // with some custom properties that TS does not know about and/or @@ -31,23 +32,71 @@ export type AgentWithInitialize = Agent & { // http.Agent is an EventEmitter, just missing from @types/node } & Partial; +class DevtoolsProxyAgent extends ProxyAgent implements AgentWithInitialize { + readonly proxyOptions: DevtoolsProxyOptions; + private sshAgent: SSHAgent | undefined; + + // Store the current ClientRequest for the time between connect() first + // being called and the corresponding _getProxyForUrl() being called. + // In practice, this is instantaneous, but that is not guaranteed by + // the `ProxyAgent` API contract. + // We use a Promise lock/mutex to avoid concurrent accesses. + private _req: ClientRequest | undefined; + private _reqLock: Promise | undefined; + private _reqLockResolve: (() => void) | undefined; + + constructor(proxyOptions: DevtoolsProxyOptions) { + super({ + getProxyForUrl: (url: string) => this._getProxyForUrl(url), + ...proxyOptions, + }); + this.proxyOptions = proxyOptions; + // This could be made a bit more flexible by actually dynamically picking + // ssh vs. other proxy protocols as part of connect(), if we want that at some point. + if (proxyOptions.proxy && new URL(proxyOptions.proxy).protocol === 'ssh:') { + this.sshAgent = new SSHAgent(proxyOptions); + } + } + + _getProxyForUrl = (url: string): string => { + if (!this._reqLockResolve || !this._req) { + throw new Error('getProxyForUrl() called without pending request'); + } + this._reqLockResolve(); + const req = this._req; + this._req = undefined; + this._reqLock = undefined; + this._reqLockResolve = undefined; + return proxyForUrl(this.proxyOptions, url, req); + }; + + async initialize(): Promise { + await this.sshAgent?.initialize(); + } + + override async connect( + req: ClientRequest, + opts: AgentConnectOpts + ): Promise { + if (this.sshAgent) return this.sshAgent; + while (this._reqLock) { + await this._reqLock; + } + this._req = req; + this._reqLock = new Promise((resolve) => (this._reqLockResolve = resolve)); + return await super.connect(req, opts); + } + + destroy(): void { + this.sshAgent?.destroy(); + super.destroy(); + } +} + export function createAgent( proxyOptions: DevtoolsProxyOptions ): AgentWithInitialize { - // This could be made a bit more flexible by creating an Agent using AgentBase - // that will dynamically choose between SSHAgent and ProxyAgent. - // Right now, this is a bit simpler in terms of lifetime management for SSHAgent. - if (proxyOptions.proxy && new URL(proxyOptions.proxy).protocol === 'ssh:') { - return new SSHAgent(proxyOptions); - } - const getProxyForUrl = proxyForUrl(proxyOptions); - return Object.assign( - new ProxyAgent({ - getProxyForUrl, - ...proxyOptions, - }), - { proxyOptions } - ); + return new DevtoolsProxyAgent(proxyOptions); } export function useOrCreateAgent( @@ -59,7 +108,7 @@ export function useOrCreateAgent( } else { if ( target !== undefined && - !proxyForUrl(proxyOptions as DevtoolsProxyOptions)(target) + !proxyForUrl(proxyOptions as DevtoolsProxyOptions, target) ) return undefined; return createAgent(proxyOptions as DevtoolsProxyOptions); diff --git a/packages/devtools-proxy-support/src/proxy-options.spec.ts b/packages/devtools-proxy-support/src/proxy-options.spec.ts index 1d9695e3..7fd844e7 100644 --- a/packages/devtools-proxy-support/src/proxy-options.spec.ts +++ b/packages/devtools-proxy-support/src/proxy-options.spec.ts @@ -37,7 +37,9 @@ describe('proxy options handling', function () { describe('proxyForUrl', function () { it('should return a proxy function for a specified proxy URL', function () { - const getProxy = proxyForUrl({ proxy: 'http://proxy.example.com' }); + const getProxy = proxyForUrl.bind(null, { + proxy: 'http://proxy.example.com', + }); expect(getProxy('http://target.com')).to.equal( 'http://proxy.example.com' @@ -45,7 +47,7 @@ describe('proxy options handling', function () { }); it('should respect noProxyHosts', function () { - const getProxy = proxyForUrl({ + const getProxy = proxyForUrl.bind(null, { proxy: 'http://proxy.example.com', noProxyHosts: 'localhost', }); @@ -57,11 +59,12 @@ describe('proxy options handling', function () { }); it('should use environment variables as a fallback', function () { - const getProxy = proxyForUrl({ + const getProxy = proxyForUrl.bind(null, { useEnvironmentVariableProxies: true, env: { HTTP_PROXY: 'socks5://env-proxy.example.com', NO_PROXY: 'localhost', + ALL_PROXY: 'http://fallback.example.com', }, }); @@ -70,6 +73,9 @@ describe('proxy options handling', function () { expect(getProxy('http://example.com')).to.equal( 'socks5://env-proxy.example.com' ); + expect(getProxy('mongodb://example.com')).to.equal( + 'http://fallback.example.com' + ); }); }); @@ -127,12 +133,14 @@ describe('proxy options handling', function () { env: { HTTP_PROXY: 'socks5://env-proxy.example.com', NO_PROXY: 'zombo.com', + ALL_PROXY: 'http://fallback.example.com', }, }) ).to.deep.equal({ mode: 'fixed_servers', proxyBypassRules: 'localhost,example.com,zombo.com', - proxyRules: 'http=socks5://env-proxy.example.com', + proxyRules: + 'http=socks5://env-proxy.example.com;https=http://fallback.example.com;ftp=http://fallback.example.com', }); }); it('translates an empty config to an empty config', function () { @@ -330,6 +338,18 @@ describe('proxy options handling', function () { expect(await testResolveProxy(config, 'https://example.com')).to.equal( 'DIRECT' ); + expect( + await testResolveProxy( + { + ...config, + env: { + ...config.env, + ALL_PROXY: 'http://fallback.example.com:1', + }, + }, + 'ftp://mongodb.net' + ) + ).to.equal('PROXY fallback.example.com:1'); }); }); }); diff --git a/packages/devtools-proxy-support/src/proxy-options.ts b/packages/devtools-proxy-support/src/proxy-options.ts index 0234f65a..6044cc4c 100644 --- a/packages/devtools-proxy-support/src/proxy-options.ts +++ b/packages/devtools-proxy-support/src/proxy-options.ts @@ -1,5 +1,6 @@ import type { ConnectionOptions } from 'tls'; import type { TunnelOptions } from './socks5'; +import type { ClientRequest } from 'http'; // Should be an opaque type, but TS does not support those. export type DevtoolsProxyOptionsSecrets = string; @@ -78,38 +79,36 @@ function shouldProxy(noProxy: string, url: URL): boolean { // Create a function which returns the proxy URL for a given target URL, // based on the proxy config passed to it. export function proxyForUrl( - proxyOptions: DevtoolsProxyOptions -): (url: string) => string { + proxyOptions: DevtoolsProxyOptions, + target: string, + req?: ClientRequest & { overrideProtocol?: string } +): string { if (proxyOptions.proxy) { const proxyUrl = proxyOptions.proxy; - if (new URL(proxyUrl).protocol === 'direct:') return () => ''; - return (target: string) => { - if (shouldProxy(proxyOptions.noProxyHosts || '', new URL(target))) { - return proxyUrl; - } - return ''; - }; + if (new URL(proxyUrl).protocol === 'direct:') return ''; + if (shouldProxy(proxyOptions.noProxyHosts || '', new URL(target))) { + return proxyUrl; + } + return ''; } if (proxyOptions.useEnvironmentVariableProxies) { const { map, noProxy } = proxyConfForEnvVars( proxyOptions.env ?? process.env ); - return (target: string) => { - const url = new URL(target); - const protocol = url.protocol.replace(/:$/, ''); - const combinedNoProxyRules = [noProxy, proxyOptions.noProxyHosts] - .filter(Boolean) - .join(','); - const proxyForProtocol = map.get(protocol); - if (proxyForProtocol && shouldProxy(combinedNoProxyRules, url)) { - return proxyForProtocol; - } - return ''; - }; + const url = new URL(target); + const protocol = (req?.overrideProtocol ?? url.protocol).replace(/:$/, ''); + const combinedNoProxyRules = [noProxy, proxyOptions.noProxyHosts] + .filter(Boolean) + .join(','); + const proxyForProtocol = map.get(protocol) || map.get('all'); + if (proxyForProtocol && shouldProxy(combinedNoProxyRules, url)) { + return proxyForProtocol; + } + return ''; } - return () => ''; + return ''; } function validateElectronProxyURL(url: URL | string): string { @@ -194,8 +193,12 @@ export function translateToElectronProxyConfig( const { map, noProxy } = proxyConfForEnvVars( proxyOptions.env ?? process.env ); - for (const [key, value] of map) + for (const key of ['http', 'https', 'ftp']) { + // supported protocols for Electron proxying + const value = map.get(key) || map.get('all'); + if (!value) continue; proxyRulesList.push(`${key}=${validateElectronProxyURL(value)}`); + } proxyBypassRulesList.push(noProxy); const proxyRules = proxyRulesList.join(';'); @@ -225,7 +228,7 @@ export function getSocks5OnlyProxyOptions( target?: string ): TunnelOptions | undefined { let proxyUrl: string | undefined; - if (target !== undefined) proxyUrl = proxyForUrl(proxyOptions)(target); + if (target !== undefined) proxyUrl = proxyForUrl(proxyOptions, target); else if (!proxyOptions.noProxyHosts) proxyUrl = proxyOptions.proxy; if (!proxyUrl) return undefined; const url = new URL(proxyUrl); diff --git a/packages/devtools-proxy-support/src/socks5.spec.ts b/packages/devtools-proxy-support/src/socks5.spec.ts index bb922af6..a50e8fab 100644 --- a/packages/devtools-proxy-support/src/socks5.spec.ts +++ b/packages/devtools-proxy-support/src/socks5.spec.ts @@ -178,4 +178,27 @@ describe('createSocks5Tunnel', function () { expect(err.message).to.include('Socks5 Authentication failed'); } }); + + it('picks the proxy specified by the target protocol, if any', async function () { + tunnel = await setupSocks5Tunnel( + { + useEnvironmentVariableProxies: true, + env: { + MONGODB_PROXY: `http://foo:bar@127.0.0.1:${setup.httpProxyPort}`, + }, + }, + {}, + 'mongodb://' + ); + if (!tunnel) { + // regular conditional instead of assertion so that TS can follow it + expect.fail('failed to create Socks5 tunnel'); + } + + const fetch = createFetch({ + proxy: `socks5://@127.0.0.1:${tunnel.config.proxyPort}`, + }); + const response = await fetch('http://example.com/hello'); + expect(await response.text()).to.equal('OK /hello'); + }); }); diff --git a/packages/devtools-proxy-support/src/socks5.ts b/packages/devtools-proxy-support/src/socks5.ts index 41d39c79..ba940102 100644 --- a/packages/devtools-proxy-support/src/socks5.ts +++ b/packages/devtools-proxy-support/src/socks5.ts @@ -50,7 +50,11 @@ export interface Tunnel { readonly config: Readonly; } -function createFakeHttpClientRequest(dstAddr: string, dstPort: number) { +function createFakeHttpClientRequest( + dstAddr: string, + dstPort: number, + overrideProtocol: string | undefined +) { const headers: Record = { host: `${isIPv6(dstAddr) ? `[${dstAddr}]` : dstAddr}:${dstPort}`, upgrade: 'websocket', // hack to make proxy-agent prefer CONNECT over HTTP proxying @@ -65,6 +69,7 @@ function createFakeHttpClientRequest(dstAddr: string, dstPort: number) { getHeader(name: string) { return headers[name]; }, + overrideProtocol, } ); } @@ -80,6 +85,7 @@ class Socks5Server extends EventEmitter implements Tunnel { private connections: Set = new Set(); private rawConfig: TunnelOptions; private generateCredentials: boolean; + private overrideProtocol: string | undefined; private closed = true; private agentInitialized = false; private agentInitPromise?: Promise; @@ -87,12 +93,15 @@ class Socks5Server extends EventEmitter implements Tunnel { constructor( agent: AgentWithInitialize, tunnelOptions: Partial, - generateCredentials: boolean + generateCredentials: boolean, + overrideProtocol: string | undefined ) { super(); this.setMaxListeners(Infinity); this.agent = agent; this.generateCredentials = generateCredentials; + this.overrideProtocol = overrideProtocol; + if (agent.logger) this.logger = agent.logger; agent.on?.('error', (err: Error) => this.emit('forwardingError', err)); this.rawConfig = getTunnelOptions(tunnelOptions); @@ -229,7 +238,11 @@ class Socks5Server extends EventEmitter implements Tunnel { private async forwardOut(dstAddr: string, dstPort: number): Promise { const channel = await new Promise((resolve, reject) => { - const req = createFakeHttpClientRequest(dstAddr, dstPort); + const req = createFakeHttpClientRequest( + dstAddr, + dstPort, + this.overrideProtocol + ); req.onSocket = (sock) => { if (sock) resolve(sock); }; @@ -329,6 +342,13 @@ class ExistingTunnel extends EventEmitter { } } +// Open a local Socks5 server, which accepts incoming connections and forwards them +// using the proxy specified in `proxyOptions`. `tunnelOptions` may specify an +// address to listen on, as well as credentials for the server. Passing +// `generate-credentials` will lead to random credentials being generated as part of +// `server.listen()`. `target` should specify an URL which is used for determining +// which proxy to use, and in particular the protocol specified will be used +// for determining this proxy. export function createSocks5Tunnel( proxyOptions: DevtoolsProxyOptions | AgentWithInitialize, tunnelOptions?: Partial | 'generate-credentials', @@ -353,5 +373,10 @@ export function createSocks5Tunnel( generateCredentials = true; } - return new Socks5Server(agent, { ...tunnelOptions }, generateCredentials); + return new Socks5Server( + agent, + { ...tunnelOptions }, + generateCredentials, + target ? new URL(target).protocol : undefined + ); } From cb123123b1ff924e2e3bad6d15c9f55a60891ab8 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 25 Jul 2024 15:40:28 +0200 Subject: [PATCH 19/30] fixup: ignore code-points-data changes for testing --- .github/workflows/check-test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/check-test.yaml b/.github/workflows/check-test.yaml index 61bc03f7..1f94ec4e 100644 --- a/.github/workflows/check-test.yaml +++ b/.github/workflows/check-test.yaml @@ -74,6 +74,8 @@ jobs: npm ci npm run bootstrap-ci -- --scope @mongodb-js/monorepo-tools --stream --include-dependencies npm run bootstrap-ci -- --stream --since ${SINCE_REF} --include-dependencies + + git checkout -- packages/saslprep/src/code-points-data.ts shell: bash - name: Info From 6d48fbc3f7081b55456dfa2834e763111408c3a7 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 25 Jul 2024 15:58:17 +0200 Subject: [PATCH 20/30] fixup: improve debug output for electron integration test --- .../src/proxy-options.spec.ts | 191 +++++++++--------- 1 file changed, 100 insertions(+), 91 deletions(-) diff --git a/packages/devtools-proxy-support/src/proxy-options.spec.ts b/packages/devtools-proxy-support/src/proxy-options.spec.ts index 7fd844e7..ccde33db 100644 --- a/packages/devtools-proxy-support/src/proxy-options.spec.ts +++ b/packages/devtools-proxy-support/src/proxy-options.spec.ts @@ -166,8 +166,9 @@ describe('proxy options handling', function () { let runJS: (js: string) => Promise; let testResolveProxy: ( proxyOptions: DevtoolsProxyOptions, - url: string - ) => Promise; + url: string, + expectation: string + ) => Promise; let setup: HTTPServerProxyTestSetup; before(async function () { @@ -219,14 +220,16 @@ describe('proxy options handling', function () { return JSON.parse(response); }; - testResolveProxy = async (proxyOptions, url) => { + testResolveProxy = async (proxyOptions, url, expectation) => { + const config = translateToElectronProxyConfig(proxyOptions); // https://www.electronjs.org/docs/latest/api/app#appsetproxyconfig // https://www.electronjs.org/docs/latest/api/app#appresolveproxyurl - return await runJS(`app.setProxy(${JSON.stringify( - translateToElectronProxyConfig(proxyOptions) + const actual = await runJS(`app.setProxy(${JSON.stringify( + config )}).then(_ => { return app.resolveProxy(${JSON.stringify(url)}); })`); + expect({ actual, config }).to.have.property('actual', expectation); }; }); @@ -243,68 +246,73 @@ describe('proxy options handling', function () { }); it('correctly handles explicit proxies', async function () { - expect( - await testResolveProxy( - { - proxy: 'http://example.com:12345', - }, - 'http://example.net' - ) - ).to.equal('PROXY example.com:12345'); - expect( - await testResolveProxy( - { - proxy: 'http://example.com:12345', - noProxyHosts: 'localhost', - }, - 'http://example.net' - ) - ).to.equal('PROXY example.com:12345'); - expect( - await testResolveProxy( - { - proxy: 'http://example.com:12345', - noProxyHosts: 'localhost', - }, - 'http://localhost' - ) - ).to.equal('DIRECT'); - expect( - await testResolveProxy( - { - proxy: 'http://example.com:12345', - noProxyHosts: 'localhost', - }, - 'http://localhost:1234' - ) - ).to.equal('DIRECT'); - expect( - await testResolveProxy( - { - proxy: 'socks5://example.com:12345', - }, - 'http://example.net' - ) - ).to.equal('SOCKS5 example.com:12345'); + await testResolveProxy( + { + proxy: 'http://example.com:12345', + }, + 'http://example.net', + + 'PROXY example.com:12345' + ); + + await testResolveProxy( + { + proxy: 'http://example.com:12345', + noProxyHosts: 'localhost', + }, + 'http://example.net', + + 'PROXY example.com:12345' + ); + + await testResolveProxy( + { + proxy: 'http://example.com:12345', + noProxyHosts: 'localhost', + }, + 'http://localhost', + + 'DIRECT' + ); + + await testResolveProxy( + { + proxy: 'http://example.com:12345', + noProxyHosts: 'localhost', + }, + 'http://localhost:1234', + + 'DIRECT' + ); + + await testResolveProxy( + { + proxy: 'socks5://example.com:12345', + }, + 'http://example.net', + + 'SOCKS5 example.com:12345' + ); }); it('correctly handles pac-script-specified proxies', async function () { - expect( - await testResolveProxy( - { - proxy: `pac+http://127.0.0.1:${setup.httpServerPort}/pac`, - }, - 'http://example.com' - ) - ).to.equal(`SOCKS5 127.0.0.1:${setup.socks5ProxyPort}`); - expect( - await testResolveProxy( - { - proxy: `pac+http://127.0.0.1:${setup.httpServerPort}/pac`, - }, - 'http://pac-invalidproxy/test' - ) - ).to.equal(`SOCKS5 127.0.0.1:1`); + await testResolveProxy( + { + proxy: `pac+http://127.0.0.1:${setup.httpServerPort}/pac`, + }, + 'http://example.com', + + `SOCKS5 127.0.0.1:${setup.socks5ProxyPort}` + ); + + await testResolveProxy( + { + proxy: `pac+http://127.0.0.1:${setup.httpServerPort}/pac`, + }, + 'http://pac-invalidproxy/test', + + `SOCKS5 127.0.0.1:1` + ); }); it('correctly handles environment-specified proxies', async function () { @@ -317,39 +325,40 @@ describe('proxy options handling', function () { noProxyHosts: 'example.net:4567', useEnvironmentVariableProxies: true, }; - expect(await testResolveProxy(config, 'http://localhost')).to.equal( - 'DIRECT' - ); - expect(await testResolveProxy(config, 'http://example.net')).to.equal( + await testResolveProxy(config, 'http://localhost', 'DIRECT'); + await testResolveProxy( + config, + 'http://example.net', 'HTTPS http-proxy.example.net:443' ); - expect(await testResolveProxy(config, 'https://example.net')).to.equal( + await testResolveProxy( + config, + 'https://example.net', 'PROXY https-proxy.example.net:80' ); - expect( - await testResolveProxy(config, 'https://example.net:1234') - ).to.equal('DIRECT'); - expect( - await testResolveProxy(config, 'https://example.net:4567') - ).to.equal('DIRECT'); - expect( - await testResolveProxy(config, 'https://example.net:9801') - ).to.equal('PROXY https-proxy.example.net:80'); - expect(await testResolveProxy(config, 'https://example.com')).to.equal( - 'DIRECT' + + await testResolveProxy(config, 'https://example.net:1234', 'DIRECT'); + + await testResolveProxy(config, 'https://example.net:4567', 'DIRECT'); + + await testResolveProxy( + config, + 'https://example.net:9801', + 'PROXY https-proxy.example.net:80' ); - expect( - await testResolveProxy( - { - ...config, - env: { - ...config.env, - ALL_PROXY: 'http://fallback.example.com:1', - }, + await testResolveProxy(config, 'https://example.com', 'DIRECT'); + await testResolveProxy( + { + ...config, + env: { + ...config.env, + ALL_PROXY: 'http://fallback.example.com:1', }, - 'ftp://mongodb.net' - ) - ).to.equal('PROXY fallback.example.com:1'); + }, + 'ftp://mongodb.net', + + 'PROXY fallback.example.com:1' + ); }); }); }); From e7c01effeb3a4be1c6841b900d037cf4b0e94f39 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 25 Jul 2024 16:06:56 +0200 Subject: [PATCH 21/30] fixup: explain github ci config change --- .github/workflows/check-test.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/check-test.yaml b/.github/workflows/check-test.yaml index 1f94ec4e..5872f971 100644 --- a/.github/workflows/check-test.yaml +++ b/.github/workflows/check-test.yaml @@ -75,6 +75,9 @@ jobs: npm run bootstrap-ci -- --scope @mongodb-js/monorepo-tools --stream --include-dependencies npm run bootstrap-ci -- --stream --since ${SINCE_REF} --include-dependencies + # saslprep source code may have been modified by bootstrapping, + # depending on the OS, so undo that change if it has happened + # (since it can influence subsequent lerna invocations) git checkout -- packages/saslprep/src/code-points-data.ts shell: bash From d5bd5fc567f10856275d8266568847c968b297cf Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 25 Jul 2024 16:12:46 +0200 Subject: [PATCH 22/30] fixup: improve test failure output --- packages/devtools-proxy-support/src/proxy-options.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/devtools-proxy-support/src/proxy-options.spec.ts b/packages/devtools-proxy-support/src/proxy-options.spec.ts index ccde33db..3ec1510b 100644 --- a/packages/devtools-proxy-support/src/proxy-options.spec.ts +++ b/packages/devtools-proxy-support/src/proxy-options.spec.ts @@ -221,12 +221,12 @@ describe('proxy options handling', function () { }; testResolveProxy = async (proxyOptions, url, expectation) => { - const config = translateToElectronProxyConfig(proxyOptions); + const config = JSON.stringify( + translateToElectronProxyConfig(proxyOptions) + ); // https://www.electronjs.org/docs/latest/api/app#appsetproxyconfig // https://www.electronjs.org/docs/latest/api/app#appresolveproxyurl - const actual = await runJS(`app.setProxy(${JSON.stringify( - config - )}).then(_ => { + const actual = await runJS(`app.setProxy(${config}).then(_ => { return app.resolveProxy(${JSON.stringify(url)}); })`); expect({ actual, config }).to.have.property('actual', expectation); From c7b8e4ccc26cc496dc7ad382df3bacf8e23a37e7 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 25 Jul 2024 16:13:06 +0200 Subject: [PATCH 23/30] DEBUG --- packages/devtools-proxy-support/test/electron-test-server.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/devtools-proxy-support/test/electron-test-server.js b/packages/devtools-proxy-support/test/electron-test-server.js index dca409ec..d2f72e67 100644 --- a/packages/devtools-proxy-support/test/electron-test-server.js +++ b/packages/devtools-proxy-support/test/electron-test-server.js @@ -25,6 +25,8 @@ const { connect } = require('net'); } catch (err) { result = { message: err.message, stack: err.stack, ...err }; } + // eslint-disable-next-line no-console + console.error({ result, readyToExecute }); socket.write(JSON.stringify(result) + '\0'); } } From 665927d8fea413c9f20e4654f2c150b0ff6973c5 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 25 Jul 2024 16:56:13 +0200 Subject: [PATCH 24/30] Revert "DEBUG" This reverts commit c7b8e4ccc26cc496dc7ad382df3bacf8e23a37e7. --- packages/devtools-proxy-support/test/electron-test-server.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/devtools-proxy-support/test/electron-test-server.js b/packages/devtools-proxy-support/test/electron-test-server.js index d2f72e67..dca409ec 100644 --- a/packages/devtools-proxy-support/test/electron-test-server.js +++ b/packages/devtools-proxy-support/test/electron-test-server.js @@ -25,8 +25,6 @@ const { connect } = require('net'); } catch (err) { result = { message: err.message, stack: err.stack, ...err }; } - // eslint-disable-next-line no-console - console.error({ result, readyToExecute }); socket.write(JSON.stringify(result) + '\0'); } } From 8e5c1426a85936835e14d81dbe61c6ea08b8563a Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 25 Jul 2024 16:56:26 +0200 Subject: [PATCH 25/30] fixup: what if this is just prefixed by a sanity check...? --- packages/devtools-proxy-support/src/proxy-options.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/devtools-proxy-support/src/proxy-options.spec.ts b/packages/devtools-proxy-support/src/proxy-options.spec.ts index 3ec1510b..950a0e5f 100644 --- a/packages/devtools-proxy-support/src/proxy-options.spec.ts +++ b/packages/devtools-proxy-support/src/proxy-options.spec.ts @@ -246,6 +246,8 @@ describe('proxy options handling', function () { }); it('correctly handles explicit proxies', async function () { + await testResolveProxy({}, 'http://example.net', 'DIRECT'); + await testResolveProxy( { proxy: 'http://example.com:12345', From 399521b12fabc95cb97499d29f6cd96a097c5352 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Thu, 25 Jul 2024 17:22:32 +0200 Subject: [PATCH 26/30] fixup: drive-by: bump oidc-plugin to 1.1.0 --- package-lock.json | 18 +++++++++--------- packages/devtools-connect/package.json | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index ee1af327..c2754d37 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5823,9 +5823,9 @@ "link": true }, "node_modules/@mongodb-js/oidc-plugin": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@mongodb-js/oidc-plugin/-/oidc-plugin-1.0.0.tgz", - "integrity": "sha512-fWEvEzBKRN3HmYw0AHpPLPGp81TwdP7CrtTnqlW+yzH/m6HnnqkYPzaPur2nVOKCpRaxufFlKXx3jg0klgf/uA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/oidc-plugin/-/oidc-plugin-1.1.0.tgz", + "integrity": "sha512-edf5cMYpuJHfbxAyc6d0fDxeO3bE50w9vagi4NDfUH+Pz3gVN4Uj7rQUYCKMwjuKVE8hxyVqTgd0oSYAqsLjmA==", "dev": true, "dependencies": { "express": "^4.18.2", @@ -25229,7 +25229,7 @@ "system-ca": "^2.0.0" }, "devDependencies": { - "@mongodb-js/oidc-plugin": "^1.0.0", + "@mongodb-js/oidc-plugin": "^1.1.0", "@mongodb-js/saslprep": "^1.1.8", "@types/lodash.merge": "^4.6.7", "@types/mocha": "^9.0.0", @@ -25262,7 +25262,7 @@ "resolve-mongodb-srv": "^1.1.1" }, "peerDependencies": { - "@mongodb-js/oidc-plugin": "^1.0.0", + "@mongodb-js/oidc-plugin": "^1.1.0", "mongodb": "^6.8.0", "mongodb-log-writer": "^1.4.2" } @@ -31971,7 +31971,7 @@ "requires": { "@mongodb-js/devtools-proxy-support": "^0.1.0", "@mongodb-js/oidc-http-server-pages": "1.1.2", - "@mongodb-js/oidc-plugin": "^1.0.0", + "@mongodb-js/oidc-plugin": "^1.1.0", "@mongodb-js/saslprep": "^1.1.8", "@types/lodash.merge": "^4.6.7", "@types/mocha": "^9.0.0", @@ -32993,9 +32993,9 @@ } }, "@mongodb-js/oidc-plugin": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@mongodb-js/oidc-plugin/-/oidc-plugin-1.0.0.tgz", - "integrity": "sha512-fWEvEzBKRN3HmYw0AHpPLPGp81TwdP7CrtTnqlW+yzH/m6HnnqkYPzaPur2nVOKCpRaxufFlKXx3jg0klgf/uA==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/oidc-plugin/-/oidc-plugin-1.1.0.tgz", + "integrity": "sha512-edf5cMYpuJHfbxAyc6d0fDxeO3bE50w9vagi4NDfUH+Pz3gVN4Uj7rQUYCKMwjuKVE8hxyVqTgd0oSYAqsLjmA==", "dev": true, "requires": { "express": "^4.18.2", diff --git a/packages/devtools-connect/package.json b/packages/devtools-connect/package.json index 303d2782..ae7c226a 100644 --- a/packages/devtools-connect/package.json +++ b/packages/devtools-connect/package.json @@ -55,12 +55,12 @@ "system-ca": "^2.0.0" }, "peerDependencies": { - "@mongodb-js/oidc-plugin": "^1.0.0", + "@mongodb-js/oidc-plugin": "^1.1.0", "mongodb": "^6.8.0", "mongodb-log-writer": "^1.4.2" }, "devDependencies": { - "@mongodb-js/oidc-plugin": "^1.0.0", + "@mongodb-js/oidc-plugin": "^1.1.0", "@mongodb-js/saslprep": "^1.1.8", "@types/lodash.merge": "^4.6.7", "@types/mocha": "^9.0.0", From c6d99f8654f8ee3bc407272c91c62c47633b69b4 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 31 Jul 2024 16:40:30 +0200 Subject: [PATCH 27/30] fixup: cr --- packages/devtools-proxy-support/src/socks5.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/devtools-proxy-support/src/socks5.ts b/packages/devtools-proxy-support/src/socks5.ts index ba940102..172b7d89 100644 --- a/packages/devtools-proxy-support/src/socks5.ts +++ b/packages/devtools-proxy-support/src/socks5.ts @@ -258,6 +258,12 @@ class Socks5Server extends EventEmitter implements Tunnel { // the agent resolved to another agent (as is the case for e.g. `ProxyAgent`). if (err) reject(err); else if (sock) resolve(sock); + else + reject( + new Error( + 'Received neither error object nor socket from agent.createSocket()' + ) + ); } ); }); From 191abb158f5160b3475f2e9c4b6e1d8169ef4f3b Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 31 Jul 2024 16:49:28 +0200 Subject: [PATCH 28/30] fixup: apply suggestions from code review Co-authored-by: Sergey Petushkov --- packages/devtools-proxy-support/src/agent.ts | 5 +++-- packages/devtools-proxy-support/src/ssh.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/devtools-proxy-support/src/agent.ts b/packages/devtools-proxy-support/src/agent.ts index 2fe1fe70..4d9f955b 100644 --- a/packages/devtools-proxy-support/src/agent.ts +++ b/packages/devtools-proxy-support/src/agent.ts @@ -47,8 +47,8 @@ class DevtoolsProxyAgent extends ProxyAgent implements AgentWithInitialize { constructor(proxyOptions: DevtoolsProxyOptions) { super({ - getProxyForUrl: (url: string) => this._getProxyForUrl(url), ...proxyOptions, + getProxyForUrl: (url: string) => this._getProxyForUrl(url), }); this.proxyOptions = proxyOptions; // This could be made a bit more flexible by actually dynamically picking @@ -109,8 +109,9 @@ export function useOrCreateAgent( if ( target !== undefined && !proxyForUrl(proxyOptions as DevtoolsProxyOptions, target) - ) + ) { return undefined; + } return createAgent(proxyOptions as DevtoolsProxyOptions); } } diff --git a/packages/devtools-proxy-support/src/ssh.ts b/packages/devtools-proxy-support/src/ssh.ts index b9b74871..381bc977 100644 --- a/packages/devtools-proxy-support/src/ssh.ts +++ b/packages/devtools-proxy-support/src/ssh.ts @@ -88,7 +88,7 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { throw err; }), (() => { - const waitForReady = once(this.sshClient, 'ready').then(() => {}); // eslint-disable-line @typescript-eslint/no-empty-function + const waitForReady = once(this.sshClient, 'ready').then(() => undefined); this.sshClient.connect(sshConnectConfig); return waitForReady; })(), From 7b7c41b66bc27929bb4f6b7a8da493b5a44d4ff8 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 31 Jul 2024 16:49:58 +0200 Subject: [PATCH 29/30] fixup: reformat after cr suggestions --- packages/devtools-proxy-support/src/ssh.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/devtools-proxy-support/src/ssh.ts b/packages/devtools-proxy-support/src/ssh.ts index 381bc977..feebc890 100644 --- a/packages/devtools-proxy-support/src/ssh.ts +++ b/packages/devtools-proxy-support/src/ssh.ts @@ -88,7 +88,9 @@ export class SSHAgent extends AgentBase implements AgentWithInitialize { throw err; }), (() => { - const waitForReady = once(this.sshClient, 'ready').then(() => undefined); + const waitForReady = once(this.sshClient, 'ready').then( + () => undefined + ); this.sshClient.connect(sshConnectConfig); return waitForReady; })(), From edc8694ad70e4345ee21daee115d83bcc74ff0b4 Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Wed, 31 Jul 2024 17:58:49 +0200 Subject: [PATCH 30/30] fixup: add comment --- packages/devtools-proxy-support/src/agent.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/devtools-proxy-support/src/agent.ts b/packages/devtools-proxy-support/src/agent.ts index 4d9f955b..fe943e1e 100644 --- a/packages/devtools-proxy-support/src/agent.ts +++ b/packages/devtools-proxy-support/src/agent.ts @@ -79,6 +79,8 @@ class DevtoolsProxyAgent extends ProxyAgent implements AgentWithInitialize { opts: AgentConnectOpts ): Promise { if (this.sshAgent) return this.sshAgent; + // Ensure that multiple concurrent invocations of connect() are processed + // sequentially until they reach _getProxyForUrl() each while (this._reqLock) { await this._reqLock; }