diff --git a/README.md b/README.md index bd7b905d..e94a67ae 100644 --- a/README.md +++ b/README.md @@ -41,8 +41,8 @@ over other authentication methods, i.e., application default credentials. ## Request Options -```js -{ +```ts +interface GaxiosOptions = { // The url to which the request should be sent. Required. url: string, @@ -155,6 +155,39 @@ over other authentication methods, i.e., application default credentials. // See https://github.com/bitinn/node-fetch#request-cancellation-with-abortsignal signal?: AbortSignal + /** + * A collection of parts to send as a `Content-Type: multipart/related` request. + */ + multipart?: GaxiosMultipartOptions; + + /** + * An optional proxy to use for requests. + * Available via `process.env.HTTP_PROXY` and `process.env.HTTPS_PROXY` as well - with a preference for the this config option when multiple are available. + * The `agent` option overrides this. + * + * @see {@link GaxiosOptions.noProxy} + * @see {@link GaxiosOptions.agent} + */ + proxy?: string | URL; + /** + * A list for excluding traffic for proxies. + * Available via `process.env.NO_PROXY` as well as a common-separated list of strings - merged with any local `noProxy` rules. + * + * - When provided a string, it is matched by + * - Wildcard `*.` and `.` matching are available. (e.g. `.example.com` or `*.example.com`) + * - When provided a URL, it is matched by the `.origin` property. + * - For example, requesting `https://example.com` with the following `noProxy`s would result in a no proxy use: + * - new URL('https://example.com') + * - new URL('https://example.com:443') + * - The following would be used with a proxy: + * - new URL('http://example.com:80') + * - new URL('https://example.com:8443') + * - When provided a regular expression it is used to match the stringified URL + * + * @see {@link GaxiosOptions.proxy} + */ + noProxy?: (string | URL | RegExp)[]; + /** * An experimental, customizable error redactor. * diff --git a/src/common.ts b/src/common.ts index 7c35d2c3..38779874 100644 --- a/src/common.ts +++ b/src/common.ts @@ -181,6 +181,9 @@ export interface GaxiosOptions { */ maxRedirects?: number; follow?: number; + /** + * A collection of parts to send as a `Content-Type: multipart/related` request. + */ multipart?: GaxiosMultipartOptions[]; params?: any; paramsSerializer?: (params: {[index: string]: string | number}) => string; @@ -213,6 +216,35 @@ export interface GaxiosOptions { // Configure client to use mTLS: cert?: string; key?: string; + + /** + * An optional proxy to use for requests. + * Available via `process.env.HTTP_PROXY` and `process.env.HTTPS_PROXY` as well - with a preference for the this config option when multiple are available. + * The {@link GaxiosOptions.agent `agent`} option overrides this. + * + * @see {@link GaxiosOptions.noProxy} + * @see {@link GaxiosOptions.agent} + */ + proxy?: string | URL; + /** + * A list for excluding traffic for proxies. + * Available via `process.env.NO_PROXY` as well as a common-separated list of strings - merged with any local `noProxy` rules. + * + * - When provided a string, it is matched by + * - Wildcard `*.` and `.` matching are available. (e.g. `.example.com` or `*.example.com`) + * - When provided a URL, it is matched by the `.origin` property. + * - For example, requesting `https://example.com` with the following `noProxy`s would result in a no proxy use: + * - new URL('https://example.com') + * - new URL('https://example.com:443') + * - The following would be used with a proxy: + * - new URL('http://example.com:80') + * - new URL('https://example.com:8443') + * - When provided a regular expression it is used to match the stringified URL + * + * @see {@link GaxiosOptions.proxy} + */ + noProxy?: (string | URL | RegExp)[]; + /** * An experimental error redactor. * diff --git a/src/gaxios.ts b/src/gaxios.ts index be9ffb0d..a0daeec5 100644 --- a/src/gaxios.ts +++ b/src/gaxios.ts @@ -31,7 +31,6 @@ import { } from './common'; import {getRetryConfig} from './retry'; import {PassThrough, Stream, pipeline} from 'stream'; -import {HttpsProxyAgent as httpsProxyAgent} from 'https-proxy-agent'; import {v4} from 'uuid'; /* eslint-disable @typescript-eslint/no-explicit-any */ @@ -64,54 +63,11 @@ function getHeader(options: GaxiosOptions, header: string): string | undefined { return undefined; } -let HttpsProxyAgent: any; - -function loadProxy() { - const proxy = - process?.env?.HTTPS_PROXY || - process?.env?.https_proxy || - process?.env?.HTTP_PROXY || - process?.env?.http_proxy; - if (proxy) { - HttpsProxyAgent = httpsProxyAgent; - } - - return proxy; -} - -loadProxy(); - -function skipProxy(url: string | URL) { - const noProxyEnv = process.env.NO_PROXY ?? process.env.no_proxy; - if (!noProxyEnv) { - return false; - } - const noProxyUrls = noProxyEnv.split(','); - const parsedURL = url instanceof URL ? url : new URL(url); - return !!noProxyUrls.find(url => { - if (url.startsWith('*.') || url.startsWith('.')) { - url = url.replace(/^\*\./, '.'); - return parsedURL.hostname.endsWith(url); - } else { - return url === parsedURL.origin || url === parsedURL.hostname; - } - }); -} - -// Figure out if we should be using a proxy. Only if it's required, load -// the https-proxy-agent module as it adds startup cost. -function getProxy(url: string | URL) { - // If there is a match between the no_proxy env variables and the url, then do not proxy - if (skipProxy(url)) { - return undefined; - // If there is not a match between the no_proxy env variables and the url, check to see if there should be a proxy - } else { - return loadProxy(); - } -} - export class Gaxios { - protected agentCache = new Map Agent)>(); + protected agentCache = new Map< + string | URL, + Agent | ((parsedUrl: URL) => Agent) + >(); /** * Default HTTP options that will be used for every HTTP request. @@ -131,7 +87,7 @@ export class Gaxios { * @param opts Set of HTTP options that will be used for this HTTP request. */ async request(opts: GaxiosOptions = {}): GaxiosPromise { - opts = this.validateOpts(opts); + opts = await this.#prepareRequest(opts); return this._request(opts); } @@ -139,7 +95,7 @@ export class Gaxios { opts: GaxiosOptions ): Promise> { const fetchImpl = opts.fetchImplementation || fetch; - const res = (await fetchImpl(opts.url!, opts)) as FetchResponse; + const res = (await fetchImpl(opts.url, opts)) as FetchResponse; const data = await this.getResponseData(opts, res); return this.translateResponse(opts, res, data); } @@ -228,11 +184,59 @@ export class Gaxios { } } + #urlMayUseProxy( + url: string | URL, + noProxy: GaxiosOptions['noProxy'] = [] + ): boolean { + const candidate = new URL(url); + const noProxyList = [...noProxy]; + const noProxyEnvList = + (process.env.NO_PROXY ?? process.env.no_proxy)?.split(',') || []; + + for (const rule of noProxyEnvList) { + noProxyList.push(rule.trim()); + } + + for (const rule of noProxyList) { + // Match regex + if (rule instanceof RegExp) { + if (rule.test(candidate.toString())) { + return false; + } + } + // Match URL + else if (rule instanceof URL) { + if (rule.origin === candidate.origin) { + return false; + } + } + // Match string regex + else if (rule.startsWith('*.') || rule.startsWith('.')) { + const cleanedRule = rule.replace(/^\*\./, '.'); + if (candidate.hostname.endsWith(cleanedRule)) { + return false; + } + } + // Basic string match + else if ( + rule === candidate.origin || + rule === candidate.hostname || + rule === candidate.href + ) { + return false; + } + } + + return true; + } + /** - * Validates the options, and merges them with defaults. - * @param opts The original options passed from the client. + * Validates the options, merges them with defaults, and prepare request. + * + * @param options The original options passed from the client. + * @returns Prepared options, ready to make a request */ - private validateOpts(options: GaxiosOptions): GaxiosOptions { + async #prepareRequest(options: GaxiosOptions): Promise { const opts = extend(true, {}, this.defaults, options); if (!opts.url) { throw new Error('URL is required.'); @@ -318,28 +322,31 @@ export class Gaxios { } opts.method = opts.method || 'GET'; - const proxy = getProxy(opts.url); - if (proxy) { + const proxy = + opts.proxy || + process?.env?.HTTPS_PROXY || + process?.env?.https_proxy || + process?.env?.HTTP_PROXY || + process?.env?.http_proxy; + const urlMayUseProxy = this.#urlMayUseProxy(opts.url, opts.noProxy); + + if (opts.agent) { + // don't do any of the following options - use the user-provided agent. + } else if (proxy && urlMayUseProxy) { + const HttpsProxyAgent = await Gaxios.#getProxyAgent(); + if (this.agentCache.has(proxy)) { opts.agent = this.agentCache.get(proxy); } else { - // Proxy is being used in conjunction with mTLS. - if (opts.cert && opts.key) { - const parsedURL = new URL(proxy); - opts.agent = new HttpsProxyAgent({ - port: parsedURL.port, - host: parsedURL.host, - protocol: parsedURL.protocol, - cert: opts.cert, - key: opts.key, - }); - } else { - opts.agent = new HttpsProxyAgent(proxy); - } - this.agentCache.set(proxy, opts.agent!); + opts.agent = new HttpsProxyAgent(proxy, { + cert: opts.cert, + key: opts.key, + }); + + this.agentCache.set(proxy, opts.agent); } } else if (opts.cert && opts.key) { - // Configure client for mTLS: + // Configure client for mTLS if (this.agentCache.has(opts.key)) { opts.agent = this.agentCache.get(opts.key); } else { @@ -347,7 +354,7 @@ export class Gaxios { cert: opts.cert, key: opts.key, }); - this.agentCache.set(opts.key, opts.agent!); + this.agentCache.set(opts.key, opts.agent); } } @@ -459,4 +466,23 @@ export class Gaxios { } yield finale; } + + /** + * A cache for the lazily-loaded proxy agent. + * + * Should use {@link Gaxios[#getProxyAgent]} to retrieve. + */ + // using `import` to dynamically import the types here + static #proxyAgent?: typeof import('https-proxy-agent').HttpsProxyAgent; + + /** + * Imports, caches, and returns a proxy agent - if not already imported + * + * @returns A proxy agent + */ + static async #getProxyAgent() { + this.#proxyAgent ||= (await import('https-proxy-agent')).HttpsProxyAgent; + + return this.#proxyAgent; + } } diff --git a/test/test.getch.ts b/test/test.getch.ts index 02b8d3c9..353ac961 100644 --- a/test/test.getch.ts +++ b/test/test.getch.ts @@ -43,6 +43,10 @@ afterEach(() => { const url = 'https://example.com'; +function setEnv(obj: {}) { + return sandbox.stub(process, 'env').value(obj); +} + describe('🦖 option validation', () => { it('should throw an error if a url is not provided', async () => { await assert.rejects(request({}), /URL is required/); @@ -337,125 +341,258 @@ describe('🥁 configuration options', () => { }); describe('proxying', () => { - it('should use an https proxy if asked nicely', async () => { - const url = 'https://fake.proxy'; - sandbox.stub(process, 'env').value({https_proxy: 'https://fake.proxy'}); - const body = {hello: '🌎'}; - const scope = nock(url).get('/').reply(200, body); - const res = await request({url}); - scope.done(); - assert.deepStrictEqual(res.data, body); - assert.ok(res.config.agent instanceof HttpsProxyAgent); + const url = 'https://domain.example.com/with-path'; + const proxy = 'https://fake.proxy/'; + let gaxios: Gaxios; + let request: Gaxios['request']; + let responseBody: {}; + let scope: nock.Scope; + + beforeEach(() => { + gaxios = new Gaxios(); + request = gaxios.request.bind(gaxios); + responseBody = {hello: '🌎'}; + + const direct = new URL(url); + scope = nock(direct.origin).get(direct.pathname).reply(200, responseBody); }); - it('should not proxy when url matches no_proxy', async () => { - const url = 'https://example.com'; - sandbox.stub(process, 'env').value({ - https_proxy: 'https://fake.proxy', - no_proxy: 'https://example.com', - }); - const body = {hello: '🌎'}; - const scope = nock(url).get('/').reply(200, body); - const res = await request({url}); + function expectDirect(res: GaxiosResponse) { scope.done(); - assert.deepStrictEqual(res.data, body); + assert.deepStrictEqual(res.data, responseBody); assert.strictEqual(res.config.agent, undefined); - }); + } - it('should proxy if url does not match no_proxy env variable', async () => { - const url = 'https://example2.com'; - sandbox.stub(process, 'env').value({ - https_proxy: 'https://fake.proxy', - no_proxy: 'https://example.com', - }); - const body = {hello: '🌎'}; - const scope = nock(url).get('/').reply(200, body); - const res = await request({url}); + function expectProxy(res: GaxiosResponse) { scope.done(); - assert.deepStrictEqual(res.data, body); + assert.deepStrictEqual(res.data, responseBody); assert.ok(res.config.agent instanceof HttpsProxyAgent); + assert.equal(res.config.agent.proxy.toString(), proxy); + } + + it('should use an https proxy if asked nicely (config)', async () => { + const res = await request({url, proxy}); + expectProxy(res); }); - it('should not proxy if no_proxy env var matches the origin or hostname of the URL', async () => { - const url = 'https://example2.com'; - sandbox.stub(process, 'env').value({ - https_proxy: 'https://fake.proxy', - no_proxy: 'example2.com', - }); - const body = {hello: '🌎'}; - const scope = nock(url).get('/').reply(200, body); + it('should use an https proxy if asked nicely (env)', async () => { + setEnv({https_proxy: proxy}); + const res = await request({url}); - scope.done(); - assert.deepStrictEqual(res.data, body); - assert.strictEqual(res.config.agent, undefined); + expectProxy(res); }); - it('should not proxy if no_proxy env variable has asterisk, and URL partially matches', async () => { - const url = 'https://domain.example.com'; - sandbox.stub(process, 'env').value({ - https_proxy: 'https://fake.proxy', - no_proxy: '*.example.com', - }); - const body = {hello: '🌎'}; - const scope = nock(url).get('/').reply(200, body); - const res = await request({url}); - scope.done(); - assert.deepStrictEqual(res.data, body); - assert.strictEqual(res.config.agent, undefined); + it('should use mTLS with proxy', async () => { + const cert = 'cert'; + const key = 'key'; + const res = await request({url, proxy, cert, key}); + expectProxy(res); + + assert(res.config.agent instanceof HttpsProxyAgent); + assert.equal(res.config.agent.connectOpts.cert, cert); + assert.equal(res.config.agent.connectOpts.key, key); }); - it('should proxy if no_proxy env variable has asterisk, but URL is not matching', async () => { - const url = 'https://domain.example2.com'; - sandbox.stub(process, 'env').value({ - https_proxy: 'https://fake.proxy', - no_proxy: '*.example.com', - }); - const body = {hello: '🌎'}; - const scope = nock(url).get('/').reply(200, body); - const res = await request({url}); - scope.done(); - assert.deepStrictEqual(res.data, body); - assert.ok(res.config.agent instanceof HttpsProxyAgent); + it('should load the proxy from the cache', async () => { + const res1 = await request({url, proxy}); + const agent = res1.config.agent; + expectProxy(res1); + + const direct = new URL(url); + + scope = nock(direct.origin).get(direct.pathname).reply(200, responseBody); + + const res2 = await request({url, proxy}); + assert.strictEqual(agent, res2.config.agent); + expectProxy(res2); }); - it('should not proxy if no_proxy env variable starts with a dot, and URL partially matches', async () => { - const url = 'https://domain.example.com'; - sandbox.stub(process, 'env').value({ - https_proxy: 'https://fake.proxy', - no_proxy: '.example.com', - }); - const body = {hello: '🌎'}; - const scope = nock(url).get('/').reply(200, body); - const res = await request({url}); - scope.done(); - assert.deepStrictEqual(res.data, body); - assert.strictEqual(res.config.agent, undefined); + it('should load the proxy from the cache with mTLS', async () => { + const cert = 'cert'; + const key = 'key'; + const res1 = await request({url, proxy, cert, key}); + + const agent = res1.config.agent; + expectProxy(res1); + + const direct = new URL(url); + + scope = nock(direct.origin).get(direct.pathname).reply(200, responseBody); + + const res2 = await request({url, proxy}); + assert.strictEqual(agent, res2.config.agent); + expectProxy(res2); + + assert(res2.config.agent instanceof HttpsProxyAgent); + assert.equal(res2.config.agent.connectOpts.cert, cert); + assert.equal(res2.config.agent.connectOpts.key, key); }); - it('should allow comma-separated lists for no_proxy env variables', async () => { - const url = 'https://api.google.com'; - sandbox.stub(process, 'env').value({ - https_proxy: 'https://fake.proxy', - no_proxy: 'example.com,*.google.com,hello.com', + describe('noProxy', () => { + it('should not proxy when url matches `noProxy` (config > string)', async () => { + const noProxy = [new URL(url).host]; + + const res = await request({url, proxy, noProxy}); + expectDirect(res); }); - const body = {hello: '🌎'}; - const scope = nock(url).get('/').reply(200, body); - const res = await request({url}); - scope.done(); - assert.deepStrictEqual(res.data, body); - assert.strictEqual(res.config.agent, undefined); - }); - }); - it('should load the proxy from the cache', async () => { - sandbox.stub(process, 'env').value({HTTPS_PROXY: 'https://fake.proxy'}); - const body = {hello: '🌎'}; - const scope = nock(url).get('/').twice().reply(200, body); - const res1 = await request({url}); - const agent = res1.config.agent; - const res2 = await request({url}); - assert.deepStrictEqual(agent, res2.config.agent); - scope.done(); + it('should not proxy when url matches `noProxy` (config > URL)', async () => { + // should match by `URL#origin` + const noProxyURL = new URL(url); + noProxyURL.pathname = '/some-other-path'; + const noProxy = [noProxyURL]; + + const res = await request({url, proxy, noProxy}); + expectDirect(res); + }); + + it('should not proxy when url matches `noProxy` (config > RegExp)', async () => { + const noProxy = [/example.com/]; + + const res = await request({url, proxy, noProxy}); + expectDirect(res); + }); + + it('should not proxy when url matches `noProxy` (config + env > match config)', async () => { + const noProxy = [url]; + setEnv({no_proxy: 'https://foo.bar'}); + + const res = await request({url, proxy, noProxy}); + expectDirect(res); + }); + + it('should not proxy when url matches `noProxy` (config + env > match env)', async () => { + const noProxy = ['https://foo.bar']; + setEnv({no_proxy: url}); + + const res = await request({url, proxy, noProxy}); + expectDirect(res); + }); + + it('should proxy when url does not match `noProxy` (config > string)', async () => { + const noProxy = [url]; + + const res = await request({url, proxy, noProxy}); + expectDirect(res); + }); + + it('should proxy if url does not match `noProxy` (config > URL > diff origin > protocol)', async () => { + const noProxyURL = new URL(url); + noProxyURL.protocol = 'http:'; + const noProxy = [noProxyURL]; + + const res = await request({url, proxy, noProxy}); + expectProxy(res); + }); + + it('should proxy if url does not match `noProxy` (config > URL > diff origin > port)', async () => { + const noProxyURL = new URL(url); + noProxyURL.port = '8443'; + const noProxy = [noProxyURL]; + + const res = await request({url, proxy, noProxy}); + expectProxy(res); + }); + + it('should proxy if url does not match `noProxy` (env)', async () => { + setEnv({https_proxy: proxy, no_proxy: 'https://blah'}); + + const res = await request({url}); + expectProxy(res); + }); + + it('should not proxy if `noProxy` env var matches the origin or hostname of the URL (config > string)', async () => { + const noProxy = [new URL(url).hostname]; + + const res = await request({url, proxy, noProxy}); + expectDirect(res); + }); + + it('should not proxy if `noProxy` env var matches the origin or hostname of the URL (env)', async () => { + setEnv({https_proxy: proxy, no_proxy: new URL(url).hostname}); + + const res = await request({url}); + expectDirect(res); + }); + + it('should not proxy if `noProxy` env variable has asterisk, and URL partially matches (config)', async () => { + const parentHost = new URL(url).hostname.split('.').slice(1).join('.'); + // ensure we have a host for a valid test + assert(parentHost); + const noProxy = [`*.${parentHost}`]; + + const res = await request({url, proxy, noProxy}); + expectDirect(res); + }); + + it('should not proxy if `noProxy` env variable has asterisk, and URL partially matches (env)', async () => { + const parentHost = new URL(url).hostname.split('.').slice(1).join('.'); + // ensure we have a host for a valid test + assert(parentHost); + setEnv({https_proxy: proxy, no_proxy: `*.${parentHost}`}); + + const res = await request({url}); + expectDirect(res); + }); + + it('should not proxy if `noProxy` env variable starts with a dot, and URL partially matches (config)', async () => { + const parentHost = new URL(url).hostname.split('.').slice(1).join('.'); + // ensure we have a host for a valid test + assert(parentHost); + const noProxy = [`.${parentHost}`]; + + const res = await request({url, proxy, noProxy}); + expectDirect(res); + }); + + it('should not proxy if `noProxy` env variable starts with a dot, and URL partially matches (env)', async () => { + const parentHost = new URL(url).hostname.split('.').slice(1).join('.'); + // ensure we have a host for a valid test + assert(parentHost); + + setEnv({https_proxy: proxy, no_proxy: '.example.com'}); + + const res = await request({url}); + expectDirect(res); + }); + + it('should proxy if `noProxy` env variable has asterisk, but URL is not matching (config)', async () => { + const noProxy = ['*.no.match']; + + const res = await request({url, proxy, noProxy}); + expectProxy(res); + }); + + it('should proxy if `noProxy` env variable has asterisk, but URL is not matching (env)', async () => { + setEnv({https_proxy: proxy, no_proxy: '*.no.match'}); + + const res = await request({url}); + expectProxy(res); + }); + + it('should allow comma-separated lists for `noProxy` env variables (config)', async () => { + const parentHost = new URL(url).hostname.split('.').slice(1).join('.'); + // ensure we have a host for a valid test + assert(parentHost); + + const noProxy = ['google.com', `*.${parentHost}`, 'hello.com']; + + const res = await request({url, proxy, noProxy}); + expectDirect(res); + }); + + it('should allow comma-separated lists for `noProxy` env variables (env)', async () => { + const parentHost = new URL(url).hostname.split('.').slice(1).join('.'); + // ensure we have a host for a valid test + assert(parentHost); + // added spaces to ensure trimming works as expected + const noProxy = [' google.com ', ` *.${parentHost} `, ' hello.com ']; + setEnv({https_proxy: proxy, no_proxy: noProxy.join(',')}); + + const res = await request({url}); + expectDirect(res); + }); + }); }); it('should include the request data in the response config', async () => {