Re-apply 'Migrate request to got (part 1)'

core/base-service/base.js

@@ -20,6 +20,7 @@ import {
   Deprecated,
 } from './errors.js'
 import { validateExample, transformExample } from './examples.js'
+import { fetchFactory } from './got.js'
 import {
   makeFullUrl,
   assertValidRoute,
@@ -431,6 +432,8 @@ class BaseService {
       ServiceClass: this,
     })
 
+    const fetcher = fetchFactory(fetchLimitBytes)
+
     camp.route(
       regex,
       handleRequest(cacheHeaderConfig, {
@@ -441,7 +444,7 @@ class BaseService {
           const namedParams = namedParamsForMatch(captureNames, match, this)
           const serviceData = await this.invoke(
             {
-              sendAndCacheRequest: request.asPromise,
+              sendAndCacheRequest: fetcher,
               sendAndCacheRequestWithCallbacks: request,
               githubApiProvider,
               metricHelper,

core/base-service/got.js

@@ -0,0 +1,96 @@
+import got from 'got'
+import { Inaccessible, InvalidResponse } from './errors.js'
+
+const userAgent = 'Shields.io/2003a'
+
+function requestOptions2GotOptions(options) {
+  const requestOptions = Object.assign({}, options)
+  const gotOptions = {}
+  const interchangableOptions = ['body', 'form', 'headers', 'method', 'url']
+
+  interchangableOptions.forEach(function (opt) {
+    if (opt in requestOptions) {
+      gotOptions[opt] = requestOptions[opt]
+      delete requestOptions[opt]
+    }
+  })
+
+  if ('qs' in requestOptions) {
+    gotOptions.searchParams = requestOptions.qs
+    delete requestOptions.qs
+  }
+
+  if ('gzip' in requestOptions) {
+    gotOptions.decompress = requestOptions.gzip
+    delete requestOptions.gzip
+  }
+
+  if ('strictSSL' in requestOptions) {
+    gotOptions.https = {
+      rejectUnauthorized: requestOptions.strictSSL,
+    }
+    delete requestOptions.strictSSL
+  }
+
+  if ('auth' in requestOptions) {
+    gotOptions.username = requestOptions.auth.user
+    gotOptions.password = requestOptions.auth.pass
+    delete requestOptions.auth
+  }
+
+  if (Object.keys(requestOptions).length > 0) {
+    throw new Error(`Found unrecognised options ${Object.keys(requestOptions)}`)
+  }
+
+  return gotOptions
+}
+
+async function sendRequest(gotWrapper, url, options) {
+  const gotOptions = requestOptions2GotOptions(options)
+  gotOptions.throwHttpErrors = false
+  gotOptions.retry = 0
+  gotOptions.headers = gotOptions.headers || {}
+  gotOptions.headers['User-Agent'] = userAgent
+  try {
+    const resp = await gotWrapper(url, gotOptions)
+    return { res: resp, buffer: resp.body }
+  } catch (err) {
+    if (err instanceof got.CancelError) {
+      throw new InvalidResponse({
+        underlyingError: new Error('Maximum response size exceeded'),
+      })
+    }
+    throw new Inaccessible({ underlyingError: err })
+  }
+}
+
+function fetchFactory(fetchLimitBytes) {
+  const gotWithLimit = got.extend({
+    handlers: [
+      (options, next) => {
+        const promiseOrStream = next(options)
+        promiseOrStream.on('downloadProgress', progress => {
+          if (
+            progress.transferred > fetchLimitBytes &&
+            // just accept the file if we've already finished downloading
+            // the entire file before we went over the limit
+            progress.percent !== 1
+          ) {
+            /*
+            TODO: we should be able to pass cancel() a message
+            https://github.com/sindresorhus/got/blob/main/documentation/advanced-creation.md#examples
+            but by the time we catch it, err.message is just "Promise was canceled"
+            */
+            promiseOrStream.cancel('Maximum response size exceeded')
+          }
+        })
+
+        return promiseOrStream
+      },
+    ],
+  })
+
+  return sendRequest.bind(sendRequest, gotWithLimit)
+}
+
+export { requestOptions2GotOptions, fetchFactory }

core/base-service/got.spec.js

@@ -0,0 +1,102 @@
+import { expect } from 'chai'
+import nock from 'nock'
+import { requestOptions2GotOptions, fetchFactory } from './got.js'
+import { Inaccessible, InvalidResponse } from './errors.js'
+
+describe('requestOptions2GotOptions function', function () {
+  it('translates valid options', function () {
+    expect(
+      requestOptions2GotOptions({
+        body: 'body',
+        form: 'form',
+        headers: 'headers',
+        method: 'method',
+        url: 'url',
+        qs: 'qs',
+        gzip: 'gzip',
+        strictSSL: 'strictSSL',
+        auth: { user: 'user', pass: 'pass' },
+      })
+    ).to.deep.equal({
+      body: 'body',
+      form: 'form',
+      headers: 'headers',
+      method: 'method',
+      url: 'url',
+      searchParams: 'qs',
+      decompress: 'gzip',
+      https: { rejectUnauthorized: 'strictSSL' },
+      username: 'user',
+      password: 'pass',
+    })
+  })
+
+  it('throws if unrecognised options are found', function () {
+    expect(() =>
+      requestOptions2GotOptions({ body: 'body', foobar: 'foobar' })
+    ).to.throw(Error, 'Found unrecognised options foobar')
+  })
+})
+
+describe('got wrapper', function () {
+  it('should not throw an error if the response <= fetchLimitBytes', async function () {
+    nock('https://www.google.com')
+      .get('/foo/bar')
+      .once()
+      .reply(200, 'x'.repeat(100))
+    const sendRequest = fetchFactory(100)
+    const { res } = await sendRequest('https://www.google.com/foo/bar')
+    expect(res.statusCode).to.equal(200)
+  })
+
+  it('should throw an InvalidResponse error if the response is > fetchLimitBytes', async function () {
+    nock('https://www.google.com')
+      .get('/foo/bar')
+      .once()
+      .reply(200, 'x'.repeat(101))
+    const sendRequest = fetchFactory(100)
+    return expect(
+      sendRequest('https://www.google.com/foo/bar')
+    ).to.be.rejectedWith(InvalidResponse, 'Maximum response size exceeded')
+  })
+
+  it('should throw an Inaccessible error if the request throws a (non-HTTP) error', async function () {
+    nock('https://www.google.com').get('/foo/bar').replyWithError('oh no')
+    const sendRequest = fetchFactory(1024)
+    return expect(
+      sendRequest('https://www.google.com/foo/bar')
+    ).to.be.rejectedWith(Inaccessible, 'oh no')
+  })
+
+  it('should throw an Inaccessible error if the host can not be accessed', async function () {
+    this.timeout(5000)
+    nock.disableNetConnect()
+    const sendRequest = fetchFactory(1024)
+    return expect(
+      sendRequest('https://www.google.com/foo/bar')
+    ).to.be.rejectedWith(
+      Inaccessible,
+      'Nock: Disallowed net connect for "www.google.com:443/foo/bar"'
+    )
+  })
+
+  it('should pass a custom user agent header', async function () {
+    nock('https://www.google.com', {
+      reqheaders: {
+        'user-agent': function (agent) {
+          return agent.startsWith('Shields.io')
+        },
+      },
+    })
+      .get('/foo/bar')
+      .once()
+      .reply(200)
+    const sendRequest = fetchFactory(1024)
+    await sendRequest('https://www.google.com/foo/bar')
+  })
+
+  afterEach(function () {
+    nock.cleanAll()
+    nock.enableNetConnect()
+  })
+})

core/server/server.js

@@ -4,6 +4,7 @@
 
 import path from 'path'
 import url, { fileURLToPath } from 'url'
+import { bootstrap } from 'global-agent'
 import cloudflareMiddleware from 'cloudflare-middleware'
 import bytes from 'bytes'
 import Camp from '@shields_io/camp'
@@ -422,6 +423,25 @@ class Server {
     )
   }
 
+  bootstrapAgent() {
+    /*
+    Bootstrap global agent.
+    This allows self-hosting users to configure a proxy with
+    HTTP_PROXY, HTTPS_PROXY, NO_PROXY variables
+    */
+    if (!('GLOBAL_AGENT_ENVIRONMENT_VARIABLE_NAMESPACE' in process.env)) {
+      process.env.GLOBAL_AGENT_ENVIRONMENT_VARIABLE_NAMESPACE = ''
+    }
+
+    const proxyPrefix = process.env.GLOBAL_AGENT_ENVIRONMENT_VARIABLE_NAMESPACE
+    const HTTP_PROXY = process.env[`${proxyPrefix}HTTP_PROXY`] || null
+    const HTTPS_PROXY = process.env[`${proxyPrefix}HTTPS_PROXY`] || null
+
+    if (HTTP_PROXY || HTTPS_PROXY) {
+      bootstrap()
+    }
+  }
+
   /**
    * Start the HTTP server:
    * Bootstrap Scoutcamp,
@@ -436,6 +456,8 @@ class Server {
       requireCloudflare,
     } = this.config.public
 
+    this.bootstrapAgent()
+
     log.log(`Server is starting up: ${this.baseUrl}`)
 
     const camp = (this.camp = Camp.create({