fix: limit concurrent HTTP requests in browser (#2304)

Adds limit of concurrent HTTP requests sent to remote API
by dns and preload calls when running in web browser contexts.

Browsers limit connections per host (~6). This change mitigates the
problem of expensive and long running calls of one type exhausting
connection pool for other uses.

It additionally limits the number of DNS lookup calls by introducing
time-bound cache with eviction rules following what browser already do.

This is similar to:
libp2p/js-libp2p-delegated-content-routing#12

.aegir.js

@@ -10,7 +10,7 @@ const preloadNode = MockPreloadNode.createNode()
 const echoServer = EchoServer.createServer()
 
 module.exports = {
-  bundlesize: { maxSize: '689kB' },
+  bundlesize: { maxSize: '692kB' },
   webpack: {
     resolve: {
       mainFields: ['browser', 'main'],

package.json

@@ -19,7 +19,6 @@
     "./src/core/runtime/add-from-fs-nodejs.js": "./src/core/runtime/add-from-fs-browser.js",
     "./src/core/runtime/config-nodejs.js": "./src/core/runtime/config-browser.js",
     "./src/core/runtime/dns-nodejs.js": "./src/core/runtime/dns-browser.js",
-    "./src/core/runtime/fetch-nodejs.js": "./src/core/runtime/fetch-browser.js",
     "./src/core/runtime/libp2p-nodejs.js": "./src/core/runtime/libp2p-browser.js",
     "./src/core/runtime/libp2p-pubsub-routers-nodejs.js": "./src/core/runtime/libp2p-pubsub-routers-browser.js",
     "./src/core/runtime/preload-nodejs.js": "./src/core/runtime/preload-browser.js",
@@ -123,6 +122,8 @@
     "it-to-stream": "^0.1.1",
     "just-safe-set": "^2.1.0",
     "kind-of": "^6.0.2",
+    "ky": "~0.13.0",
+    "ky-universal": "~0.3.0",
     "libp2p": "~0.26.1",
     "libp2p-bootstrap": "~0.9.3",
     "libp2p-crypto": "~0.16.0",
@@ -151,7 +152,7 @@
     "multicodec": "~0.5.5",
     "multihashes": "~0.4.14",
     "multihashing-async": "~0.6.0",
-    "node-fetch": "^2.3.0",
+    "p-queue": "^6.1.0",
     "peer-book": "~0.9.0",
     "peer-id": "~0.12.3",
     "peer-info": "~0.15.0",

src/cli/commands/daemon.js

@@ -3,6 +3,7 @@
 const os = require('os')
 const toUri = require('multiaddr-to-uri')
 const { ipfsPathHelp } = require('../utils')
+const { isTest } = require('ipfs-utils/src/env')
 
 module.exports = {
   command: 'daemon',
@@ -27,7 +28,7 @@ module.exports = {
       })
       .option('enable-preload', {
         type: 'boolean',
-        default: true
+        default: !isTest // preload by default, unless in test env
       })
   },
 

src/core/components/files-regular/add-from-url.js

@@ -1,41 +1,22 @@
 'use strict'
 
 const { URL } = require('iso-url')
-const fetch = require('../../runtime/fetch-nodejs')
-
-module.exports = (self) => {
-  return async (url, options, callback) => {
-    if (typeof options === 'function') {
-      callback = options
-      options = {}
-    }
-
-    let files
-
-    try {
-      const parsedUrl = new URL(url)
-      const res = await fetch(url)
-
-      if (!res.ok) {
-        throw new Error('unexpected status code: ' + res.status)
-      }
-
-      // TODO: use res.body when supported
-      const content = Buffer.from(await res.arrayBuffer())
-      const path = decodeURIComponent(parsedUrl.pathname.split('/').pop())
-
-      files = await self.add({ content, path }, options)
-    } catch (err) {
-      if (callback) {
-        return callback(err)
-      }
-      throw err
-    }
+const nodeify = require('promise-nodeify')
+const { default: ky } = require('ky-universal')
+
+module.exports = (ipfs) => {
+  const addFromURL = async (url, opts = {}) => {
+    const res = await ky.get(url)
+    const path = decodeURIComponent(new URL(res.url).pathname.split('/').pop())
+    const content = Buffer.from(await res.arrayBuffer())
+    return ipfs.add({ content, path }, opts)
+  }
 
-    if (callback) {
-      callback(null, files)
+  return (name, opts = {}, cb) => {
+    if (typeof opts === 'function') {
+      cb = opts
+      opts = {}
     }
-
-    return files
+    return nodeify(addFromURL(name, opts), cb)
   }
 }

src/core/config.js

@@ -3,6 +3,7 @@
 const Multiaddr = require('multiaddr')
 const mafmt = require('mafmt')
 const { struct, superstruct } = require('superstruct')
+const { isTest } = require('ipfs-utils/src/env')
 
 const { optional, union } = struct
 const s = superstruct({
@@ -31,7 +32,7 @@ const configSchema = s({
     enabled: 'boolean?',
     addresses: optional(s(['multiaddr'])),
     interval: 'number?'
-  }, { enabled: true, interval: 30 * 1000 }),
+  }, { enabled: !isTest, interval: 30 * 1000 }),
   init: optional(union(['boolean', s({
     bits: 'number?',
     emptyRepo: 'boolean?',

src/core/index.js

@@ -27,6 +27,7 @@ const defaultRepo = require('./runtime/repo-nodejs')
 const preload = require('./preload')
 const mfsPreload = require('./mfs-preload')
 const ipldOptions = require('./runtime/ipld-nodejs')
+const { isTest } = require('ipfs-utils/src/env')
 
 /**
  * @typedef { import("./ipns/index") } IPNS
@@ -47,7 +48,7 @@ class IPFS extends EventEmitter {
       start: true,
       EXPERIMENTAL: {},
       preload: {
-        enabled: true,
+        enabled: !isTest, // preload by default, unless in test env
         addresses: [
           '/dnsaddr/node0.preload.ipfs.io/https',
           '/dnsaddr/node1.preload.ipfs.io/https'

src/core/runtime/dns-browser.js

@@ -1,33 +1,62 @@
-/* global self */
+/* eslint-env browser */
 'use strict'
 
-module.exports = (domain, opts, callback) => {
+const TLRU = require('../../utils/tlru')
+const { default: PQueue } = require('p-queue')
+const { default: ky } = require('ky-universal')
+const nodeify = require('promise-nodeify')
+
+// Avoid sending multiple queries for the same hostname by caching results
+const cache = new TLRU(1000)
+// TODO: /api/v0/dns does not return TTL yet: https://github.com/ipfs/go-ipfs/issues/5884
+// However we know browsers themselves cache DNS records for at least 1 minute,
+// which acts a provisional default ttl: https://stackoverflow.com/a/36917902/11518426
+const ttl = 60 * 1000
+
+// browsers limit concurrent connections per host,
+// we don't want preload calls to exhaust the limit (~6)
+const httpQueue = new PQueue({ concurrency: 4 })
+
+// Delegated HTTP resolver sending DNSLink queries to ipfs.io
+// TODO: replace hardcoded host with configurable DNS over HTTPS: https://github.com/ipfs/js-ipfs/issues/2212
+const api = ky.create({
+  prefixUrl: 'https://ipfs.io/api/v0/',
+  hooks: {
+    afterResponse: [
+      async (input, options, response) => {
+        const query = new URL(response.url).search.slice(1)
+        const json = await response.json()
+        cache.set(query, json, ttl)
+      }
+    ]
+  }
+})
+
+const ipfsPath = (response) => {
+  if (response.Path) return response.Path
+  throw new Error(response.Message)
+}
+
+module.exports = (fqdn, opts = {}, cb) => {
   if (typeof opts === 'function') {
-    callback = opts
+    cb = opts
     opts = {}
   }
+  const resolveDnslink = async (fqdn, opts = {}) => {
+    const searchParams = new URLSearchParams(opts)
+    searchParams.set('arg', fqdn)
 
-  opts = opts || {}
-
-  domain = encodeURIComponent(domain)
-  let url = `https://ipfs.io/api/v0/dns?arg=${domain}`
+    // try cache first
+    const query = searchParams.toString()
+    if (!opts.nocache && cache.has(query)) {
+      const response = cache.get(query)
+      return ipfsPath(response)
+    }
 
-  Object.keys(opts).forEach(prop => {
-    url += `&${encodeURIComponent(prop)}=${encodeURIComponent(opts[prop])}`
-  })
+    // fallback to delegated DNS resolver
+    const response = await httpQueue.add(() => api.get('dns', { searchParams }).json())
+    return ipfsPath(response)
+  }
 
-  self.fetch(url, { mode: 'cors' })
-    .then((response) => {
-      return response.json()
-    })
-    .then((response) => {
-      if (response.Path) {
-        return callback(null, response.Path)
-      } else {
-        return callback(new Error(response.Message))
-      }
-    })
-    .catch((error) => {
-      callback(error)
-    })
+  return nodeify(resolveDnslink(fqdn, opts), cb)
 }

src/core/runtime/preload-browser.js

@@ -1,27 +1,25 @@
 /* eslint-env browser */
 'use strict'
 
+const { default: PQueue } = require('p-queue')
+const { default: ky } = require('ky-universal')
 const debug = require('debug')
 
 const log = debug('ipfs:preload')
 log.error = debug('ipfs:preload:error')
 
+// browsers limit concurrent connections per host,
+// we don't want preload calls to exhaust the limit (~6)
+const httpQueue = new PQueue({ concurrency: 4 })
+
 module.exports = function preload (url, callback) {
   log(url)
 
   const controller = new AbortController()
   const signal = controller.signal
+  const cb = () => setImmediate(callback) // https://github.com/ipfs/js-ipfs/pull/2304#discussion_r320700893
 
-  fetch(url, { signal })
-    .then(res => {
-      if (!res.ok) {
-        log.error('failed to preload', url, res.status, res.statusText)
-        throw new Error(`failed to preload ${url}`)
-      }
-      return res.text()
-    })
-    .then(() => callback())
-    .catch(callback)
+  httpQueue.add(() => ky.get(url, { signal })).then(cb, cb)
 
   return {
     cancel: () => controller.abort()

src/utils/tlru.js

@@ -33,8 +33,9 @@ class TLRU {
         this.lru.remove(key)
         return undefined
       }
+      return value.value
     }
-    return value.value
+    return undefined
   }
 
   /**

test/core/interface.spec.js

@@ -64,9 +64,6 @@ describe('interface-ipfs-core tests', function () {
     }, {
       name: 'addFromFs',
       reason: 'Not designed to run in the browser'
-    }, {
-      name: 'addFromURL',
-      reason: 'Not designed to run in the browser'
     }]
   })