feat: preload content (#1464)

refs #1459

This PR adds a new config property `preload`:

```js
new IPFS({
  preload: {
    enabled: false,
    addresses: ['/multiaddr/api/address']
  }
})
```

* `preload.enabled` (default `false`) enables/disabled preloading - **should the default be false?**
* `preload.addresses` array of node API addresses to preload content on. This are the addresses we make a `/api/v0/refs?arg=QmHash` request to, to initiate the preload

**This PR upgrades the following APIs to preload content**. After adding content with `ipfs.files.add` (for example), we make a request to the first preload gateway addresses (providing `preload.enabled` is true), and will fall back to the second etc.

* [x] `dag.put`
* [x] `block.put`
* [x] `object.new`
* [x] `object.put`
* [x] `object.patch.*`
* [x] `mfs.*`

MFS preloading is slightly different - we periodically submit your MFS root to the preload nodes when it changes.

NOTE: this PR adds an option to `dag`, `block` and `object` APIs allowing users to opt out of preloading by specifying `preload: false` in their options object.

License: MIT
Signed-off-by: Alan Shaw <[email protected]>

.aegir.js

@@ -1,8 +1,11 @@
 'use strict'
 
-const createServer = require('ipfsd-ctl').createServer
+const IPFSFactory = require('ipfsd-ctl')
+const parallel = require('async/parallel')
+const MockPreloadNode = require('./test/utils/mock-preload-node')
 
-const server = createServer()
+const ipfsdServer = IPFSFactory.createServer()
+const preloadNode = MockPreloadNode.createNode()
 
 module.exports = {
   webpack: {
@@ -21,9 +24,29 @@ module.exports = {
     singleRun: true
   },
   hooks: {
+    node: {
+      pre: (cb) => preloadNode.start(cb),
+      post: (cb) => preloadNode.stop(cb)
+    },
     browser: {
-      pre: server.start.bind(server),
-      post: server.stop.bind(server)
+      pre: (cb) => {
+        parallel([
+          (cb) => {
+            ipfsdServer.start()
+            cb()
+          },
+          (cb) => preloadNode.start(cb)
+        ], cb)
+      },
+      post: (cb) => {
+        parallel([
+          (cb) => {
+            ipfsdServer.stop()
+            cb()
+          },
+          (cb) => preloadNode.stop(cb)
+        ], cb)
+      }
     }
   }
 }

README.md

@@ -231,6 +231,9 @@ Creates and returns an instance of an IPFS node. Use the `options` argument to s
         - `enabled` (boolean): Make this node a relay (other nodes can connect *through* it). (Default: `false`)
         - `active` (boolean): Make this an *active* relay node. Active relay nodes will attempt to dial a destination peer even if that peer is not yet connected to the relay. (Default: `false`)
 
+- `preload` (object): Configure external nodes that will preload content added to this node
+    - `enabled` (boolean): Enable content preloading (Default: `true`)
+    - `addresses` (array): Multiaddr API addresses of nodes that should preload content. NOTE: nodes specified here should also be added to your node's bootstrap address list at `config.Boostrap`
 - `EXPERIMENTAL` (object): Enable and configure experimental features.
     - `pubsub` (boolean): Enable libp2p pub-sub. (Default: `false`)
     - `sharding` (boolean): Enable directory sharding. Directories that have many child objects will be represented by multiple DAG nodes instead of just one. It can improve lookup performance when a directory has several thousand files or more. (Default: `false`)

package.json

@@ -11,6 +11,7 @@
     "./src/core/components/init-assets.js": false,
     "./src/core/runtime/config-nodejs.js": "./src/core/runtime/config-browser.js",
     "./src/core/runtime/libp2p-nodejs.js": "./src/core/runtime/libp2p-browser.js",
+    "./src/core/runtime/preload-nodejs.js": "./src/core/runtime/preload-browser.js",
     "./src/core/runtime/repo-nodejs.js": "./src/core/runtime/repo-browser.js",
     "./src/core/runtime/dns-nodejs.js": "./src/core/runtime/dns-browser.js",
     "./test/utils/create-repo-nodejs.js": "./test/utils/create-repo-browser.js",
@@ -140,6 +141,7 @@
     "mime-types": "^2.1.18",
     "mkdirp": "~0.5.1",
     "multiaddr": "^5.0.0",
+    "multiaddr-to-uri": "^4.0.0",
     "multibase": "~0.4.0",
     "multihashes": "~0.4.13",
     "once": "^1.4.0",

src/core/components/block.js

@@ -52,6 +52,11 @@ module.exports = function block (self) {
           if (err) {
             return cb(err)
           }
+
+          if (options.preload !== false) {
+            self._preload(block.cid)
+          }
+
           cb(null, block)
         })
       ], callback)

src/core/components/dag.js

@@ -24,7 +24,15 @@ module.exports = function dag (self) {
 
       options = options.cid ? options : Object.assign({}, optionDefaults, options)
 
-      self._ipld.put(dagNode, options, callback)
+      self._ipld.put(dagNode, options, (err, cid) => {
+        if (err) return callback(err)
+
+        if (options.preload !== false) {
+          self._preload(cid)
+        }
+
+        callback(null, cid)
+      })
     }),
 
     get: promisify((cid, path, options, callback) => {

src/core/components/files.js

@@ -89,6 +89,20 @@ function normalizeContent (opts, content) {
   })
 }
 
+function preloadFile (self, opts, file) {
+  const isRootFile = opts.wrapWithDirectory
+    ? file.path === ''
+    : !file.path.includes('/')
+
+  const shouldPreload = isRootFile && !opts.onlyHash && opts.preload !== false
+
+  if (shouldPreload) {
+    self._preload(file.hash)
+  }
+
+  return file
+}
+
 function pinFile (self, opts, file, cb) {
   // Pin a file if it is the root dir of a recursive add or the single file
   // of a direct add.
@@ -158,6 +172,7 @@ module.exports = function files (self) {
       pull.flatten(),
       importer(self._ipld, opts),
       pull.asyncMap(prepareFile.bind(null, self, opts)),
+      pull.map(preloadFile.bind(null, self, opts)),
       pull.asyncMap(pinFile.bind(null, self, opts))
     )
   }

src/core/components/index.js

@@ -26,4 +26,4 @@ exports.dht = require('./dht')
 exports.dns = require('./dns')
 exports.key = require('./key')
 exports.stats = require('./stats')
-exports.mfs = require('ipfs-mfs/core')
+exports.mfs = require('./mfs')

src/core/components/mfs.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const promisify = require('promisify-es6')
+const mfs = require('ipfs-mfs/core')
+
+module.exports = self => {
+  const mfsSelf = Object.assign({}, self)
+
+  // A patched dag API to ensure preload doesn't happen for MFS operations
+  mfsSelf.dag = Object.assign({}, self.dag, {
+    put: promisify((node, opts, cb) => {
+      if (typeof opts === 'function') {
+        cb = opts
+        opts = {}
+      }
+
+      opts = Object.assign({}, opts, { preload: false })
+
+      return self.dag.put(node, opts, cb)
+    })
+  })
+
+  return mfs(mfsSelf, mfsSelf._options)
+}

src/core/components/object.js

@@ -80,10 +80,17 @@ module.exports = function object (self) {
             if (err) {
               return cb(err)
             }
-            self._ipld.put(node, {
-              cid: new CID(node.multihash)
-            }, (err) => {
-              cb(err, node)
+
+            const cid = new CID(node.multihash)
+
+            self._ipld.put(node, { cid }, (err) => {
+              if (err) return cb(err)
+
+              if (options.preload !== false) {
+                self._preload(cid)
+              }
+
+              cb(null, node)
             })
           })
         }
@@ -92,12 +99,20 @@ module.exports = function object (self) {
   }
 
   return {
-    new: promisify((template, callback) => {
+    new: promisify((template, options, callback) => {
       if (typeof template === 'function') {
         callback = template
         template = undefined
+        options = {}
+      }
+
+      if (typeof options === 'function') {
+        callback = options
+        options = {}
       }
 
+      options = options || {}
+
       let data
 
       if (template) {
@@ -111,13 +126,18 @@ module.exports = function object (self) {
         if (err) {
           return callback(err)
         }
-        self._ipld.put(node, {
-          cid: new CID(node.multihash)
-        }, (err) => {
+
+        const cid = new CID(node.multihash)
+
+        self._ipld.put(node, { cid }, (err) => {
           if (err) {
             return callback(err)
           }
 
+          if (options.preload !== false) {
+            self._preload(cid)
+          }
+
           callback(null, node)
         })
       })
@@ -166,13 +186,17 @@ module.exports = function object (self) {
       }
 
       function next () {
-        self._ipld.put(node, {
-          cid: new CID(node.multihash)
-        }, (err) => {
+        const cid = new CID(node.multihash)
+
+        self._ipld.put(node, { cid }, (err) => {
           if (err) {
             return callback(err)
           }
 
+          if (options.preload !== false) {
+            self._preload(cid)
+          }
+
           self.object.get(node.multihash, callback)
         })
       }
@@ -282,6 +306,8 @@ module.exports = function object (self) {
         editAndSave((node, cb) => {
           if (DAGLink.isDAGLink(linkRef)) {
             linkRef = linkRef._name
+          } else if (linkRef && linkRef.name) {
+            linkRef = linkRef.name
           }
           DAGNode.rmLink(node, linkRef, cb)
         })(multihash, options, callback)

src/core/components/pin-set.js

@@ -90,7 +90,7 @@ exports = module.exports = function (dag) {
 
       pinSet.storeItems(pins, (err, rootNode) => {
         if (err) { return callback(err) }
-        const opts = { cid: new CID(rootNode.multihash) }
+        const opts = { cid: new CID(rootNode.multihash), preload: false }
         dag.put(rootNode, opts, (err, cid) => {
           if (err) { return callback(err) }
           callback(null, rootNode)
@@ -168,7 +168,8 @@ exports = module.exports = function (dag) {
         function storeChild (err, child, binIdx, cb) {
           if (err) { return cb(err) }
 
-          dag.put(child, { cid: new CID(child._multihash) }, err => {
+          const opts = { cid: new CID(child._multihash), preload: false }
+          dag.put(child, opts, err => {
             if (err) { return cb(err) }
             fanoutLinks[binIdx] = new DAGLink('', child.size, child.multihash)
             cb(null)

src/core/components/pin.js

@@ -80,14 +80,14 @@ module.exports = (self) => {
       // the pin-set nodes link to a special 'empty' node, so make sure it exists
       cb => DAGNode.create(Buffer.alloc(0), (err, empty) => {
         if (err) { return cb(err) }
-        dag.put(empty, { cid: new CID(empty.multihash) }, cb)
+        dag.put(empty, { cid: new CID(empty.multihash), preload: false }, cb)
       }),
 
       // create a root node with DAGLinks to the direct and recursive DAGs
       cb => DAGNode.create(Buffer.alloc(0), [dLink, rLink], (err, node) => {
         if (err) { return cb(err) }
         root = node
-        dag.put(root, { cid: new CID(root.multihash) }, cb)
+        dag.put(root, { cid: new CID(root.multihash), preload: false }, cb)
       }),
 
       // hack for CLI tests

src/core/components/start.js

@@ -42,7 +42,9 @@ module.exports = (self) => {
 
         self._bitswap.start()
         self._blockService.setExchange(self._bitswap)
-        cb()
+
+        self._preload.start()
+        self._mfsPreload.start(cb)
       }
     ], done)
   })

src/core/components/stop.js

@@ -30,8 +30,10 @@ module.exports = (self) => {
     self.state.stop()
     self._blockService.unsetExchange()
     self._bitswap.stop()
+    self._preload.stop()
 
     series([
+      (cb) => self._mfsPreload.stop(cb),
       (cb) => self.libp2p.stop(cb),
       (cb) => self._repo.close(cb)
     ], done)

src/core/config.js

@@ -8,6 +8,10 @@ const schema = Joi.object().keys({
     Joi.string()
   ).allow(null),
   repoOwner: Joi.boolean().default(true),
+  preload: Joi.object().keys({
+    enabled: Joi.boolean().default(true),
+    addresses: Joi.array().items(Joi.multiaddr().options({ convert: false }))
+  }).allow(null),
   init: Joi.alternatives().try(
     Joi.boolean(),
     Joi.object().keys({ bits: Joi.number().integer() })

src/core/index.js

@@ -22,6 +22,8 @@ const boot = require('./boot')
 const components = require('./components')
 // replaced by repo-browser when running in the browser
 const defaultRepo = require('./runtime/repo-nodejs')
+const preload = require('./preload')
+const mfsPreload = require('./mfs-preload')
 
 class IPFS extends EventEmitter {
   constructor (options) {
@@ -30,7 +32,14 @@ class IPFS extends EventEmitter {
     this._options = {
       init: true,
       start: true,
-      EXPERIMENTAL: {}
+      EXPERIMENTAL: {},
+      preload: {
+        enabled: true,
+        addresses: [
+          '/dnsaddr/node0.preload.ipfs.io/https',
+          '/dnsaddr/node1.preload.ipfs.io/https'
+        ]
+      }
     }
 
     options = config.validate(options || {})
@@ -78,6 +87,8 @@ class IPFS extends EventEmitter {
     this._blockService = new BlockService(this._repo)
     this._ipld = new Ipld(this._blockService)
     this._pubsub = undefined
+    this._preload = preload(this)
+    this._mfsPreload = mfsPreload(this)
 
     // IPFS Core exposed components
     //   - for booting up a node
@@ -134,7 +145,7 @@ class IPFS extends EventEmitter {
     }
 
     // ipfs.files
-    const mfs = components.mfs(this, this._options)
+    const mfs = components.mfs(this)
 
     Object.keys(mfs).forEach(key => {
       this.files[key] = mfs[key]