[WIP] feat: load IPLD formats lazily from IPFS

package.json

@@ -13,6 +13,7 @@
     "./src/core/runtime/config-nodejs.js": "./src/core/runtime/config-browser.js",
     "./src/core/runtime/dns-nodejs.js": "./src/core/runtime/dns-browser.js",
     "./src/core/runtime/fetch-nodejs.js": "./src/core/runtime/fetch-browser.js",
+    "./src/core/runtime/ipld-formats-nodejs.js": "./src/core/runtime/ipld-formats-browser.js",
     "./src/core/runtime/libp2p-nodejs.js": "./src/core/runtime/libp2p-browser.js",
     "./src/core/runtime/preload-nodejs.js": "./src/core/runtime/preload-browser.js",
     "./src/core/runtime/repo-nodejs.js": "./src/core/runtime/repo-browser.js",

src/core/components/files-regular/cat-pull-stream.js

@@ -16,7 +16,8 @@ module.exports = function (self) {
     ipfsPath = normalizePath(ipfsPath)
     const pathComponents = ipfsPath.split('/')
     const restPath = normalizePath(pathComponents.slice(1).join('/'))
-    const filterFile = (file) => (restPath && file.path === restPath) || (file.path === ipfsPath)
+    const fileName = restPath.includes('/') ? restPath.split('/').pop() : restPath
+    const filterFile = (file) => (restPath && file.path === restPath) || (file.path === ipfsPath) || file.path === fileName
 
     if (options.preload !== false) {
       self._preload(pathComponents[0])

src/core/index.js

@@ -23,40 +23,7 @@ const components = require('./components')
 const defaultRepo = require('./runtime/repo-nodejs')
 const preload = require('./preload')
 const mfsPreload = require('./mfs-preload')
-
-// All known (non-default) IPLD formats
-const IpldFormats = {
-  get 'bitcoin-block' () {
-    return require('ipld-bitcoin')
-  },
-  get 'eth-account-snapshot' () {
-    return require('ipld-ethereum').ethAccountSnapshot
-  },
-  get 'eth-block' () {
-    return require('ipld-ethereum').ethBlock
-  },
-  get 'eth-block-list' () {
-    return require('ipld-ethereum').ethBlockList
-  },
-  get 'eth-state-trie' () {
-    return require('ipld-ethereum').ethStateTrie
-  },
-  get 'eth-storage-trie' () {
-    return require('ipld-ethereum').ethStorageTrie
-  },
-  get 'eth-tx' () {
-    return require('ipld-ethereum').ethTx
-  },
-  get 'eth-tx-trie' () {
-    return require('ipld-ethereum').ethTxTrie
-  },
-  get 'git-raw' () {
-    return require('ipld-git')
-  },
-  get 'zcash-block' () {
-    return require('ipld-zcash')
-  }
-}
+const ipldFormats = require('./runtime/ipld-formats-nodejs')
 
 class IPFS extends EventEmitter {
   constructor (options) {
@@ -109,6 +76,8 @@ class IPFS extends EventEmitter {
       CID: CID
     }
 
+    const loadIpldFormat = ipldFormats(this)
+
     // IPFS Core Internals
     // this._repo - assigned above
     this._peerInfoBook = new PeerBook()
@@ -119,9 +88,8 @@ class IPFS extends EventEmitter {
     this._ipld = new Ipld({
       blockService: this._blockService,
       loadFormat: (codec, callback) => {
-        this.log('Loading IPLD format', codec)
-        if (IpldFormats[codec]) return callback(null, IpldFormats[codec])
-        callback(new Error(`Missing IPLD format "${codec}"`))
+        console.log('Loading IPLD format', codec)
+        loadIpldFormat(codec, callback)
       }
     })
     this._preload = preload(this)

src/core/runtime/ipld-formats-browser.js

@@ -0,0 +1,99 @@
+'use strict'
+
+const IPLD_FORMATS_CID = 'QmP7qfTriY43hb2fUppkd5NvCSV5GxMAHM6cYHMq9uFTeX'
+
+module.exports = self => {
+  const options = self._options.ipld || {}
+  const jsLoader = createIpfsJsLoader(self, `/ipfs/${options.formatsCid || IPLD_FORMATS_CID}`)
+
+  // All known (non-default) IPLD formats
+  const IpldFormatLoaders = {
+    'bitcoin-block': jsLoader(
+      '/[email protected]/index.min.js',
+      () => window.IpldBitcoin
+    ),
+    'eth-account-snapshot': jsLoader(
+      '/[email protected]/index.min.js',
+      () => window.IpldEthereum.ethAccountSnapshot
+    ),
+    'eth-block': jsLoader(
+      '/[email protected]/index.min.js',
+      () => window.IpldEthereum.ethBlock
+    ),
+    'eth-block-list': jsLoader(
+      '/[email protected]/index.min.js',
+      () => window.IpldEthereum.ethBlockList
+    ),
+    'eth-state-trie': jsLoader(
+      '/[email protected]/index.min.js',
+      () => window.IpldEthereum.ethStateTrie
+    ),
+    'eth-storage-trie': jsLoader(
+      '/[email protected]/index.min.js',
+      () => window.IpldEthereum.ethStorageTrie
+    ),
+    'eth-tx': jsLoader(
+      '/[email protected]/index.min.js',
+      () => window.IpldEthereum.ethTx
+    ),
+    'eth-tx-trie': jsLoader(
+      '/[email protected]/index.min.js',
+      () => window.IpldEthereum.ethTxTrie
+    ),
+    'git-raw': jsLoader(
+      '/[email protected]/index.min.js',
+      () => window.IpldGit
+    ),
+    'zcash-block': jsLoader(
+      '/[email protected]/index.min.js',
+      () => window.IpldZcash
+    )
+  }
+
+  return (codec, callback) => {
+    if (IpldFormatLoaders[codec]) return IpldFormatLoaders[codec](callback)
+    callback(new Error(`Missing IPLD format "${codec}"`))
+  }
+}
+
+// Create a module loader for the passed root path e.g. /ipfs/QmHash
+function createIpfsJsLoader (ipfs, rootPath) {
+  const Modules = {}
+
+  // Create a loader for the given path that will extract a JS object from
+  // the exports of the loaded module using getExport
+  return (path, getExport) => {
+    return callback => {
+      if (Modules[path]) {
+        switch (Modules[path].state) {
+          case 'loading':
+            return Modules[path].callbacks.push({ getExport, callback })
+          case 'loaded':
+            return callback(null, getExport(Modules[path].exports))
+          case 'error':
+            return callback(Modules[path].error)
+        }
+        return callback(new Error('unknown format load state'))
+      }
+
+      Modules[path] = { state: 'loading', callbacks: [{ getExport, callback }] }
+
+      ipfs.cat(`${rootPath}${path}`, (err, data) => {
+        if (err) {
+          Object.assign(Modules[path], { state: 'error', error: err })
+          Modules[path].callbacks.forEach(({ callback }) => callback(err))
+          Modules[path].callbacks = []
+          return
+        }
+
+        const exports = (new Function(data.toString()))() // eslint-disable-line no-new-func
+        Object.assign(Modules[path], { state: 'loaded', exports })
+
+        Modules[path].callbacks.forEach(({ getExport, callback }) => {
+          callback(null, getExport(exports))
+        })
+        Modules[path].callbacks = []
+      })
+    }
+  }
+}

src/core/runtime/ipld-formats-nodejs.js

@@ -0,0 +1,42 @@
+'use strict'
+
+// All known (non-default) IPLD formats
+const IpldFormats = {
+  get 'bitcoin-block' () {
+    return require('ipld-bitcoin')
+  },
+  get 'eth-account-snapshot' () {
+    return require('ipld-ethereum').ethAccountSnapshot
+  },
+  get 'eth-block' () {
+    return require('ipld-ethereum').ethBlock
+  },
+  get 'eth-block-list' () {
+    return require('ipld-ethereum').ethBlockList
+  },
+  get 'eth-state-trie' () {
+    return require('ipld-ethereum').ethStateTrie
+  },
+  get 'eth-storage-trie' () {
+    return require('ipld-ethereum').ethStorageTrie
+  },
+  get 'eth-tx' () {
+    return require('ipld-ethereum').ethTx
+  },
+  get 'eth-tx-trie' () {
+    return require('ipld-ethereum').ethTxTrie
+  },
+  get 'git-raw' () {
+    return require('ipld-git')
+  },
+  get 'zcash-block' () {
+    return require('ipld-zcash')
+  }
+}
+
+module.exports = () => {
+  return (codec, callback) => {
+    if (IpldFormats[codec]) return callback(null, IpldFormats[codec])
+    callback(new Error(`Missing IPLD format "${codec}"`))
+  }
+}