perf: concurrent file import

README.md

@@ -134,9 +134,7 @@ The input's file paths and directory structure will be preserved in the [`dag-pb
 - `maxChildrenPerNode` (positive integer, defaults to `174`): the maximum children per node for the `balanced` and `trickle` DAG builder strategies
 - `layerRepeat` (positive integer, defaults to 4): (only applicable to the `trickle` DAG builder strategy). The maximum repetition of parent nodes for each layer of the tree.
 - `reduceSingleLeafToSelf` (boolean, defaults to `true`): optimization for, when reducing a set of nodes with one node, reduce it to that node.
-- `dirBuilder` (object): the options for the directory builder
-  - `hamt` (object): the options for the HAMT sharded directory builder
-    - bits (positive integer, defaults to `8`): the number of bits at each bucket of the HAMT
+- `bits` (positive integer, defaults to `8`): the number of bits at each bucket of the HAMT
 - `progress` (function): a function that will be called with the byte length of chunks as a file is added to ipfs.
 - `onlyHash` (boolean, defaults to false): Only chunk and hash - do not write to disk
 - `hashAlg` (string): multihash hashing algorithm to use

package.json

@@ -61,7 +61,6 @@
     "it-all": "^1.0.1",
     "it-batch": "^1.0.3",
     "it-first": "^1.0.1",
-    "it-flat-batch": "^1.0.2",
     "it-parallel-batch": "1.0.2",
     "merge-options": "^2.0.0",
     "multicodec": "~0.5.1",

src/dag-builder/file/balanced.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const batch = require('it-flat-batch')
+const batch = require('it-batch')
 
 async function * balanced (source, reduce, options) {
   yield await reduceToParents(source, reduce, options)

src/dag-builder/file/flat.js

@@ -1,13 +1,7 @@
 'use strict'
 
-const batch = require('it-flat-batch')
+const all = require('it-all')
 
 module.exports = async function * (source, reduce) {
-  const roots = []
-
-  for await (const chunk of batch(source, Number.MAX_SAFE_INTEGER)) {
-    roots.push(await reduce(chunk))
-  }
-
-  yield roots[0]
+  yield await reduce(await all(source))
 }

src/dag-builder/file/trickle.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const batch = require('it-flat-batch')
+const batch = require('it-batch')
 
 module.exports = function * trickleReduceToRoot (source, reduce, options) {
   yield trickleStream(source, reduce, options)