From 0cf77a9bdc1c3b436af44f75a792f1eba20ea97b Mon Sep 17 00:00:00 2001
From: Mikeal Rogers <mikeal.rogers@gmail.com>
Date: Fri, 11 Sep 2020 04:16:58 +0000
Subject: [PATCH 1/6] init: new fbl based file

---
 README.md            |  44 ----------
 browser.js           |   6 --
 cli.js               |  88 --------------------
 index.js             | 117 ++++++++++++++++++++++++--
 package.json         |  27 ++----
 src/data-layout.json | 114 --------------------------
 src/data.js          | 190 -------------------------------------------
 src/file.js          |  26 ------
 src/fs.js            |  57 -------------
 src/local-storage.js |  23 ------
 src/parse.js         |   9 --
 src/reader.js        |  58 -------------
 src/schema.json      | 156 -----------------------------------
 src/types.js         |  13 ---
 test/bytes.spec.js   |  88 --------------------
 test/encode.spec.js  |  36 --------
 test/reader.spec.js  |  55 -------------
 test/test-basics.js  |  54 ++++++++++++
 18 files changed, 170 insertions(+), 991 deletions(-)
 delete mode 100644 browser.js
 delete mode 100755 cli.js
 delete mode 100644 src/data-layout.json
 delete mode 100644 src/data.js
 delete mode 100644 src/file.js
 delete mode 100644 src/fs.js
 delete mode 100644 src/local-storage.js
 delete mode 100644 src/parse.js
 delete mode 100644 src/reader.js
 delete mode 100644 src/schema.json
 delete mode 100644 src/types.js
 delete mode 100644 test/bytes.spec.js
 delete mode 100644 test/encode.spec.js
 delete mode 100644 test/reader.spec.js
 create mode 100644 test/test-basics.js

diff --git a/README.md b/README.md
index 710b12d..469b615 100644
--- a/README.md
+++ b/README.md
@@ -1,47 +1,3 @@
 # js-unixfsv2
 
-This library is a full implementation of [`unixfs-v2`](https://github.com/ipfs/unixfs-v2) for JavaScript.
-
-## encoder(path, options={})
-
-Async generator that yields blocks for every file and
-and directory in the path.
-
-Last block is the root block.
-
-Runs recursively through directories. Accepts any valid
-file or directory string path.
-
-```javascript
-const { encoder } = require('unixfsv2')
-const storage = require('any-key-value-store')
-
-const putBlock = async b => storage.put((await b.cid()).toString(), b.encode())
-
-const storeDirectory = async path => {
-  for await (const { block, root } of encoder(__dirname)) {
-    await storage.putBlock(block || root.block())
-    if (root) return root.block().cid()
-  }
-}
-```
-
-## reader(rootBlock, getBlock)
-
-Returns a new Reader instance for the
-root block.
-
-```javascript
-const { reader } = require('unixfsv2')
-const storage = require('any-key-value-store')
-const Block = require('@ipld/block')
-
-const getBlock = async cid => Block.create(await storage.get(cid.toString()), cid)
-
-/* rootBlock would be the same as the last block in
-/  encode example.
-*/
-const r = reader(rootBlock, getBlock)
-```
-
 
diff --git a/browser.js b/browser.js
deleted file mode 100644
index 5e1dd0a..0000000
--- a/browser.js
+++ /dev/null
@@ -1,6 +0,0 @@
-'use strict'
-const createTypes = require('./src/types')
-const reader = require('./src/reader')
-
-exports.reader = reader
-exports.createTypes = createTypes
diff --git a/cli.js b/cli.js
deleted file mode 100755
index 75fb127..0000000
--- a/cli.js
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/usr/bin/env node
-'use strict'
-const parseSchema = require('./src/parse.js')
-const fs = require('fs')
-const printify = require('@ipld/printify')
-const { inspect } = require('util')
-const api = require('./src/fs')
-const reader = require('./src/reader')
-const createStorage = require('./src/local-storage')
-
-/* eslint-disable no-console */
-
-const parse = async argv => {
-  const s = await parseSchema(argv.input, argv.output)
-  if (!argv.output) console.log(inspect(s, { depth: Infinity }))
-  else fs.writeFileSync(argv.output, JSON.stringify(s))
-}
-
-const runImport = async argv => {
-  const iter = await api.fromFileSystem(argv.input)
-  let store
-  if (argv.storage) store = createStorage(argv.storage)
-  for await (let { block, root } of iter) {
-    if (root) block = root.block()
-    if (store) {
-      await store.put(block)
-      if (root) console.log('Root:', (await root.block().cid()).toString())
-    } else {
-      if (block.codec === 'raw') {
-        console.log('Block<raw>', (await block.cid()).toString())
-      } else {
-        console.log('Block<' + block.codec + '>', printify(block.decode()))
-      }
-    }
-  }
-}
-
-const createReader = argv => {
-  const store = createStorage(argv.storage)
-  const _reader = reader(argv.rootCID, store.get)
-  return _reader
-}
-
-const runRead = async argv => {
-  const reader = createReader(argv)
-  for await (const buffer of reader.read(argv.path, argv.start, argv.end)) {
-    process.stdout.write(buffer)
-  }
-}
-const runLs = async argv => {
-  const reader = createReader(argv)
-  const files = await reader.ls(argv.path)
-  files.forEach(f => console.log(f))
-}
-
-const storageOptions = yargs => {
-  yargs.option('storage', { desc: 'Directory to store blocks' })
-}
-const importOptions = yargs => {
-  yargs.positional('input', { desc: 'File or directory to import' })
-  storageOptions(yargs)
-}
-const readerOptions = yargs => {
-  yargs.positional('storage', { desc: 'Directory of stored blocks' })
-  yargs.positional('rootCID', { desc: 'CID of root node for file or directory' })
-}
-const readOptions = yargs => {
-  readerOptions(yargs)
-  yargs.positional('path', { desc: 'Path to filename' })
-  yargs.option('start', { desc: 'starting position, defaults to 0' })
-  yargs.option('end', { desc: 'ending position, defaults to end of file' })
-}
-const lsOptions = yargs => {
-  readerOptions(yargs)
-  yargs.positional('path', { desc: 'Path to directory' })
-}
-
-const yargs = require('yargs')
-const args = yargs
-  .command('parse <input> [output]', 'Parse schema file', importOptions, parse)
-  .command('import <input>', 'Import file or directory', storageOptions, runImport)
-  .command('ls <storage> <rootCID> <path>', 'List directory contents', lsOptions, runLs)
-  .command('read <storage> <rootCID> <path>', 'Read file contents', readOptions, runRead)
-  .argv
-
-if (!args._.length) {
-  yargs.showHelp()
-}
diff --git a/index.js b/index.js
index bd2561f..b32cdf0 100644
--- a/index.js
+++ b/index.js
@@ -1,9 +1,108 @@
-'use strict'
-const fs = require('./src/fs')
-const createTypes = require('./src/types')
-const reader = require('./src/reader')
-
-exports.encoder = fs
-exports.reader = reader
-exports.createTypes = createTypes
-exports.types = fs.types
+import { promises as fs } from 'fs'
+import * as hamt from 'hamt-utils'
+import fbl from '@ipld/fbl'
+
+/* bit of a hack since it buffers the whole
+   file in memory but i don't want to take
+   a larger dep that will stream properly.
+   eventaully, i'll come back and make this properly stream.
+  */
+const onemeg = async function * (buffer) {
+  let chunk = buffer
+  while (chunk.byteLength) {
+    yield chunk.subarray(0, 1024 * 1000)
+    chunk = chunk.subarray(1024 * 1000)
+  }
+}
+
+const encode = async function * (Block, path, chunker=onemeg) {
+  const stat = await fs.stat(path)
+  if (stat.isDirectory()) {
+    const files = await fs.readdir(path)
+    const dir = {}
+    for (const file of files) {
+      let last
+      for await (const block of encode(Block, new URL(file, path + '/sub'), chunker)) {
+        if (last) yield last
+        last = block
+      }
+      dir[file] = last
+    }
+    let last
+    for await (const block of hamt.from(Block, dir)) {
+      yield block
+      last = block
+    }
+    yield { content: { d: await last.cid() } }
+  } else {
+    let last
+    for await (const block of fbl.from(chunker(await fs.readFile(path)))) {
+      yield block
+      last = block
+    }
+    yield { content: { f: await last.cid() } }
+  }
+}
+
+const encoder = async function * (Block, path, chunker) {
+  let last
+  for await (const block of encode(Block, path, chunker)) {
+    if (last) yield last
+    last = block
+  }
+  yield Block.encoder(last, 'dag-cbor')
+}
+
+const readFile = async function * (reader, parts, start, end) {
+  const block = await reader.traverse(parts)
+  yield * fbl.read(block, reader.get, start, end)
+}
+
+const toString = b => (new TextDecoder()).decode(b)
+
+const lsDirectory = async function * (reader, parts) {
+  const block = await reader.traverse(parts)
+  const decoded = block.decodeUnsafe()
+  if (!decoded.content) throw new Error('Not a valid DirEnt')
+  if (!decoded.content.d) throw new Error('Not a file')
+  for await (const { key } of hamt.all(decoded.content.d, reader.get, start, end)) {
+    yield toString(key)
+  }
+}
+
+class Reader {
+  constructor (head, get) {
+    this.head = head
+    this.get = get
+  }
+  async traverse (parts) {
+    let head = await this.get(this.head)
+    if (!parts.length) {
+      const { d, f } = head.decodeUnsafe().content
+      return d || f
+    }
+    while (parts.length) {
+      const key = parts.shift()
+      const decoded = head.decodeUnsafe()
+      if (!decoded.content) throw new Error('Not a valid DirEnt')
+      if (!decoded.content.d) throw new Error('Not a directory')
+      const dirEnt = await hamt.get(decoded.content.d, key, this.get)
+      const { d, f } = dirEnt.content
+      if (f && parts.length) throw new Error(`${key} is not a directory`)
+      head = await this.get(d || f)
+    }
+    return head
+  }
+  read (path='', start, end) {
+    path = path.split('/').filter(x => x)
+    return readFile(this, path, start, end)
+  }
+  ls (path='') {
+    path = path.split('/').filter(x => x)
+    return lsDirectory(this, path)
+  }
+}
+
+const reader = (...args) => new Reader(...args)
+
+export { encoder, reader }
diff --git a/package.json b/package.json
index 005fc17..3490db1 100644
--- a/package.json
+++ b/package.json
@@ -1,24 +1,15 @@
 {
-  "name": "@ipld/unixfsv2",
+  "name": "@ipld/fs",
   "version": "0.0.0-dev",
-  "description": "Filesystem in a merkle tree.",
-  "browser": "./browser.js",
-  "scripts": {
-    "lint": "aegir lint",
-    "pretest": "npm run lint",
-    "build": "ipld-schema to-json doc/DataLayout.md > src/data-layout.json && ipld-schema to-json doc/README.md > src/schema.json",
-    "test": "aegir test -t node"
-  },
+  "description": "Implementation of UnixFSv2.",
+  "type": "module",
+  "scripts": {},
   "keywords": [],
   "author": "Mikeal Rogers <mikeal.rogers@gmail.com> (http://www.mikealrogers.com)",
   "dependencies": {
-    "@ipld/block": "^2.0.6",
-    "@ipld/printify": "0.0.0",
-    "@ipld/schema-gen": "0.0.1",
-    "bytesish": "^0.4.1",
-    "mime-types": "^2.1.24",
-    "rabin-generator": "0.0.1",
-    "stream-chunker": "^1.2.8"
+    "@ipld/fbl": "^2.0.1",
+    "@ipld/block": "^6.0.3",
+    "hamt-utils": "^0.0.4"
   },
   "directories": {
     "test": "test"
@@ -33,8 +24,6 @@
   },
   "homepage": "https://github.com/ipld/js-unixfsv2#readme",
   "devDependencies": {
-    "aegir": "^20.4.1",
-    "ipld-schema": "^0.3.1",
-    "tsame": "^2.0.1"
+    "estest": "^10.3.5"
   }
 }
diff --git a/src/data-layout.json b/src/data-layout.json
deleted file mode 100644
index 24c9dea..0000000
--- a/src/data-layout.json
+++ /dev/null
@@ -1,114 +0,0 @@
-{
-  "types": {
-    "Index": {
-      "kind": "list",
-      "valueType": "Int"
-    },
-    "IndexList": {
-      "kind": "list",
-      "valueType": "Index"
-    },
-    "ByteUnionList": {
-      "kind": "list",
-      "valueType": {
-        "kind": "link",
-        "expectedType": "BytesUnion"
-      }
-    },
-    "NestedByteListLayout": {
-      "kind": "struct",
-      "fields": {
-        "indexes": {
-          "type": "IndexList"
-        },
-        "parts": {
-          "type": "ByteUnionList"
-        },
-        "algo": {
-          "type": "String"
-        }
-      },
-      "representation": {
-        "map": {}
-      }
-    },
-    "NestedByteList": {
-      "kind": "bytes",
-      "representation": {
-        "advanced": "NestedByteListLayout"
-      }
-    },
-    "ByteList": {
-      "kind": "list",
-      "valueType": {
-        "kind": "link",
-        "expectedType": "Bytes"
-      }
-    },
-    "ByteLinksLayout": {
-      "kind": "struct",
-      "fields": {
-        "indexes": {
-          "type": "IndexList"
-        },
-        "parts": {
-          "type": "ByteList"
-        }
-      },
-      "representation": {
-        "map": {}
-      }
-    },
-    "ByteLinks": {
-      "kind": "bytes",
-      "representation": {
-        "advanced": "ByteLinksLayout"
-      }
-    },
-    "BytesUnion": {
-      "kind": "union",
-      "representation": {
-        "keyed": {
-          "bytes": "Bytes",
-          "bytesLink": {
-            "kind": "link",
-            "expectedType": "Bytes"
-          },
-          "byteLinks": "ByteLinks",
-          "nbl": "NestedByteList"
-        }
-      }
-    },
-    "DataLayout": {
-      "kind": "struct",
-      "fields": {
-        "bytes": {
-          "type": "BytesUnion"
-        },
-        "size": {
-          "type": "Int"
-        }
-      },
-      "representation": {
-        "map": {}
-      }
-    },
-    "Data": {
-      "kind": "bytes",
-      "representation": {
-        "advanced": "DataLayout"
-      }
-    }
-  },
-  "advanced": {
-    "NestedByteListLayout": {
-      "kind": "advanced"
-    },
-    "ByteLinksLayout": {
-      "kind": "advanced"
-    },
-    "DataLayout": {
-      "kind": "advanced"
-    }
-  }
-}
diff --git a/src/data.js b/src/data.js
deleted file mode 100644
index 1684f37..0000000
--- a/src/data.js
+++ /dev/null
@@ -1,190 +0,0 @@
-'use strict'
-const gen = require('@ipld/schema-gen')
-const schema = require('./data-layout.json')
-const Block = require('@ipld/block')
-
-const indexLength = ii => ii[ii.length - 1].reduce((x, y) => x + y)
-
-const readAnything = async function * (selector, node, start = 0, end = Infinity) {
-  const offsets = (offset, length) => {
-    let l = end - offset
-    if (l === Infinity || l > length) l = undefined
-    let s = start - offset
-    if (s < 0) s = 0
-    return [s, l]
-  }
-
-  const indexes = node.resolve('indexes').decode()
-  const parts = node.resolve('parts')
-  let i = 0
-
-  for (const [offset, length] of indexes) {
-    if ((offset + length) >= start && offset < end) {
-      const b = await parts.getNode(i + '/' + selector)
-      yield * b.read(...offsets(offset, length))
-    }
-    i++
-  }
-}
-
-const readByteLinkArray = (node, start, end) => readAnything('', node, start, end)
-const readNestedByteList = (node, start, end) => readAnything('*', node, start, end)
-
-const readBytes = async function * (node, start = 0, end) {
-  const bytes = await node.getNode('bytes/*')
-  yield * bytes.read(start, end)
-}
-
-const getLength = node => {
-  const indexes = node.resolve('indexes').value
-  if (!indexes.length) return 0
-  const [offset, length] = indexes[indexes.length - 1].encode()
-  return offset + length
-}
-
-const advanced = {
-  DataLayout: { read: readBytes, length: node => node.size ? node.size.value : null },
-  ByteLinksLayout: { read: readByteLinkArray, length: getLength },
-  NestedByteListLayout: { read: readNestedByteList, length: getLength }
-}
-
-module.exports = opts => {
-  opts = { ...opts, advanced }
-  const classes = gen(schema, opts)
-  const _writer = () => {
-    const indexes = []
-    const parts = []
-    let offset = 0
-    const write = buffer => {
-      indexes.push([offset, buffer.length])
-      offset += buffer.length
-      const block = Block.encoder(buffer, 'raw')
-      parts.push(block.cid())
-      return block
-    }
-    const end = async () => ({ indexes, parts: await Promise.all(parts) })
-    return { write, end }
-  }
-  classes.ByteLinks.fromArray = async function * (arr) {
-    const { write, end } = _writer()
-    for (const buffer of arr) {
-      const block = write(buffer)
-      yield { block }
-    }
-    const data = await end()
-    yield { root: classes.ByteLinks.encoder(data) }
-  }
-  const defaults = {
-    maxListLength: 500,
-    algorithm: 'balanced',
-    inline: 0
-  }
-  const balancedDag = async function * (indexes, parts, max, codec) {
-    if (indexes.length !== parts.length) {
-      throw new Error('index length must match part length')
-    }
-    indexes = [...indexes]
-    parts = [...parts]
-    const size = Math.ceil(indexes.length / max)
-
-    const main = { indexes: [], parts: [] }
-    if (size > max) {
-      while (indexes.length) {
-        const chunk = {
-          indexes: indexes.splice(0, max),
-          parts: parts.splice(0, max),
-          algo: 'balanced'
-        }
-        let union
-        for await (const { len, block, root } of balancedDag(chunk.indexes, chunk.parts, max, codec)) {
-          if (block) {
-            yield { len, block }
-          }
-          union = root
-        }
-        const node = classes.BytesUnion.encoder(union)
-        const block = node.block()
-        const len = indexLength(union.nbl.indexes)
-        yield { len, block }
-        main.indexes.push(len)
-        main.parts.push(block.cid())
-      }
-    } else {
-      while (indexes.length) {
-        const chunk = {
-          indexes: indexes.splice(0, max),
-          parts: parts.splice(0, max)
-        }
-        let offset = 0
-        for (const part of chunk.indexes) {
-          part[0] = offset
-          offset += part[1]
-        }
-        const block = classes.BytesUnion.encoder({ byteLinks: chunk }).block()
-        yield { len: offset, block }
-        main.indexes.push(offset)
-        main.parts.push(block.cid())
-      }
-    }
-    let offset = 0
-    indexes = main.indexes.map(i => {
-      const ret = [offset, i]
-      offset += i
-      return ret
-    })
-    parts = await Promise.all(main.parts)
-    yield { root: { nbl: { indexes, parts, algo: 'balanced' } } }
-  }
-
-  classes.Data.writer = (opts = {}) => {
-    opts = { ...defaults, ...opts }
-    const { write, end } = _writer()
-    let first
-    const _write = block => {
-      if (!first) first = block
-      return write(block)
-    }
-    const _end = async () => {
-      const { indexes, parts } = await end()
-      if (!indexes.length) return { bytes: { bytes: Buffer.from('') }, size: 0 }
-
-      const size = indexLength(indexes)
-
-      if (indexes.length === 1) {
-        const length = indexes[0][1]
-        if (length > opts.inline) {
-          return { bytes: { bytesLink: parts[0] }, size }
-        } else {
-          return { bytes: { bytes: first.encode() }, size }
-        }
-      }
-      if (indexes.length > opts.maxListLength) {
-        const results = []
-        if (opts.algorithm === 'balanced') {
-          for await (const result of balancedDag(indexes, parts, opts.maxListLength, opts.codec)) {
-            results.push(result)
-          }
-        } else {
-          throw new Error(`Not Implemented: algorith (${opts.algorithm})`)
-        }
-        const last = results.pop()
-        const blocks = results.map(r => r.block)
-        return { blocks, bytes: last.root, size }
-      } else {
-        return { bytes: { byteLinks: { indexes, parts } }, size }
-      }
-    }
-    return { write: _write, end: _end }
-  }
-  classes.Data.from = async function * (arr, opts) {
-    const { write, end } = classes.Data.writer(opts)
-    for (const buffer of arr) {
-      yield { block: write(buffer) }
-    }
-    const data = await end()
-    yield * data.blocks.map(u => ({ block: u }))
-    delete data.blocks
-    yield { root: classes.Data.encoder(data) }
-  }
-  return classes
-}
diff --git a/src/file.js b/src/file.js
deleted file mode 100644
index bbd16c1..0000000
--- a/src/file.js
+++ /dev/null
@@ -1,26 +0,0 @@
-'use strict'
-const bytes = require('bytesish')
-
-const attach = types => {
-  const fromIter = async function * (iter, name, opts = {}) {
-    const { write, end } = types.Data.writer(opts)
-    for await (let chunk of iter) {
-      chunk = bytes.native(chunk)
-      const block = write(chunk)
-      yield { block }
-    }
-    const data = await end()
-    if (data.blocks) {
-      for (const block of (data.blocks)) {
-        yield { block }
-      }
-      delete data.blocks
-    }
-
-    const file = types.File.encoder({ name, data })
-    yield { root: file }
-  }
-  types.File.fromIter = fromIter
-}
-
-module.exports = attach
diff --git a/src/fs.js b/src/fs.js
deleted file mode 100644
index 2dc89e2..0000000
--- a/src/fs.js
+++ /dev/null
@@ -1,57 +0,0 @@
-'use strict'
-const types = require('./types')()
-const rabin = require('rabin-generator')
-const path = require('path')
-const { createReadStream, promises } = require('fs')
-const fs = promises
-
-// eslint bug: https://github.com/eslint/eslint/issues/12459
-// eslint-disable-next-line require-await
-const fromFile = async function * (f, stat, opts = {}) {
-  opts.chunker = opts.chunker || rabin
-  let iter = createReadStream(f)
-  iter = opts.chunker(iter)
-  yield * types.File.fromIter(iter, path.parse(f).base, opts)
-}
-
-const fromDirectory = async function * (f, stat, opts = {}) {
-  const codec = opts.codec || 'dag-cbor'
-  const dirfiles = await fs.readdir(f)
-  const results = {}
-  let size = 0
-  for (const file of dirfiles) {
-    const iter = fromFileSystem(path.join(f, file), opts)
-    for await (let { block, root } of iter) {
-      if (block) yield { block }
-      if (root) {
-        block = root.block(codec)
-        yield { block }
-        if (root instanceof types.File) {
-          size += await root.get('data/size')
-          results[file] = { fileLink: await block.cid() }
-        } else if (root instanceof types.Directory) {
-          size += await root.get('size')
-          results[file] = { dirLink: await block.cid() }
-        }
-      }
-    }
-  }
-  const data = types.DirData.encoder({ map: results })
-  const name = path.parse(f).base
-  const dir = types.Directory.encoder({ data, size, name })
-  yield { root: dir }
-}
-
-const fromFileSystem = async function * (f, opts = {}) {
-  const stat = await fs.stat(f)
-  if (stat.isFile()) {
-    yield * fromFile(f, stat, opts)
-  } else if (stat.isDirectory()) {
-    yield * fromDirectory(f, stat, opts)
-  }
-}
-
-exports = module.exports = fromFileSystem
-exports.fromFileSystem = fromFileSystem
-exports.fromDirectory = fromDirectory
-exports.fromFile = fromFile
diff --git a/src/local-storage.js b/src/local-storage.js
deleted file mode 100644
index b619626..0000000
--- a/src/local-storage.js
+++ /dev/null
@@ -1,23 +0,0 @@
-'use strict'
-const mkdirp = require('mkdirp')
-const path = require('path')
-const fs = require('fs').promises
-const Block = require('@ipld/block')
-
-const storage = dir => {
-  mkdirp.sync(dir)
-  const exports = {}
-  const key = cid => path.join(dir, cid.toString('base32'))
-  exports.put = async block => {
-    const f = key(await block.cid())
-    return fs.writeFile(f, block.encode())
-  }
-  exports.get = async cid => {
-    const f = key(cid)
-    const buffer = await fs.readFile(f)
-    return Block.create(buffer, cid)
-  }
-  return exports
-}
-
-module.exports = storage
diff --git a/src/parse.js b/src/parse.js
deleted file mode 100644
index 151a5de..0000000
--- a/src/parse.js
+++ /dev/null
@@ -1,9 +0,0 @@
-'use strict'
-const schema = require('ipld-schema')
-const { readFile } = require('fs').promises
-
-const main = async (input) => {
-  return schema.parse((await readFile(input)).toString())
-}
-
-module.exports = main
diff --git a/src/reader.js b/src/reader.js
deleted file mode 100644
index fc41633..0000000
--- a/src/reader.js
+++ /dev/null
@@ -1,58 +0,0 @@
-'use strict'
-const CID = require('cids')
-const Block = require('@ipld/block')
-const createTypes = require('./types')
-
-const buildPath = (path) => {
-  return [].concat(
-    ...path
-      .split('/')
-      .filter(x => x)
-      .map(name => ['data', '*', name, '*'])
-  )
-}
-
-const readIterator = async function * (reader, path, start, end) {
-  const data = await reader.get(path, 'data')
-  yield * data.read(start, end)
-}
-
-class Reader {
-  constructor (root, get) {
-    this.types = createTypes({ getBlock: get })
-    if (Block.isBlock(root)) {
-      root = this.types.Directory.decoder(root.decode())
-    }
-    if (typeof root === 'string') root = new CID(root)
-    if (CID.isCID(root)) {
-      root = get(root).then(b => this.types.Directory.decoder(b.decode()))
-    }
-    this.root = root
-    this.getBlock = get
-  }
-
-  async ls (path) {
-    const root = await this.root
-    const files = []
-    path = [].concat(buildPath(path), ['data', '*']).join('/')
-    const node = await root.getNode(path)
-    const keys = node.keys()
-    for await (const file of keys) {
-      files.push(file)
-    }
-    return files
-  }
-
-  async get (path, ...props) {
-    const root = await this.root
-    path = [...buildPath(path), ...props]
-    const file = await root.getNode(path.join('/'))
-    return file
-  }
-
-  read (path, start, end) {
-    return readIterator(this, path, start, end)
-  }
-}
-
-module.exports = (...args) => new Reader(...args)
diff --git a/src/schema.json b/src/schema.json
deleted file mode 100644
index 15e7976..0000000
--- a/src/schema.json
+++ /dev/null
@@ -1,156 +0,0 @@
-{
-  "types": {
-    "DirData": {
-      "kind": "union",
-      "representation": {
-        "keyed": {
-          "map": "EntryMap",
-          "hamt": "Hamt"
-        }
-      }
-    },
-    "EntryMap": {
-      "kind": "map",
-      "keyType": "String",
-      "valueType": "EntryUnion"
-    },
-    "EntryUnion": {
-      "kind": "union",
-      "representation": {
-        "keyed": {
-          "file": "File",
-          "fileLink": {
-            "kind": "link",
-            "expectedType": "File"
-          },
-          "dir": "Directory",
-          "dirLink": {
-            "kind": "link",
-            "expectedType": "Directory"
-          }
-        }
-      }
-    },
-    "Directory": {
-      "kind": "struct",
-      "fields": {
-        "name": {
-          "type": "String",
-          "optional": true
-        },
-        "size": {
-          "type": "Int",
-          "optional": true
-        },
-        "data": {
-          "type": "DirData"
-        }
-      },
-      "representation": {
-        "map": {}
-      }
-    },
-    "Permissions": {
-      "kind": "struct",
-      "fields": {
-        "uid": {
-          "type": "Int"
-        },
-        "gid": {
-          "type": "Int"
-        },
-        "posix": {
-          "type": "Int"
-        },
-        "sticky": {
-          "type": "Bool"
-        },
-        "setuid": {
-          "type": "Bool"
-        },
-        "setgid": {
-          "type": "Bool"
-        }
-      },
-      "representation": {
-        "map": {
-          "fields": {
-            "sticky": {
-              "implicit": false
-            },
-            "setuid": {
-              "implicit": false
-            },
-            "setgid": {
-              "implicit": false
-            }
-          }
-        }
-      }
-    },
-    "Attributes": {
-      "kind": "struct",
-      "fields": {
-        "mtime": {
-          "type": "Int",
-          "optional": true
-        },
-        "atime": {
-          "type": "Int",
-          "optional": true
-        },
-        "ctime": {
-          "type": "Int",
-          "optional": true
-        },
-        "mtime64": {
-          "type": "Int",
-          "optional": true
-        },
-        "atime64": {
-          "type": "Int",
-          "optional": true
-        },
-        "ctime64": {
-          "type": "Int",
-          "optional": true
-        },
-        "permissions": {
-          "type": "Permissions",
-          "optional": true
-        },
-        "devMajor": {
-          "type": "Int",
-          "optional": true
-        },
-        "devMinor": {
-          "type": "Int",
-          "optional": true
-        }
-      },
-      "representation": {
-        "map": {}
-      }
-    },
-    "File": {
-      "kind": "struct",
-      "fields": {
-        "name": {
-          "type": "String",
-          "optional": true
-        },
-        "data": {
-          "type": "Data",
-          "optional": true
-        },
-        "size": {
-          "type": "Int",
-          "optional": true
-        }
-      },
-      "representation": {
-        "map": {}
-      }
-    }
-  }
-}
diff --git a/src/types.js b/src/types.js
deleted file mode 100644
index b7c3369..0000000
--- a/src/types.js
+++ /dev/null
@@ -1,13 +0,0 @@
-'use strict'
-const gen = require('@ipld/schema-gen')
-const data = require('./data.js')
-const main = require('./schema.json')
-const attach = require('./file.js')
-
-module.exports = opts => {
-  const types = data(opts)
-  opts = { ...opts, types }
-  const newTypes = { ...types, ...gen(main, opts) }
-  attach(newTypes)
-  return newTypes
-}
diff --git a/test/bytes.spec.js b/test/bytes.spec.js
deleted file mode 100644
index 3f4d0c2..0000000
--- a/test/bytes.spec.js
+++ /dev/null
@@ -1,88 +0,0 @@
-'use strict'
-const assert = require('assert')
-const tsame = require('tsame')
-const { it } = require('mocha')
-const { createTypes } = require('../')
-const { promisify } = require('util')
-const crypto = require('crypto')
-// const Block = require('@ipld/block')
-
-const test = it
-
-const storage = () => {
-  const db = {}
-  const get = cid => db[cid.toString()]
-  const put = async b => {
-    db[(await b.cid()).toString()] = b
-  }
-  return { get, put, db, getBlock: get }
-}
-
-const same = (x, y) => assert.ok(tsame(x, y))
-const buffer = Buffer.from('hello world')
-// const bb = Block.encoder(buffer, 'raw')
-
-const concat = async itr => {
-  const buffers = []
-  for await (const block of itr) {
-    buffers.push(block)
-  }
-  return Buffer.concat(buffers)
-}
-
-test('basic byteLink', async () => {
-  const { getBlock, put } = storage()
-  const types = createTypes({ getBlock })
-  const buffers = [buffer, buffer, buffer]
-  let bl
-  let blocks = 0
-  for await (const { block, root } of types.ByteLinks.fromArray(buffers)) {
-    if (block) {
-      blocks += 1
-      await put(block)
-    }
-    bl = root
-  }
-  assert.ok(bl)
-  assert.strictEqual(blocks, 3)
-
-  let iter = bl.read()
-  let str = (await concat(iter)).toString()
-  const fixture = Buffer.concat([buffer, buffer, buffer])
-  same(str, fixture.toString())
-  same(bl.length(), fixture.length)
-
-  iter = bl.read(0, 6)
-  str = (await concat(iter)).toString()
-  same(str, fixture.slice(0, 6).toString())
-
-  iter = bl.read(15, 18)
-  str = (await concat(iter)).toString()
-  same(str, fixture.slice(15, 18).toString())
-})
-
-const random = () => promisify(crypto.randomBytes)(4)
-
-test('nested byte tree', async () => {
-  const { getBlock, put } = storage()
-  const types = createTypes({ getBlock })
-  let i = 0
-  let buffers = []
-  while (i < 1000) {
-    buffers.push(random())
-    i++
-  }
-  buffers = await Promise.all(buffers)
-  const blocks = []
-  let data
-  for await (const { block, root } of types.Data.from(buffers, { maxListLength: 30 })) {
-    if (block) {
-      blocks.push(block)
-      await put(block)
-    }
-    data = root
-  }
-  const b1 = Buffer.concat(buffers).slice(51, 1300)
-  const b2 = await concat(data.read(51, 1300))
-  same(b1.length, b2.length)
-})
diff --git a/test/encode.spec.js b/test/encode.spec.js
deleted file mode 100644
index dbff164..0000000
--- a/test/encode.spec.js
+++ /dev/null
@@ -1,36 +0,0 @@
-'use strict'
-const assert = require('assert')
-const tsame = require('tsame')
-const { it } = require('mocha')
-const { encoder } = require('../')
-const path = require('path')
-
-const test = it
-
-const same = (x, y) => assert.ok(tsame(x, y))
-
-const fixture = path.join(__dirname, 'fixture')
-
-const parse = async p => {
-  const blocks = []
-  const counts = { raw: 0, 'dag-cbor': 0 }
-  const iter = await encoder.fromFileSystem(fixture)
-  for await (let { block, root } of iter) {
-    if (root) {
-      block = root.block()
-      same(root.constructor.name, 'Directory')
-    }
-    blocks.push(block)
-    counts[block.codec] += 1
-  }
-  return { blocks, counts }
-}
-
-test('basic encode', async () => {
-  const { blocks, counts } = await parse(fixture)
-  same(blocks.length, 39)
-  same(counts, { raw: 29, 'dag-cbor': 10 })
-  const last = blocks[blocks.length - 1]
-  const root = last.decode()
-  same(root.size, 3117)
-})
diff --git a/test/reader.spec.js b/test/reader.spec.js
deleted file mode 100644
index d1f2fc9..0000000
--- a/test/reader.spec.js
+++ /dev/null
@@ -1,55 +0,0 @@
-'use strict'
-const assert = require('assert')
-const tsame = require('tsame')
-const { it } = require('mocha')
-const { readFile } = require('fs').promises
-const path = require('path')
-const { encoder, reader } = require('../')
-
-const test = it
-
-const same = (x, y) => assert.ok(tsame(x, y))
-
-const fixture = path.join(__dirname, 'fixture')
-
-const parse = async p => {
-  const blocks = []
-  const db = {}
-  const counts = { raw: 0, 'dag-cbor': 0 }
-  for await (let { block, root } of encoder(fixture)) {
-    if (root) block = root.block()
-    db[(await block.cid()).toString('base32')] = block
-    blocks.push(block)
-    counts[block.codec] += 1
-  }
-  const getBlock = cid => db[cid.toString('base32')] || null
-  return { blocks, counts, getBlock }
-}
-
-test('basic reader', async () => {
-  const { blocks, getBlock } = await parse(fixture)
-  same(blocks.length, 39)
-  const last = blocks[blocks.length - 1]
-  const r = reader(last, getBlock)
-  const files = await r.ls('/')
-  same(files.sort(), ['bits', 'dir2', 'file1', 'file2', 'index.html', 'small.txt'].sort())
-
-  const sub = await r.ls('dir2')
-  same(sub, ['dir3'])
-  same(sub, await r.ls('/dir2'))
-})
-
-test('basic data read', async () => {
-  const { blocks, getBlock } = await parse(fixture)
-  same(blocks.length, 39)
-  const last = blocks[blocks.length - 1]
-  const r = reader(last, getBlock)
-
-  const buffers = []
-  const fileReader = r.read('index.html')
-  for await (const chunk of fileReader) {
-    buffers.push(chunk)
-  }
-  const buffer = Buffer.concat(buffers)
-  same(buffer.toString(), (await readFile(path.join(fixture, 'index.html'))).toString())
-})
diff --git a/test/test-basics.js b/test/test-basics.js
new file mode 100644
index 0000000..9a74225
--- /dev/null
+++ b/test/test-basics.js
@@ -0,0 +1,54 @@
+import Block from '@ipld/block/defaults'
+import { encoder, reader } from '../index.js'
+import { promises as fsAsync } from 'fs'
+import { deepStrictEqual as same } from 'assert'
+
+const store = () => {
+  const blocks = {}
+  const get = async cid => {
+    if (!cid) throw new Error('Not CID')
+    if (!blocks[cid.toString()]) throw new Error('not found')
+    return blocks[cid.toString()]
+  }
+  const put = async block => {
+    const cid = await block.cid()
+    blocks[cid.toString()] = block
+  }
+  return { get, put }
+}
+
+const collect = async iter => {
+  const results = []
+  for await (const entry of iter) {
+    results.push(entry)
+  }
+  return results
+}
+
+export default async test => {
+  test('single file', async test => {
+    const u = new URL('fixture/small.txt', import.meta.url)
+    const { get, put } = store()
+    let last
+    for await (const block of encoder(Block, u)) {
+      last = block
+      await put(block)
+    }
+    const fs = reader(await last.cid(), get)
+    const buffers = await collect(fs.read())
+    same(await fsAsync.readFile(u), Buffer.concat(buffers))
+  })
+  test('full directory', async test => {
+    const u = new URL('fixture', import.meta.url)
+    const { get, put } = store()
+    let last
+    for await (const block of encoder(Block, u)) {
+      last = block
+      await put(block)
+    }
+    const fs = reader(await last.cid(), get)
+    const buffers = await collect(fs.read('small.txt'))
+    const local = new URL('fixture/small.txt', import.meta.url)
+    same(await fsAsync.readFile(local), Buffer.concat(buffers))
+  })
+}

From d6a584492fcb863dbc04203e3cc0883bfcbf21d1 Mon Sep 17 00:00:00 2001
From: Mikeal Rogers <mikeal.rogers@gmail.com>
Date: Fri, 11 Sep 2020 16:38:09 +0000
Subject: [PATCH 2/6] fix: deep files and directories

---
 index.js            | 23 +++++++++++------------
 test/test-basics.js | 13 +++++++++++--
 2 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/index.js b/index.js
index b32cdf0..42d9969 100644
--- a/index.js
+++ b/index.js
@@ -54,18 +54,15 @@ const encoder = async function * (Block, path, chunker) {
 }
 
 const readFile = async function * (reader, parts, start, end) {
-  const block = await reader.traverse(parts)
-  yield * fbl.read(block, reader.get, start, end)
+  const link = await reader.traverse(parts)
+  yield * fbl.read(link, reader.get, start, end)
 }
 
 const toString = b => (new TextDecoder()).decode(b)
 
 const lsDirectory = async function * (reader, parts) {
-  const block = await reader.traverse(parts)
-  const decoded = block.decodeUnsafe()
-  if (!decoded.content) throw new Error('Not a valid DirEnt')
-  if (!decoded.content.d) throw new Error('Not a file')
-  for await (const { key } of hamt.all(decoded.content.d, reader.get, start, end)) {
+  let link = await reader.traverse(parts)
+  for await (const { key } of hamt.all(link, reader.get)) {
     yield toString(key)
   }
 }
@@ -80,16 +77,18 @@ class Reader {
     if (!parts.length) {
       const { d, f } = head.decodeUnsafe().content
       return d || f
-    }
-    while (parts.length) {
-      const key = parts.shift()
+    } else {
       const decoded = head.decodeUnsafe()
       if (!decoded.content) throw new Error('Not a valid DirEnt')
       if (!decoded.content.d) throw new Error('Not a directory')
-      const dirEnt = await hamt.get(decoded.content.d, key, this.get)
+      head = decoded.content.d
+    }
+    while (parts.length) {
+      const key = parts.shift()
+      const dirEnt = await hamt.get(head, key, this.get)
       const { d, f } = dirEnt.content
       if (f && parts.length) throw new Error(`${key} is not a directory`)
-      head = await this.get(d || f)
+      head = d || f
     }
     return head
   }
diff --git a/test/test-basics.js b/test/test-basics.js
index 9a74225..e5cf7fa 100644
--- a/test/test-basics.js
+++ b/test/test-basics.js
@@ -47,8 +47,17 @@ export default async test => {
       await put(block)
     }
     const fs = reader(await last.cid(), get)
-    const buffers = await collect(fs.read('small.txt'))
-    const local = new URL('fixture/small.txt', import.meta.url)
+    let buffers = await collect(fs.read('small.txt'))
+    let local = new URL('fixture/small.txt', import.meta.url)
+    same(await fsAsync.readFile(local), Buffer.concat(buffers))
+    let files = await collect(fs.ls())
+    same(files, [ 'small.txt', 'dir2', 'index.html', 'file2', 'file1', 'bits' ])
+    files = await collect(fs.ls('dir2'))
+    same(files, [ 'dir3' ])
+    files = await collect(fs.ls('dir2/dir3'))
+    same(files, [ 'file3', 'index.html' ])
+    buffers = await collect(fs.read('dir2/dir3/index.html'))
+    local = new URL('fixture/dir2/dir3/index.html', import.meta.url)
     same(await fsAsync.readFile(local), Buffer.concat(buffers))
   })
 }

From 88d41399decc782a36d8ccdef90ba9ef5679ccec Mon Sep 17 00:00:00 2001
From: Mikeal Rogers <mikeal.rogers@gmail.com>
Date: Fri, 11 Sep 2020 17:11:41 +0000
Subject: [PATCH 3/6] fix: test all files and directories

---
 package.json        | 12 +++++++++---
 test/test-basics.js | 29 +++++++++++++++++------------
 2 files changed, 26 insertions(+), 15 deletions(-)

diff --git a/package.json b/package.json
index 3490db1..ae12f00 100644
--- a/package.json
+++ b/package.json
@@ -3,12 +3,16 @@
   "version": "0.0.0-dev",
   "description": "Implementation of UnixFSv2.",
   "type": "module",
-  "scripts": {},
+  "scripts": {
+    "build": "npm_config_yes=true npx ipjs@latest build --tests",
+    "publish": "npm_config_yes=true npx ipjs@latest publish",
+    "test": "estest test/test-*.js",
+    "coverage": "c8 --reporter=html estest test/test-*.js && npx st -d coverage -p 8080"
+  },
   "keywords": [],
   "author": "Mikeal Rogers <mikeal.rogers@gmail.com> (http://www.mikealrogers.com)",
   "dependencies": {
     "@ipld/fbl": "^2.0.1",
-    "@ipld/block": "^6.0.3",
     "hamt-utils": "^0.0.4"
   },
   "directories": {
@@ -24,6 +28,8 @@
   },
   "homepage": "https://github.com/ipld/js-unixfsv2#readme",
   "devDependencies": {
-    "estest": "^10.3.5"
+    "@ipld/block": "^6.0.3",
+    "estest": "^10.3.5",
+    "hundreds": "^0.0.8"
   }
 }
diff --git a/test/test-basics.js b/test/test-basics.js
index e5cf7fa..f2b9a2c 100644
--- a/test/test-basics.js
+++ b/test/test-basics.js
@@ -47,17 +47,22 @@ export default async test => {
       await put(block)
     }
     const fs = reader(await last.cid(), get)
-    let buffers = await collect(fs.read('small.txt'))
-    let local = new URL('fixture/small.txt', import.meta.url)
-    same(await fsAsync.readFile(local), Buffer.concat(buffers))
-    let files = await collect(fs.ls())
-    same(files, [ 'small.txt', 'dir2', 'index.html', 'file2', 'file1', 'bits' ])
-    files = await collect(fs.ls('dir2'))
-    same(files, [ 'dir3' ])
-    files = await collect(fs.ls('dir2/dir3'))
-    same(files, [ 'file3', 'index.html' ])
-    buffers = await collect(fs.read('dir2/dir3/index.html'))
-    local = new URL('fixture/dir2/dir3/index.html', import.meta.url)
-    same(await fsAsync.readFile(local), Buffer.concat(buffers))
+
+    const testDir = async (dir, target) => {
+      const ls = await fsAsync.readdir(dir)
+      const files = await collect(fs.ls(target))
+      same(ls.sort(), files.sort())
+      for (const file of files) {
+        const u = new URL(file, dir + '/sub')
+        const stat = await fsAsync.stat(u)
+        if (stat.isDirectory()) {
+          await testDir(u, target + '/' + file)
+        } else {
+          const buffers = await collect(fs.read(target + '/' + file))
+          same(await fsAsync.readFile(u), Buffer.concat(buffers))
+        }
+      }
+    }
+    await testDir(new URL('fixture', import.meta.url), '')
   })
 }

From b3d709009436d75b7115a47fa7204b29ad6d075c Mon Sep 17 00:00:00 2001
From: Mikeal Rogers <mikeal.rogers@gmail.com>
Date: Fri, 11 Sep 2020 17:57:49 +0000
Subject: [PATCH 4/6] fix: full coverage

---
 index.js            |  7 +++++--
 package.json        |  2 +-
 test/test-basics.js | 25 +++++++++++++++++++++++++
 3 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/index.js b/index.js
index 42d9969..6f6b619 100644
--- a/index.js
+++ b/index.js
@@ -79,8 +79,11 @@ class Reader {
       return d || f
     } else {
       const decoded = head.decodeUnsafe()
-      if (!decoded.content) throw new Error('Not a valid DirEnt')
-      if (!decoded.content.d) throw new Error('Not a directory')
+      // TODO: replace with proper schema validation once we have a schema
+      /* c8 ignore next */
+      if (!decoded.content) /* c8 ignore next */ throw new Error('Not a valid DirEnt')
+      /* c8 ignore next */
+      if (!decoded.content.d) /* c8 ignore next */ throw new Error('Not a directory')
       head = decoded.content.d
     }
     while (parts.length) {
diff --git a/package.json b/package.json
index ae12f00..64a0788 100644
--- a/package.json
+++ b/package.json
@@ -6,7 +6,7 @@
   "scripts": {
     "build": "npm_config_yes=true npx ipjs@latest build --tests",
     "publish": "npm_config_yes=true npx ipjs@latest publish",
-    "test": "estest test/test-*.js",
+    "test": "hundreds estest test/test-*.js",
     "coverage": "c8 --reporter=html estest test/test-*.js && npx st -d coverage -p 8080"
   },
   "keywords": [],
diff --git a/test/test-basics.js b/test/test-basics.js
index f2b9a2c..204e964 100644
--- a/test/test-basics.js
+++ b/test/test-basics.js
@@ -65,4 +65,29 @@ export default async test => {
     }
     await testDir(new URL('fixture', import.meta.url), '')
   })
+  test('errors', async test => {
+    const u = new URL('fixture', import.meta.url)
+    const { get, put } = store()
+    let last
+    for await (const block of encoder(Block, u)) {
+      last = block
+      await put(block)
+    }
+    const fs = reader(await last.cid(), get)
+    let threw = true
+    try {
+      await collect(fs.ls('dir2/dir3/index.html/small.txt'))
+      threw = false
+    } catch (e) {
+      if (e.message !== 'index.html is not a directory') throw e
+    }
+    same(threw, true)
+    try {
+      await collect(fs.read('small.txt/nope'))
+      threw = false
+    } catch (e) {
+      if (e.message !== 'small.txt is not a directory') throw e
+    }
+    same(threw, true)
+  })
 }

From a1d14f5b7810def2b1dc3c1e0824f92c82234d3d Mon Sep 17 00:00:00 2001
From: Mikeal Rogers <mikeal.rogers@gmail.com>
Date: Fri, 11 Sep 2020 19:06:34 +0000
Subject: [PATCH 5/6] fix: disable coverage on v12

---
 package.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/package.json b/package.json
index 64a0788..3bb5c5f 100644
--- a/package.json
+++ b/package.json
@@ -7,6 +7,7 @@
     "build": "npm_config_yes=true npx ipjs@latest build --tests",
     "publish": "npm_config_yes=true npx ipjs@latest publish",
     "test": "hundreds estest test/test-*.js",
+    "test:node-v12": "estest test/test-*.js",
     "coverage": "c8 --reporter=html estest test/test-*.js && npx st -d coverage -p 8080"
   },
   "keywords": [],

From 35f9fc62a5c0017e6a9dbfa27df8084665b47c97 Mon Sep 17 00:00:00 2001
From: Mikeal Rogers <mikeal.rogers@gmail.com>
Date: Fri, 11 Sep 2020 19:19:48 +0000
Subject: [PATCH 6/6] fix: add max file size

---
 index.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/index.js b/index.js
index 6f6b619..806593e 100644
--- a/index.js
+++ b/index.js
@@ -21,6 +21,9 @@ const encode = async function * (Block, path, chunker=onemeg) {
     const files = await fs.readdir(path)
     const dir = {}
     for (const file of files) {
+      // Can't test this because this is the linux max file size already
+      /* c8 ignore next */
+      if (file.length > 255) /* c8 ignore next */ throw new Error('file is over max filesize')
       let last
       for await (const block of encode(Block, new URL(file, path + '/sub'), chunker)) {
         if (last) yield last