Skip to content
This repository has been archived by the owner on Dec 6, 2022. It is now read-only.

Switch to @ipld/multiformats #7

Merged
merged 23 commits into from
Aug 28, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,5 @@ node_modules/
.nyc_output/
coverage/
example.car
build/
dist/
133 changes: 84 additions & 49 deletions README.md

Large diffs are not rendered by default.

30 changes: 21 additions & 9 deletions car-browser.js
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
const { NoWriter } = require('./lib/reader-writer-iface')
const { createBufferReader } = require('./lib/reader-browser')
const CarDatastore = require('./datastore')
import { NoWriter } from './lib/reader-writer-iface.js'
import { createBufferReader } from './lib/reader-browser.js'
import CarDatastore from './datastore.js'

/**
* @name CarDatastore.readBuffer
* @description
* Read a CarDatastore from a Buffer containing the contents of an existing
* Read a CarDatastore from a Uint8Array containing the contents of an existing
* CAR archive. Mutation operations (`put()`, `delete()` and `setRoots()`) are
* not available.
*
Expand All @@ -20,13 +20,25 @@ const CarDatastore = require('./datastore')
* @memberof CarDatastore
* @static
* @async
* @param {Buffer|Uint8Array} buffer the byte contents of a CAR archive
* @param {Uint8Array} buffer the byte contents of a CAR archive
* @returns {CarDatastore} a read-only CarDatastore.
*/
async function readBuffer (buffer) {
const reader = await createBufferReader(buffer)
async function readBuffer (multiformats, buffer) {
const reader = await createBufferReader(multiformats, buffer)
const writer = new NoWriter()
return new CarDatastore(reader, writer)
return new CarDatastore(multiformats, reader, writer)
}

module.exports.readBuffer = readBuffer
export default (multiformats) => {
function wrap (fn) {
return function (...args) {
return fn(multiformats, ...args)
}
}

return {
readBuffer: wrap(readBuffer)
}
}

export { readBuffer }
110 changes: 79 additions & 31 deletions car.js
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
const { Reader, NoWriter } = require('./lib/reader-writer-iface')
const { createStreamCompleteReader, createStreamingReader, createFileReader } = require('./lib/reader')
const createStreamWriter = require('./lib/writer-stream')
const CarDatastore = require('./datastore')
const { readBuffer } = require('./car-browser')
const { indexer, readRaw } = require('./lib/raw')
import { Reader, NoWriter } from './lib/reader-writer-iface.js'
import { createStreamCompleteReader, createStreamingReader, createFileReader, createFileIndexedReader } from './lib/reader.js'
import createStreamWriter from './lib/writer-stream.js'
import CarDatastore from './datastore.js'
import browser from './car-browser.js'
import { indexer, readRaw } from './lib/raw.js'

/**
* @name CarDatastore.readFileComplete
* @description
* Read a CAR archive from a file and return a CarDatastore. The CarDatastore
* returned will _only_ support read operations: `getRoots()`, `get()`, `has()`
* and `query()`. Caching makes `get()` and `has()` possible as the entire
* and `query()`. Caching makes `get()` and `has()`. This is possible as the entire
* file is read and decoded before the CarDatastore is returned. mutation
* operations (`put()`, `delete()` and `setRoots()`) are not available as there
* is no ability to modify the archive.
Expand All @@ -35,18 +35,18 @@ const { indexer, readRaw } = require('./lib/raw')
* @param {string} file a path to a file containing CAR archive data.
* @returns {CarDatastore} a read-only CarDatastore.
*/
async function readFileComplete (file) {
const reader = await createFileReader(file)
async function readFileComplete (multiformats, file) {
const reader = await createFileReader(multiformats, file)
const writer = new NoWriter()
return new CarDatastore(reader, writer)
return new CarDatastore(multiformats, reader, writer)
}

/**
* @name CarDatastore.readStreamComplete
* @description
* Read a CAR archive as a CarDataStore from a ReadableStream. The CarDatastore
* returned will _only_ support read operations: `getRoots()`, `get()`, `has()`
* and `query()`. Caching makes `get()` and `has()` possible as the entire
* and `query()`. Caching makes `get()` and `has()`. This is possible as the entire
* stream is read and decoded before the CarDatastore is returned. Mutation
* operations (`put()`, `delete()` and `setRoots()`) are not available as there
* is no ability to modify the archive.
Expand All @@ -65,10 +65,10 @@ async function readFileComplete (file) {
* archive as a binary stream.
* @returns {CarDatastore} a read-only CarDatastore.
*/
async function readStreamComplete (stream) {
const reader = await createStreamCompleteReader(stream)
async function readStreamComplete (multiformats, stream) {
const reader = await createStreamCompleteReader(multiformats, stream)
const writer = new NoWriter()
return new CarDatastore(reader, writer)
return new CarDatastore(multiformats, reader, writer)
}

/**
Expand Down Expand Up @@ -98,10 +98,46 @@ async function readStreamComplete (stream) {
* archive as a binary stream.
* @returns {CarDatastore} a read-only CarDatastore.
*/
async function readStreaming (stream) {
const reader = await createStreamingReader(stream)
async function readStreaming (multiformats, stream) {
const reader = await createStreamingReader(multiformats, stream)
const writer = new NoWriter()
return new CarDatastore(reader, writer)
return new CarDatastore(multiformats, reader, writer)
}

/**
* @name CarDatastore.readFileIndexed
* @description
* Read a CAR archive as a CarDataStore from a local file. The CarDatastore
* returned will _only_ support read operations: `getRoots()`, `get()`, `has()`
* and `query()`. Caching makes `get()` and `has()`. This is possible as the entire
* stream is read and _indexed_ before the CarDatastore is returned. Mutation
* operations (`put()`, `delete()` and `setRoots()`) are not available as there
* is no ability to modify the archive.
*
* The indexing operation uses {@link indexer} to catalogue the contents of the
* CAR and store a mapping of CID to byte locations for each entry. This method
* of parsing is not as memory intensive as {@link readStreamComplete} as only
* the index is stored in memory. When blocks are read, the index tells the
* reader where to fetch the block from within the CAR file.
*
* This mode is suitable for large files where random-access operations are
* required. Where a full sequential read is only required, use
* {@link createReadStreaming} which consumes the file in a single pass with no
* memory used for indexing.
*
* This create-mode is not available in the browser environment.
* @function
* @memberof CarDatastore
* @static
* @async
* @param {ReadableStream} stream a ReadableStream that provides an entire CAR
* archive as a binary stream.
* @returns {CarDatastore} a read-only CarDatastore.
*/
async function readFileIndexed (multiformats, filePath) {
const reader = await createFileIndexedReader(multiformats, filePath)
const writer = new NoWriter()
return new CarDatastore(multiformats, reader, writer)
}

/**
Expand Down Expand Up @@ -129,17 +165,17 @@ async function readStreaming (stream) {
* @param {WritableStream} stream a writable stream
* @returns {CarDatastore} an append-only, streaming CarDatastore.
*/
async function writeStream (stream) {
async function writeStream (multiformats, stream) {
const reader = new Reader()
const writer = await createStreamWriter(stream)
return new CarDatastore(reader, writer)
const writer = await createStreamWriter(multiformats, stream)
return new CarDatastore(multiformats, reader, writer)
}

async function traverseBlock (block, get, car, concurrency = 1, seen = new Set()) {
const cid = await block.cid()
await car.put(cid, block.encodeUnsafe())
seen.add(cid.toString('base58btc'))
if (cid.codec === 'raw') {
if (cid.code === 0x55) { // raw
return
}
const reader = block.reader()
Expand Down Expand Up @@ -168,8 +204,8 @@ async function traverseBlock (block, get, car, concurrency = 1, seen = new Set()
* @memberof CarDatastore
* @static
* @async
* @param {Block} root the root of the graph to start at, this block will be
* included in the CAR and its CID will be set as the single root.
* @param {CID} root the CID of the root of the graph to start at, this block
* will be included in the CAR and the CID will be set as the single root.
* @param {AsyncFunction} get an `async` function that takes a CID and returns
* a `Block`. Can be used to attach to an arbitrary data store.
* @param {CarDatastore} car a writable `CarDatastore` that has not yet been
Expand All @@ -184,11 +220,23 @@ async function completeGraph (root, get, car, concurrency) {
await car.close()
}

module.exports.readBuffer = readBuffer
module.exports.readFileComplete = readFileComplete
module.exports.readStreamComplete = readStreamComplete
module.exports.readStreaming = readStreaming
module.exports.writeStream = writeStream
module.exports.indexer = indexer
module.exports.readRaw = readRaw
module.exports.completeGraph = completeGraph
function create (multiformats) {
function wrap (fn) {
return function (...args) {
return fn(multiformats, ...args)
}
}

return Object.assign(browser(multiformats), {
readFileComplete: wrap(readFileComplete),
readStreamComplete: wrap(readStreamComplete),
readStreaming: wrap(readStreaming),
readFileIndexed: wrap(readFileIndexed),
writeStream: wrap(writeStream),
indexer: wrap(indexer),
readRaw,
completeGraph
})
}

export default create
34 changes: 22 additions & 12 deletions datastore.js
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
const { filter, map } = require('interface-datastore').utils
const { toKey } = require('./lib/util')
import interfaceDatastore from 'interface-datastore'

const { filter, map } = interfaceDatastore.utils

/**
* CarDatastore is a class to manage reading from, and writing to a CAR archives
* using [CID](https://github.com/multiformats/js-cid)s as keys and file names
* in the CAR and binary block data as the file contents.
* using [CID](https://github.com/multiformats/js-multiformats)s as keys and
* file names in the CAR and binary block data as the file contents.
*
* @class
*/
class CarDatastore {
constructor (reader, writer) {
constructor (multiformats, reader, writer) {
this.multiformats = multiformats
this._reader = reader
this._writer = writer
}
Expand All @@ -28,10 +30,10 @@ class CarDatastore {
* @memberof CarDatastore
* @param {string|Key|CID} key a `CID` or `CID`-convertable object to identify
* the block.
* @return {Buffer} the IPLD block data referenced by the CID.
* @return {Uint8Array} the IPLD block data referenced by the CID.
*/
async get (key) {
key = toKey(key, 'get')
key = toKey(this.multiformats, key, 'get')
return this._reader.get(key)
}

Expand All @@ -52,7 +54,7 @@ class CarDatastore {
* @return {boolean} indicating whether the key exists in this Datastore.
*/
async has (key) {
key = toKey(key, 'has')
key = toKey(this.multiformats, key, 'has')
return this._reader.has(key)
}

Expand All @@ -71,11 +73,11 @@ class CarDatastore {
* @memberof CarDatastore
* @param {string|Key|CID} key a `CID` or `CID`-convertable object to identify
* the `value`.
* @param {Buffer|Uint8Array} value an IPLD block matching the given `key`
* @param {Uint8Array} value an IPLD block matching the given `key`
* `CID`.
*/
async put (key, value) {
key = toKey(key, 'put')
key = toKey(this.multiformats, key, 'put')
if (!(value instanceof Uint8Array)) {
throw new TypeError('put() can only receive Uint8Arrays or Buffers')
}
Expand All @@ -94,7 +96,7 @@ class CarDatastore {
* the block.
*/
async delete (key) {
key = toKey(key, 'delete')
key = toKey(this.multiformats, key, 'delete')
return this._writer.delete(key)
}

Expand Down Expand Up @@ -224,4 +226,12 @@ class CarDatastore {
}
}

module.exports = CarDatastore
function toKey (multiformats, key, method) {
try {
return multiformats.CID.from(key.toString())
} catch (e) {
throw new TypeError(`${method}() only accepts CIDs or CID strings`)
}
}

export default CarDatastore
22 changes: 14 additions & 8 deletions example.js
Original file line number Diff line number Diff line change
@@ -1,18 +1,24 @@
const fs = require('fs')
const CarDatastore = require('./')
const Block = require('@ipld/block')
import fs from 'fs'
import multiformats from 'multiformats/basics'
import car from 'datastore-car'
import dagCbor from '@ipld/dag-cbor'

// dag-cbor is required for the CAR root block
multiformats.add(dagCbor)
const CarDatastore = car(multiformats)

async function example () {
const block = Block.encoder(Buffer.from('random meaningless bytes'), 'raw')
const cid = await block.cid()
const binary = new TextEncoder().encode('random meaningless bytes')
const mh = await multiformats.multihash.hash(binary, 'sha2-256')
const cid = multiformats.CID.create(1, multiformats.get('raw').code, mh)

const outStream = fs.createWriteStream('example.car')
const writeDs = await CarDatastore.writeStream(outStream)

// set the header with a single root
await writeDs.setRoots(cid)
// store a new block, creates a new file entry in the CAR archive
await writeDs.put(cid, await block.encode())
await writeDs.put(cid, binary)
await writeDs.close()

const inStream = fs.createReadStream('example.car')
Expand All @@ -25,10 +31,10 @@ async function example () {
const roots = await readDs.getRoots()
// retrieve a block, as a UInt8Array, reading from the ZIP archive
const got = await readDs.get(roots[0])
// also possible: for await (const {key, data} = readDs.query()) { ... }
// also possible: for await (const { key, value } of readDs.query()) { ... }

console.log('Retrieved [%s] from example.car with CID [%s]',
Buffer.from(got).toString(),
new TextDecoder().decode(got),
roots[0].toString())

await readDs.close()
Expand Down
Loading