-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: download tars from @helia/verified-fetch (#442)
Adds support for downloading tar archives of UnixFS directories --------- Co-authored-by: Russell Dempsey <[email protected]>
- Loading branch information
1 parent
703980c
commit 70ddd00
Showing
6 changed files
with
239 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| import { CodeError } from '@libp2p/interface' | ||
| import { exporter, recursive, type UnixFSEntry } from 'ipfs-unixfs-exporter' | ||
| import map from 'it-map' | ||
| import { pipe } from 'it-pipe' | ||
| import { pack, type TarEntryHeader, type TarImportCandidate } from 'it-tar' | ||
| import type { AbortOptions } from '@libp2p/interface' | ||
| import type { Blockstore } from 'interface-blockstore' | ||
|
|
||
| const EXPORTABLE = ['file', 'raw', 'directory'] | ||
|
|
||
| function toHeader (file: UnixFSEntry): Partial<TarEntryHeader> & { name: string } { | ||
| let mode: number | undefined | ||
| let mtime: Date | undefined | ||
|
|
||
| if (file.type === 'file' || file.type === 'directory') { | ||
| mode = file.unixfs.mode | ||
| mtime = file.unixfs.mtime != null ? new Date(Number(file.unixfs.mtime.secs * 1000n)) : undefined | ||
| } | ||
|
|
||
| return { | ||
| name: file.path, | ||
| mode, | ||
| mtime, | ||
| size: Number(file.size), | ||
| type: file.type === 'directory' ? 'directory' : 'file' | ||
| } | ||
| } | ||
|
|
||
| function toTarImportCandidate (entry: UnixFSEntry): TarImportCandidate { | ||
| if (!EXPORTABLE.includes(entry.type)) { | ||
| throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS') | ||
| } | ||
|
|
||
| const candidate: TarImportCandidate = { | ||
| header: toHeader(entry) | ||
| } | ||
|
|
||
| if (entry.type === 'file' || entry.type === 'raw') { | ||
| candidate.body = entry.content() | ||
| } | ||
|
|
||
| return candidate | ||
| } | ||
|
|
||
| export async function * tarStream (ipfsPath: string, blockstore: Blockstore, options?: AbortOptions): AsyncGenerator<Uint8Array> { | ||
| const file = await exporter(ipfsPath, blockstore, options) | ||
|
|
||
| if (file.type === 'file' || file.type === 'raw') { | ||
| yield * pipe( | ||
| [toTarImportCandidate(file)], | ||
| pack() | ||
| ) | ||
|
|
||
| return | ||
| } | ||
|
|
||
| if (file.type === 'directory') { | ||
| yield * pipe( | ||
| recursive(ipfsPath, blockstore, options), | ||
| (source) => map(source, (entry) => toTarImportCandidate(entry)), | ||
| pack() | ||
| ) | ||
|
|
||
| return | ||
| } | ||
|
|
||
| throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS') | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,149 @@ | ||
| import { unixfs } from '@helia/unixfs' | ||
| import { stop } from '@libp2p/interface' | ||
| import { expect } from 'aegir/chai' | ||
| import browserReadableStreamToIt from 'browser-readablestream-to-it' | ||
| import all from 'it-all' | ||
| import last from 'it-last' | ||
| import { pipe } from 'it-pipe' | ||
| import { extract } from 'it-tar' | ||
| import toBuffer from 'it-to-buffer' | ||
| import { VerifiedFetch } from '../src/verified-fetch.js' | ||
| import { createHelia } from './fixtures/create-offline-helia.js' | ||
| import type { Helia } from '@helia/interface' | ||
| import type { FileCandidate } from 'ipfs-unixfs-importer' | ||
|
|
||
| describe('tar files', () => { | ||
| let helia: Helia | ||
| let verifiedFetch: VerifiedFetch | ||
|
|
||
| beforeEach(async () => { | ||
| helia = await createHelia() | ||
| verifiedFetch = new VerifiedFetch({ | ||
| helia | ||
| }) | ||
| }) | ||
|
|
||
| afterEach(async () => { | ||
| await stop(helia, verifiedFetch) | ||
| }) | ||
|
|
||
| it('should support fetching a TAR file', async () => { | ||
| const file = Uint8Array.from([0, 1, 2, 3, 4]) | ||
| const fs = unixfs(helia) | ||
| const cid = await fs.addBytes(file) | ||
|
|
||
| const resp = await verifiedFetch.fetch(cid, { | ||
| headers: { | ||
| accept: 'application/x-tar' | ||
| } | ||
| }) | ||
| expect(resp.status).to.equal(200) | ||
| expect(resp.headers.get('content-type')).to.equal('application/x-tar') | ||
| expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${cid.toString()}.tar"`) | ||
|
|
||
| if (resp.body == null) { | ||
| throw new Error('Download failed') | ||
| } | ||
|
|
||
| const entries = await pipe( | ||
| browserReadableStreamToIt(resp.body), | ||
| extract(), | ||
| async source => all(source) | ||
| ) | ||
|
|
||
| expect(entries).to.have.lengthOf(1) | ||
| await expect(toBuffer(entries[0].body)).to.eventually.deep.equal(file) | ||
| }) | ||
|
|
||
| it('should support fetching a TAR file containing a directory', async () => { | ||
| const directory: FileCandidate[] = [{ | ||
| path: 'foo.txt', | ||
| content: Uint8Array.from([0, 1, 2, 3, 4]) | ||
| }, { | ||
| path: 'bar.txt', | ||
| content: Uint8Array.from([5, 6, 7, 8, 9]) | ||
| }, { | ||
| path: 'baz/qux.txt', | ||
| content: Uint8Array.from([1, 2, 3, 4, 5]) | ||
| }] | ||
|
|
||
| const fs = unixfs(helia) | ||
| const importResult = await last(fs.addAll(directory, { | ||
| wrapWithDirectory: true | ||
| })) | ||
|
|
||
| if (importResult == null) { | ||
| throw new Error('Import failed') | ||
| } | ||
|
|
||
| const resp = await verifiedFetch.fetch(importResult.cid, { | ||
| headers: { | ||
| accept: 'application/x-tar' | ||
| } | ||
| }) | ||
| expect(resp.status).to.equal(200) | ||
| expect(resp.headers.get('content-type')).to.equal('application/x-tar') | ||
| expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${importResult.cid.toString()}.tar"`) | ||
|
|
||
| if (resp.body == null) { | ||
| throw new Error('Download failed') | ||
| } | ||
|
|
||
| const entries = await pipe( | ||
| browserReadableStreamToIt(resp.body), | ||
| extract(), | ||
| async source => all(source) | ||
| ) | ||
|
|
||
| expect(entries).to.have.lengthOf(5) | ||
| expect(entries[0]).to.have.nested.property('header.name', importResult.cid.toString()) | ||
|
|
||
| expect(entries[1]).to.have.nested.property('header.name', `${importResult.cid}/${directory[1].path}`) | ||
| await expect(toBuffer(entries[1].body)).to.eventually.deep.equal(directory[1].content) | ||
|
|
||
| expect(entries[2]).to.have.nested.property('header.name', `${importResult.cid}/${directory[2].path?.split('/')[0]}`) | ||
|
|
||
| expect(entries[3]).to.have.nested.property('header.name', `${importResult.cid}/${directory[2].path}`) | ||
| await expect(toBuffer(entries[3].body)).to.eventually.deep.equal(directory[2].content) | ||
|
|
||
| expect(entries[4]).to.have.nested.property('header.name', `${importResult.cid}/${directory[0].path}`) | ||
| await expect(toBuffer(entries[4].body)).to.eventually.deep.equal(directory[0].content) | ||
| }) | ||
|
|
||
| it('should support fetching a TAR file by format', async () => { | ||
| const file = Uint8Array.from([0, 1, 2, 3, 4]) | ||
| const fs = unixfs(helia) | ||
| const cid = await fs.addBytes(file) | ||
|
|
||
| const resp = await verifiedFetch.fetch(`ipfs://${cid}?format=tar`) | ||
| expect(resp.status).to.equal(200) | ||
| expect(resp.headers.get('content-type')).to.equal('application/x-tar') | ||
| expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${cid.toString()}.tar"`) | ||
| }) | ||
|
|
||
| it('should support specifying a filename for a TAR file', async () => { | ||
| const file = Uint8Array.from([0, 1, 2, 3, 4]) | ||
| const fs = unixfs(helia) | ||
| const cid = await fs.addBytes(file) | ||
|
|
||
| const resp = await verifiedFetch.fetch(`ipfs://${cid}?filename=foo.bar`, { | ||
| headers: { | ||
| accept: 'application/x-tar' | ||
| } | ||
| }) | ||
| expect(resp.status).to.equal(200) | ||
| expect(resp.headers.get('content-type')).to.equal('application/x-tar') | ||
| expect(resp.headers.get('content-disposition')).to.equal('attachment; filename="foo.bar"') | ||
| }) | ||
|
|
||
| it('should support fetching a TAR file by format with a filename', async () => { | ||
| const file = Uint8Array.from([0, 1, 2, 3, 4]) | ||
| const fs = unixfs(helia) | ||
| const cid = await fs.addBytes(file) | ||
|
|
||
| const resp = await verifiedFetch.fetch(`ipfs://${cid}?format=tar&filename=foo.bar`) | ||
| expect(resp.status).to.equal(200) | ||
| expect(resp.headers.get('content-type')).to.equal('application/x-tar') | ||
| expect(resp.headers.get('content-disposition')).to.equal('attachment; filename="foo.bar"') | ||
| }) | ||
| }) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters