diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 669964ac..dc4b94fc 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -141,8 +141,8 @@ "release": "aegir release" }, "dependencies": { - "@helia/car": "^3.0.0", "@helia/block-brokers": "^2.0.1", + "@helia/car": "^3.0.0", "@helia/http": "^1.0.1", "@helia/interface": "^4.0.0", "@helia/ipns": "^6.0.0", @@ -155,7 +155,11 @@ "@libp2p/peer-id": "^4.0.5", "cborg": "^4.0.9", "hashlru": "^2.3.0", + "interface-blockstore": "^5.2.10", "ipfs-unixfs-exporter": "^13.5.0", + "it-map": "^3.0.5", + "it-pipe": "^3.0.1", + "it-tar": "^6.0.4", "it-to-browser-readablestream": "^2.0.6", "multiformats": "^13.1.0", "progress-events": "^1.0.0" @@ -173,9 +177,12 @@ "@types/sinon": "^17.0.3", "aegir": "^42.2.2", "blockstore-core": "^4.4.0", + "browser-readablestream-to-it": "^2.0.5", "datastore-core": "^9.2.8", "helia": "^4.0.1", + "ipfs-unixfs-importer": "^15.2.4", "ipns": "^9.0.0", + "it-all": "^3.0.4", "it-last": "^3.0.4", "it-to-buffer": "^4.0.5", "magic-bytes.js": "^1.8.0", diff --git a/packages/verified-fetch/src/utils/get-tar-stream.ts b/packages/verified-fetch/src/utils/get-tar-stream.ts new file mode 100644 index 00000000..eeec0b4b --- /dev/null +++ b/packages/verified-fetch/src/utils/get-tar-stream.ts @@ -0,0 +1,68 @@ +import { CodeError } from '@libp2p/interface' +import { exporter, recursive, type UnixFSEntry } from 'ipfs-unixfs-exporter' +import map from 'it-map' +import { pipe } from 'it-pipe' +import { pack, type TarEntryHeader, type TarImportCandidate } from 'it-tar' +import type { AbortOptions } from '@libp2p/interface' +import type { Blockstore } from 'interface-blockstore' + +const EXPORTABLE = ['file', 'raw', 'directory'] + +function toHeader (file: UnixFSEntry): Partial & { name: string } { + let mode: number | undefined + let mtime: Date | undefined + + if (file.type === 'file' || file.type === 'directory') { + mode = file.unixfs.mode + mtime = file.unixfs.mtime != null ? new Date(Number(file.unixfs.mtime.secs * 1000n)) : undefined + } + + return { + name: file.path, + mode, + mtime, + size: Number(file.size), + type: file.type === 'directory' ? 'directory' : 'file' + } +} + +function toTarImportCandidate (entry: UnixFSEntry): TarImportCandidate { + if (!EXPORTABLE.includes(entry.type)) { + throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS') + } + + const candidate: TarImportCandidate = { + header: toHeader(entry) + } + + if (entry.type === 'file' || entry.type === 'raw') { + candidate.body = entry.content() + } + + return candidate +} + +export async function * tarStream (ipfsPath: string, blockstore: Blockstore, options?: AbortOptions): AsyncGenerator { + const file = await exporter(ipfsPath, blockstore, options) + + if (file.type === 'file' || file.type === 'raw') { + yield * pipe( + [toTarImportCandidate(file)], + pack() + ) + + return + } + + if (file.type === 'directory') { + yield * pipe( + recursive(ipfsPath, blockstore, options), + (source) => map(source, (entry) => toTarImportCandidate(entry)), + pack() + ) + + return + } + + throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS') +} diff --git a/packages/verified-fetch/src/utils/select-output-type.ts b/packages/verified-fetch/src/utils/select-output-type.ts index 2c2cf959..0f7c40c9 100644 --- a/packages/verified-fetch/src/utils/select-output-type.ts +++ b/packages/verified-fetch/src/utils/select-output-type.ts @@ -55,7 +55,8 @@ const CID_TYPE_MAP: Record = { 'application/octet-stream', 'application/vnd.ipld.raw', 'application/vnd.ipfs.ipns-record', - 'application/vnd.ipld.car' + 'application/vnd.ipld.car', + 'application/x-tar' ] } diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index 8af88976..b7817ffd 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -14,6 +14,7 @@ import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js' import { getContentDispositionFilename } from './utils/get-content-disposition-filename.js' import { getETag } from './utils/get-e-tag.js' import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js' +import { tarStream } from './utils/get-tar-stream.js' import { parseResource } from './utils/parse-resource.js' import { notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js' import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js' @@ -151,11 +152,16 @@ export class VerifiedFetch { * directory structure referenced by the `CID`. */ private async handleTar ({ cid, path, options }: FetchHandlerFunctionArg): Promise { - if (cid.code !== dagPbCode) { - return notAcceptableResponse('only dag-pb CIDs can be returned in TAR files') + if (cid.code !== dagPbCode && cid.code !== rawCode) { + return notAcceptableResponse('only UnixFS data can be returned in a TAR file') } - return notSupportedResponse('application/tar support is not implemented') + const stream = toBrowserReadableStream(tarStream(`/ipfs/${cid}/${path}`, this.helia.blockstore, options)) + + const response = okResponse(stream) + response.headers.set('content-type', 'application/x-tar') + + return response } private async handleJson ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise { @@ -397,6 +403,8 @@ export class VerifiedFetch { } else if (accept === 'application/x-tar') { // the user requested a TAR file reqFormat = 'tar' + query.download = true + query.filename = query.filename ?? `${cid.toString()}.tar` response = await this.handleTar({ cid, path, options }) } else { // derive the handler from the CID type diff --git a/packages/verified-fetch/test/tar.spec.ts b/packages/verified-fetch/test/tar.spec.ts new file mode 100644 index 00000000..5b3dc091 --- /dev/null +++ b/packages/verified-fetch/test/tar.spec.ts @@ -0,0 +1,149 @@ +import { unixfs } from '@helia/unixfs' +import { stop } from '@libp2p/interface' +import { expect } from 'aegir/chai' +import browserReadableStreamToIt from 'browser-readablestream-to-it' +import all from 'it-all' +import last from 'it-last' +import { pipe } from 'it-pipe' +import { extract } from 'it-tar' +import toBuffer from 'it-to-buffer' +import { VerifiedFetch } from '../src/verified-fetch.js' +import { createHelia } from './fixtures/create-offline-helia.js' +import type { Helia } from '@helia/interface' +import type { FileCandidate } from 'ipfs-unixfs-importer' + +describe('tar files', () => { + let helia: Helia + let verifiedFetch: VerifiedFetch + + beforeEach(async () => { + helia = await createHelia() + verifiedFetch = new VerifiedFetch({ + helia + }) + }) + + afterEach(async () => { + await stop(helia, verifiedFetch) + }) + + it('should support fetching a TAR file', async () => { + const file = Uint8Array.from([0, 1, 2, 3, 4]) + const fs = unixfs(helia) + const cid = await fs.addBytes(file) + + const resp = await verifiedFetch.fetch(cid, { + headers: { + accept: 'application/x-tar' + } + }) + expect(resp.status).to.equal(200) + expect(resp.headers.get('content-type')).to.equal('application/x-tar') + expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${cid.toString()}.tar"`) + + if (resp.body == null) { + throw new Error('Download failed') + } + + const entries = await pipe( + browserReadableStreamToIt(resp.body), + extract(), + async source => all(source) + ) + + expect(entries).to.have.lengthOf(1) + await expect(toBuffer(entries[0].body)).to.eventually.deep.equal(file) + }) + + it('should support fetching a TAR file containing a directory', async () => { + const directory: FileCandidate[] = [{ + path: 'foo.txt', + content: Uint8Array.from([0, 1, 2, 3, 4]) + }, { + path: 'bar.txt', + content: Uint8Array.from([5, 6, 7, 8, 9]) + }, { + path: 'baz/qux.txt', + content: Uint8Array.from([1, 2, 3, 4, 5]) + }] + + const fs = unixfs(helia) + const importResult = await last(fs.addAll(directory, { + wrapWithDirectory: true + })) + + if (importResult == null) { + throw new Error('Import failed') + } + + const resp = await verifiedFetch.fetch(importResult.cid, { + headers: { + accept: 'application/x-tar' + } + }) + expect(resp.status).to.equal(200) + expect(resp.headers.get('content-type')).to.equal('application/x-tar') + expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${importResult.cid.toString()}.tar"`) + + if (resp.body == null) { + throw new Error('Download failed') + } + + const entries = await pipe( + browserReadableStreamToIt(resp.body), + extract(), + async source => all(source) + ) + + expect(entries).to.have.lengthOf(5) + expect(entries[0]).to.have.nested.property('header.name', importResult.cid.toString()) + + expect(entries[1]).to.have.nested.property('header.name', `${importResult.cid}/${directory[1].path}`) + await expect(toBuffer(entries[1].body)).to.eventually.deep.equal(directory[1].content) + + expect(entries[2]).to.have.nested.property('header.name', `${importResult.cid}/${directory[2].path?.split('/')[0]}`) + + expect(entries[3]).to.have.nested.property('header.name', `${importResult.cid}/${directory[2].path}`) + await expect(toBuffer(entries[3].body)).to.eventually.deep.equal(directory[2].content) + + expect(entries[4]).to.have.nested.property('header.name', `${importResult.cid}/${directory[0].path}`) + await expect(toBuffer(entries[4].body)).to.eventually.deep.equal(directory[0].content) + }) + + it('should support fetching a TAR file by format', async () => { + const file = Uint8Array.from([0, 1, 2, 3, 4]) + const fs = unixfs(helia) + const cid = await fs.addBytes(file) + + const resp = await verifiedFetch.fetch(`ipfs://${cid}?format=tar`) + expect(resp.status).to.equal(200) + expect(resp.headers.get('content-type')).to.equal('application/x-tar') + expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${cid.toString()}.tar"`) + }) + + it('should support specifying a filename for a TAR file', async () => { + const file = Uint8Array.from([0, 1, 2, 3, 4]) + const fs = unixfs(helia) + const cid = await fs.addBytes(file) + + const resp = await verifiedFetch.fetch(`ipfs://${cid}?filename=foo.bar`, { + headers: { + accept: 'application/x-tar' + } + }) + expect(resp.status).to.equal(200) + expect(resp.headers.get('content-type')).to.equal('application/x-tar') + expect(resp.headers.get('content-disposition')).to.equal('attachment; filename="foo.bar"') + }) + + it('should support fetching a TAR file by format with a filename', async () => { + const file = Uint8Array.from([0, 1, 2, 3, 4]) + const fs = unixfs(helia) + const cid = await fs.addBytes(file) + + const resp = await verifiedFetch.fetch(`ipfs://${cid}?format=tar&filename=foo.bar`) + expect(resp.status).to.equal(200) + expect(resp.headers.get('content-type')).to.equal('application/x-tar') + expect(resp.headers.get('content-disposition')).to.equal('attachment; filename="foo.bar"') + }) +}) diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 56d0e38a..57d1fd4e 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -80,7 +80,7 @@ describe('@helia/verifed-fetch', () => { }) const formatsAndAcceptHeaders = [ - ['tar', 'application/x-tar'] + ['ipns-record', 'application/vnd.ipfs.ipns-record'] ] for (const [format, acceptHeader] of formatsAndAcceptHeaders) {