feat: download tars from @helia/verified-fetch

packages/verified-fetch/README.md

@@ -347,6 +347,39 @@ if (res.headers.get('Content-Type') === 'application/json') {
 console.info(obj) // ...
 ```
 
+## The `Accept` header
+
+The `Accept` header can be passed to override certain response processing, or to ensure that the final `Content-Type` of the response is the one that is expected.
+
+If the final `Content-Type` does not match the `Accept` header, or if the content cannot be represented in the format dictated by the `Accept` header, or you have configured a custom content type parser, and that parser returns a value that isn't in the accept header, a [406: Not Acceptible](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/406) response will be returned:
+
+```typescript
+import { verifiedFetch } from '@helia/verified-fetch'
+
+const res = await verifiedFetch('ipfs://bafyJPEGImageCID', {
+  headers: {
+    accept: 'image/png'
+  }
+})
+
+console.info(res.status) // 406 - the image was a JPEG but we specified PNG as the accept header
+```
+
+It can also be used to skip processing the data from some formats such as `DAG-CBOR` if you wish to handle decoding it yourself:
+
+```typescript
+import { verifiedFetch } from '@helia/verified-fetch'
+
+const res = await verifiedFetch('ipfs://bafyDAGCBORCID', {
+  headers: {
+    accept: 'application/octet-stream'
+  }
+})
+
+console.info(res.headers.get('accept')) // application/octet-stream
+const buf = await res.arrayBuffer() // raw bytes, not processed as JSON
+```
+
 ## Comparison to fetch
 
 This module attempts to act as similarly to the `fetch()` API as possible.

packages/verified-fetch/package.json

@@ -142,19 +142,26 @@
   },
   "dependencies": {
     "@helia/block-brokers": "^2.0.1",
+    "@helia/car": "^3.0.0",
     "@helia/http": "^1.0.1",
     "@helia/interface": "^4.0.0",
     "@helia/ipns": "^6.0.0",
     "@helia/routers": "^1.0.0",
     "@helia/unixfs": "^3.0.0",
+    "@ipld/car": "^5.2.6",
     "@ipld/dag-cbor": "^9.2.0",
     "@ipld/dag-json": "^10.2.0",
     "@ipld/dag-pb": "^4.1.0",
     "@libp2p/interface": "^1.1.2",
     "@libp2p/peer-id": "^4.0.5",
     "cborg": "^4.0.9",
     "hashlru": "^2.3.0",
+    "interface-blockstore": "^5.2.10",
     "ipfs-unixfs-exporter": "^13.5.0",
+    "it-map": "^3.0.5",
+    "it-pipe": "^3.0.1",
+    "it-tar": "^6.0.4",
+    "it-to-browser-readablestream": "^2.0.6",
     "multiformats": "^13.1.0",
     "progress-events": "^1.0.0"
   },
@@ -169,11 +176,16 @@
     "@types/sinon": "^17.0.3",
     "aegir": "^42.2.2",
     "blockstore-core": "^4.4.0",
+    "browser-readablestream-to-it": "^2.0.5",
     "datastore-core": "^9.2.8",
     "helia": "^4.0.1",
+    "ipfs-unixfs-importer": "^15.2.4",
+    "ipns": "^9.0.0",
+    "it-all": "^3.0.4",
     "it-last": "^3.0.4",
     "it-to-buffer": "^4.0.5",
     "magic-bytes.js": "^1.8.0",
+    "p-defer": "^4.0.0",
     "sinon": "^17.0.1",
     "sinon-ts": "^2.0.0",
     "uint8arrays": "^5.0.1"

packages/verified-fetch/src/index.ts

@@ -320,6 +320,39 @@
  * console.info(obj) // ...
  * ```
  *
+ * ## The `Accept` header
+ *
+ * The `Accept` header can be passed to override certain response processing, or to ensure that the final `Content-Type` of the response is the one that is expected.
+ *
+ * If the final `Content-Type` does not match the `Accept` header, or if the content cannot be represented in the format dictated by the `Accept` header, or you have configured a custom content type parser, and that parser returns a value that isn't in the accept header, a [406: Not Acceptible](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/406) response will be returned:
+ *
+ * ```typescript
+ * import { verifiedFetch } from '@helia/verified-fetch'
+ *
+ * const res = await verifiedFetch('ipfs://bafyJPEGImageCID', {
+ *   headers: {
+ *     accept: 'image/png'
+ *   }
+ * })
+ *
+ * console.info(res.status) // 406 - the image was a JPEG but we specified PNG as the accept header
+ * ```
+ *
+ * It can also be used to skip processing the data from some formats such as `DAG-CBOR` if you wish to handle decoding it yourself:
+ *
+ * ```typescript
+ * import { verifiedFetch } from '@helia/verified-fetch'
+ *
+ * const res = await verifiedFetch('ipfs://bafyDAGCBORCID', {
+ *   headers: {
+ *     accept: 'application/octet-stream'
+ *   }
+ * })
+ *
+ * console.info(res.headers.get('accept')) // application/octet-stream
+ * const buf = await res.arrayBuffer() // raw bytes, not processed as JSON
+ * ```
+ *
  * ## Comparison to fetch
  *
  * This module attempts to act as similarly to the `fetch()` API as possible.
@@ -449,6 +482,10 @@ import type { ProgressEvent, ProgressOptions } from 'progress-events'
  */
 export type Resource = string | CID
 
+export interface ResourceDetail {
+  resource: Resource
+}
+
 export interface CIDDetail {
   cid: CID
   path: string

packages/verified-fetch/src/utils/get-content-disposition-filename.ts

@@ -0,0 +1,18 @@
+/**
+ * Takes a filename URL param and returns a string for use in a
+ * `Content-Disposition` header
+ */
+export function getContentDispositionFilename (filename: string): string {
+  const asciiOnly = replaceNonAsciiCharacters(filename)
+
+  if (asciiOnly === filename) {
+    return `filename="${filename}"`
+  }
+
+  return `filename="${asciiOnly}"; filename*=UTF-8''${encodeURIComponent(filename)}`
+}
+
+function replaceNonAsciiCharacters (filename: string): string {
+  // eslint-disable-next-line no-control-regex
+  return filename.replace(/[^\x00-\x7F]/g, '_')
+}

packages/verified-fetch/src/utils/get-tar-stream.ts

@@ -0,0 +1,68 @@
+import { CodeError } from '@libp2p/interface'
+import { exporter, recursive, type UnixFSEntry } from 'ipfs-unixfs-exporter'
+import map from 'it-map'
+import { pipe } from 'it-pipe'
+import { pack, type TarEntryHeader, type TarImportCandidate } from 'it-tar'
+import type { AbortOptions } from '@libp2p/interface'
+import type { Blockstore } from 'interface-blockstore'
+
+const EXPORTABLE = ['file', 'raw', 'directory']
+
+function toHeader (file: UnixFSEntry): Partial<TarEntryHeader> & { name: string } {
+  let mode: number | undefined
+  let mtime: Date | undefined
+
+  if (file.type === 'file' || file.type === 'directory') {
+    mode = file.unixfs.mode
+    mtime = file.unixfs.mtime != null ? new Date(Number(file.unixfs.mtime.secs * 1000n)) : undefined
+  }
+
+  return {
+    name: file.path,
+    mode,
+    mtime,
+    size: Number(file.size),
+    type: file.type === 'directory' ? 'directory' : 'file'
+  }
+}
+
+function toTarImportCandidate (entry: UnixFSEntry): TarImportCandidate {
+  if (!EXPORTABLE.includes(entry.type)) {
+    throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS')
+  }
+
+  const candidate: TarImportCandidate = {
+    header: toHeader(entry)
+  }
+
+  if (entry.type === 'file' || entry.type === 'raw') {
+    candidate.body = entry.content()
+  }
+
+  return candidate
+}
+
+export async function * tarStream (ipfsPath: string, blockstore: Blockstore, options?: AbortOptions): AsyncGenerator<Uint8Array> {
+  const file = await exporter(ipfsPath, blockstore, options)
+
+  if (file.type === 'file' || file.type === 'raw') {
+    yield * pipe(
+      [toTarImportCandidate(file)],
+      pack()
+    )
+
+    return
+  }
+
+  if (file.type === 'directory') {
+    yield * pipe(
+      recursive(ipfsPath, blockstore, options),
+      (source) => map(source, (entry) => toTarImportCandidate(entry)),
+      pack()
+    )
+
+    return
+  }
+
+  throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS')
+}

packages/verified-fetch/src/utils/parse-url-string.ts

@@ -19,6 +19,8 @@
 
 export interface ParsedUrlQuery extends Record<string, string | unknown> {
   format?: RequestFormatShorthand
+  download?: boolean
+  filename?: string
 }
 
 export interface ParsedUrlStringResults {
@@ -109,14 +111,22 @@
   }
 
   // parse query string
-  const query: Record<string, string> = {}
+  const query: Record<string, any> = {}
 
   if (queryString != null && queryString.length > 0) {
     const queryParts = queryString.split('&')
     for (const part of queryParts) {
       const [key, value] = part.split('=')
       query[key] = decodeURIComponent(value)
     }
+
+    if (query.download != null) {
+      query.download = query.download === 'true'
+    }
+
+    if (query.filename != null) {
+      query.filename = query.filename.toString()
+    }
   }
 
   /**