Skip to content

Commit

Permalink
fix: use ipfs-unixfs-exporter directly (#42)
Browse files Browse the repository at this point in the history
`@helia/unixfs` pulls in the importer and other deps, we only need
to stat and cat a DAG so we can just use the exporter for this.

---------

Co-authored-by: Russell Dempsey <[email protected]>
  • Loading branch information
achingbrain and SgtPooki authored Apr 15, 2024
1 parent 7771af2 commit 4532bf1
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 97 deletions.
27 changes: 14 additions & 13 deletions packages/verified-fetch/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -57,13 +57,12 @@
"release": "aegir release"
},
"dependencies": {
"@helia/block-brokers": "^2.0.3",
"@helia/car": "^3.1.2",
"@helia/http": "^1.0.3",
"@helia/interface": "^4.1.0",
"@helia/ipns": "^7.2.0",
"@helia/routers": "^1.0.2",
"@helia/unixfs": "^3.0.3",
"@helia/block-brokers": "^2.1.0",
"@helia/car": "^3.1.3",
"@helia/http": "^1.0.4",
"@helia/interface": "^4.2.0",
"@helia/ipns": "^7.2.1",
"@helia/routers": "^1.0.3",
"@ipld/dag-cbor": "^9.2.0",
"@ipld/dag-json": "^10.2.0",
"@ipld/dag-pb": "^4.1.0",
Expand All @@ -85,11 +84,12 @@
"uint8arrays": "^5.0.3"
},
"devDependencies": {
"@helia/car": "^3.1.2",
"@helia/dag-cbor": "^3.0.2",
"@helia/dag-json": "^3.0.2",
"@helia/json": "^3.0.2",
"@helia/utils": "^0.1.0",
"@helia/car": "^3.1.3",
"@helia/dag-cbor": "^3.0.3",
"@helia/dag-json": "^3.0.3",
"@helia/json": "^3.0.3",
"@helia/unixfs": "^3.0.4",
"@helia/utils": "^0.2.0",
"@ipld/car": "^5.3.0",
"@libp2p/interface-compliance-tests": "^5.3.4",
"@libp2p/logger": "^4.0.9",
Expand All @@ -100,10 +100,11 @@
"blockstore-core": "^4.4.1",
"browser-readablestream-to-it": "^2.0.5",
"datastore-core": "^9.2.9",
"helia": "^4.1.0",
"helia": "^4.1.1",
"ipfs-unixfs-importer": "^15.2.5",
"ipns": "^9.1.0",
"it-all": "^3.0.4",
"it-drain": "^3.0.5",
"it-last": "^3.0.4",
"it-to-buffer": "^4.0.5",
"magic-bytes.js": "^1.10.0",
Expand Down
10 changes: 6 additions & 4 deletions packages/verified-fetch/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -594,11 +594,11 @@ import { createHeliaHTTP } from '@helia/http'
import { delegatedHTTPRouting } from '@helia/routers'
import { dns } from '@multiformats/dns'
import { VerifiedFetch as VerifiedFetchClass } from './verified-fetch.js'
import type { Helia } from '@helia/interface'
import type { GetBlockProgressEvents, Helia } from '@helia/interface'
import type { ResolveDNSLinkProgressEvents } from '@helia/ipns'
import type { GetEvents } from '@helia/unixfs'
import type { DNSResolvers, DNS } from '@multiformats/dns'
import type { DNSResolver } from '@multiformats/dns/resolvers'
import type { ExporterProgressEvents } from 'ipfs-unixfs-exporter'
import type { CID } from 'multiformats/cid'
import type { ProgressEvent, ProgressOptions } from 'progress-events'

Expand Down Expand Up @@ -674,8 +674,10 @@ export interface ContentTypeParser {
}

export type BubbledProgressEvents =
// unixfs
GetEvents |
// unixfs-exporter
ExporterProgressEvents |
// helia blockstore
GetBlockProgressEvents |
// ipns
ResolveDNSLinkProgressEvents

Expand Down
39 changes: 21 additions & 18 deletions packages/verified-fetch/src/verified-fetch.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
import { car } from '@helia/car'
import { ipns as heliaIpns, type IPNS } from '@helia/ipns'
import { unixfs as heliaUnixFs, type UnixFS as HeliaUnixFs } from '@helia/unixfs'
import * as ipldDagCbor from '@ipld/dag-cbor'
import * as ipldDagJson from '@ipld/dag-json'
import { code as dagPbCode } from '@ipld/dag-pb'
import { type AbortOptions, type Logger, type PeerId } from '@libp2p/interface'
import { Record as DHTRecord } from '@libp2p/kad-dht'
import { peerIdFromString } from '@libp2p/peer-id'
import { Key } from 'interface-datastore'
import { exporter } from 'ipfs-unixfs-exporter'
import toBrowserReadableStream from 'it-to-browser-readablestream'
import { code as jsonCode } from 'multiformats/codecs/json'
import { code as rawCode } from 'multiformats/codecs/raw'
Expand Down Expand Up @@ -39,7 +39,6 @@ import type { CID } from 'multiformats/cid'
interface VerifiedFetchComponents {
helia: Helia
ipns?: IPNS
unixfs?: HeliaUnixFs
}

/**
Expand Down Expand Up @@ -128,15 +127,13 @@ function getOverridenRawContentType ({ headers, accept }: { headers?: HeadersIni
export class VerifiedFetch {
private readonly helia: Helia
private readonly ipns: IPNS
private readonly unixfs: HeliaUnixFs
private readonly log: Logger
private readonly contentTypeParser: ContentTypeParser | undefined

constructor ({ helia, ipns, unixfs }: VerifiedFetchComponents, init?: VerifiedFetchInit) {
constructor ({ helia, ipns }: VerifiedFetchComponents, init?: VerifiedFetchInit) {
this.helia = helia
this.log = helia.logger.forComponent('helia:verified-fetch')
this.ipns = ipns ?? heliaIpns(helia)
this.unixfs = unixfs ?? heliaUnixFs(helia)
this.contentTypeParser = init?.contentTypeParser
this.log.trace('created VerifiedFetch instance')
}
Expand Down Expand Up @@ -350,14 +347,15 @@ export class VerifiedFetch {
const rootFilePath = 'index.html'
try {
this.log.trace('found directory at %c/%s, looking for index.html', cid, path)
const stat = await this.unixfs.stat(dirCid, {
path: rootFilePath,

const entry = await exporter(`/ipfs/${dirCid}/${rootFilePath}`, this.helia.blockstore, {
signal: options?.signal,
onProgress: options?.onProgress
})
this.log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, stat.cid)

this.log.trace('found root file at %c/%s with cid %c', dirCid, rootFilePath, entry.cid)
path = rootFilePath
resolvedCID = stat.cid
resolvedCID = entry.cid
} catch (err: any) {
options?.signal?.throwIfAborted()
this.log('error loading path %c/%s', dirCid, rootFilePath, err)
Expand All @@ -375,16 +373,22 @@ export class VerifiedFetch {
}
const offset = byteRangeContext.offset
const length = byteRangeContext.length
this.log.trace('calling unixfs.cat for %c/%s with offset=%o & length=%o', resolvedCID, path, offset, length)
const asyncIter = this.unixfs.cat(resolvedCID, {
signal: options?.signal,
onProgress: options?.onProgress,
offset,
length
})
this.log('got async iterator for %c/%s', cid, path)
this.log.trace('calling exporter for %c/%s with offset=%o & length=%o', resolvedCID, path, offset, length)

try {
const entry = await exporter(resolvedCID, this.helia.blockstore, {
signal: options?.signal,
onProgress: options?.onProgress
})

const asyncIter = entry.content({
signal: options?.signal,
onProgress: options?.onProgress,
offset,
length
})
this.log('got async iterator for %c/%s', cid, path)

const { stream, firstChunk } = await getStreamFromAsyncIterable(asyncIter, path ?? '', this.helia.logger, {
onProgress: options?.onProgress,
signal: options?.signal
Expand All @@ -400,7 +404,6 @@ export class VerifiedFetch {
if (ipfsRoots != null) {
response.headers.set('X-Ipfs-Roots', ipfsRoots.map(cid => cid.toV1().toString()).join(',')) // https://specs.ipfs.tech/http-gateways/path-gateway/#x-ipfs-roots-response-header
}

return response
} catch (err: any) {
options?.signal?.throwIfAborted()
Expand Down
Loading

0 comments on commit 4532bf1

Please sign in to comment.