wip: car gateway example

examples/gateway/car-file-gateway/.gitignore

@@ -0,0 +1 @@
+*.car

examples/gateway/car-file-gateway/api.go

@@ -0,0 +1,221 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	gopath "path"
+
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	bsfetcher "github.com/ipfs/go-fetcher/impl/blockservice"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
+	offline "github.com/ipfs/go-ipfs-exchange-offline"
+	ipld "github.com/ipfs/go-ipld-format"
+	"github.com/ipfs/go-libipfs/blocks"
+	"github.com/ipfs/go-libipfs/files"
+	"github.com/ipfs/go-merkledag"
+	ipfspath "github.com/ipfs/go-path"
+	ipfspathresolver "github.com/ipfs/go-path/resolver"
+	ft "github.com/ipfs/go-unixfs"
+	unixfile "github.com/ipfs/go-unixfs/file"
+	uio "github.com/ipfs/go-unixfs/io"
+	"github.com/ipfs/go-unixfsnode"
+	iface "github.com/ipfs/interface-go-ipfs-core"
+	"github.com/ipfs/interface-go-ipfs-core/options"
+	"github.com/ipfs/interface-go-ipfs-core/path"
+	carblockstore "github.com/ipld/go-car/v2/blockstore"
+)
+
+type carGateway struct {
+	bs   blockstore.Blockstore
+	bsrv blockservice.BlockService
+	dsrv ipld.DAGService
+}
+
+func newCarGateway(r io.ReaderAt) (*carGateway, error) {
+	bs, err := carblockstore.NewReadOnly(r, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	bsrv := blockservice.New(bs, offline.Exchange(bs))
+	dsrv := merkledag.NewDAGService(bsrv)
+
+	return &carGateway{
+		bs:   bs,
+		bsrv: bsrv,
+		dsrv: dsrv,
+	}, nil
+}
+
+func (api *carGateway) ResolvePath(ctx context.Context, pth path.Path) (path.Resolved, error) {
+	if _, ok := pth.(path.Resolved); ok {
+		return pth.(path.Resolved), nil
+	}
+
+	if err := pth.IsValid(); err != nil {
+		return nil, fmt.Errorf("path: %w", err)
+	}
+
+	if pth.Namespace() != "ipfs" {
+		return nil, fmt.Errorf("unsupported path namespace: %s", pth.Namespace())
+	}
+
+	ipath := ipfspath.Path(pth.String())
+
+	ipldFetcher := bsfetcher.NewFetcherConfig(api.bsrv)
+	fetcher := ipldFetcher.WithReifier(unixfsnode.Reify)
+	resolver := ipfspathresolver.NewBasicResolver(fetcher)
+
+	node, rest, err := resolver.ResolveToLastNode(ctx, ipath)
+	if err != nil {
+		return nil, fmt.Errorf("resolve to last node: %w", err)
+	}
+
+	root, err := cid.Parse(ipath.Segments()[1])
+	if err != nil {
+		return nil, fmt.Errorf("parse root cid: %w", err)
+	}
+
+	return path.NewResolvedPath(ipath, node, root, gopath.Join(rest...)), nil
+}
+
+func (api *carGateway) resolveNode(ctx context.Context, pth path.Path) (ipld.Node, error) {
+	rp, err := api.ResolvePath(ctx, pth)
+	if err != nil {
+		return nil, fmt.Errorf("resolve path: %w", err)
+	}
+
+	node, err := api.dsrv.Get(ctx, rp.Cid())
+	if err != nil {
+		return nil, fmt.Errorf("get node: %w", err)
+	}
+	return node, nil
+}
+
+func (api *carGateway) GetUnixFsNode(ctx context.Context, pth path.Path) (files.Node, error) {
+	nd, err := api.resolveNode(ctx, pth)
+	if err != nil {
+		return nil, err
+	}
+
+	return unixfile.NewUnixfsFile(ctx, api.dsrv, nd)
+}
+
+func (api *carGateway) LsUnixFsDir(ctx context.Context, p path.Path, opts ...options.UnixfsLsOption) (<-chan iface.DirEntry, error) {
+	settings, err := options.UnixfsLsOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	dagnode, err := api.resolveNode(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	dir, err := uio.NewDirectoryFromNode(api.dsrv, dagnode)
+	if err == uio.ErrNotADir {
+		return api.lsFromLinks(ctx, dagnode.Links(), settings)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return api.lsFromLinksAsync(ctx, dir, settings)
+}
+
+func (api *carGateway) processLink(ctx context.Context, linkres ft.LinkResult, settings *options.UnixfsLsSettings) iface.DirEntry {
+	if linkres.Err != nil {
+		return iface.DirEntry{Err: linkres.Err}
+	}
+
+	lnk := iface.DirEntry{
+		Name: linkres.Link.Name,
+		Cid:  linkres.Link.Cid,
+	}
+
+	switch lnk.Cid.Type() {
+	case cid.Raw:
+		// No need to check with raw leaves
+		lnk.Type = iface.TFile
+		lnk.Size = linkres.Link.Size
+	case cid.DagProtobuf:
+		if !settings.ResolveChildren {
+			break
+		}
+
+		linkNode, err := linkres.Link.GetNode(ctx, api.dsrv)
+		if err != nil {
+			lnk.Err = err
+			break
+		}
+
+		if pn, ok := linkNode.(*merkledag.ProtoNode); ok {
+			d, err := ft.FSNodeFromBytes(pn.Data())
+			if err != nil {
+				lnk.Err = err
+				break
+			}
+			switch d.Type() {
+			case ft.TFile, ft.TRaw:
+				lnk.Type = iface.TFile
+			case ft.THAMTShard, ft.TDirectory, ft.TMetadata:
+				lnk.Type = iface.TDirectory
+			case ft.TSymlink:
+				lnk.Type = iface.TSymlink
+				lnk.Target = string(d.Data())
+			}
+			lnk.Size = d.FileSize()
+		}
+	}
+
+	return lnk
+}
+
+func (api *carGateway) lsFromLinksAsync(ctx context.Context, dir uio.Directory, settings *options.UnixfsLsSettings) (<-chan iface.DirEntry, error) {
+	out := make(chan iface.DirEntry, uio.DefaultShardWidth)
+
+	go func() {
+		defer close(out)
+		for l := range dir.EnumLinksAsync(ctx) {
+			select {
+			case out <- api.processLink(ctx, l, settings): // TODO: perf: processing can be done in background and in parallel
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	return out, nil
+}
+
+func (api *carGateway) lsFromLinks(ctx context.Context, ndlinks []*ipld.Link, settings *options.UnixfsLsSettings) (<-chan iface.DirEntry, error) {
+	links := make(chan iface.DirEntry, len(ndlinks))
+	for _, l := range ndlinks {
+		lr := ft.LinkResult{Link: &ipld.Link{Name: l.Name, Size: l.Size, Cid: l.Cid}}
+
+		links <- api.processLink(ctx, lr, settings) // TODO: can be parallel if settings.Async
+	}
+	close(links)
+	return links, nil
+}
+
+func (api *carGateway) GetBlock(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
+	return api.bsrv.GetBlock(ctx, cid)
+}
+
+func (api *carGateway) IsCached(ctx context.Context, pth path.Path) bool {
+	rp, err := api.ResolvePath(ctx, pth)
+	if err != nil {
+		return false
+	}
+
+	_, err = api.bsrv.GetBlock(ctx, rp.Cid())
+	return err == nil
+}
+
+func (api *carGateway) GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) {
+	return nil, errors.New("not implemented")
+}

examples/gateway/car-file-gateway/go.mod

@@ -0,0 +1,100 @@
+module car-file-gateway
+
+go 1.19
+
+require (
+	github.com/ipfs/go-blockservice v0.5.0
+	github.com/ipfs/go-cid v0.3.2
+	github.com/ipfs/go-fetcher v1.6.1
+	github.com/ipfs/go-ipfs-blockstore v1.2.0
+	github.com/ipfs/go-ipfs-exchange-offline v0.3.0
+	github.com/ipfs/go-ipld-format v0.4.0
+	github.com/ipfs/go-libipfs v0.4.0
+	github.com/ipfs/go-merkledag v0.9.0
+	github.com/ipfs/go-path v0.3.0
+	github.com/ipfs/go-unixfs v0.3.1
+	github.com/ipfs/go-unixfsnode v1.5.1
+	github.com/ipfs/interface-go-ipfs-core v0.10.0
+	github.com/ipld/go-car/v2 v2.6.0
+)
+
+require (
+	github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash v1.1.0 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.1 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/ipfs/bbloom v0.0.4 // indirect
+	github.com/ipfs/go-bitfield v1.0.0 // indirect
+	github.com/ipfs/go-block-format v0.1.1 // indirect
+	github.com/ipfs/go-datastore v0.6.0 // indirect
+	github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
+	github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
+	github.com/ipfs/go-ipfs-files v0.3.0 // indirect
+	github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
+	github.com/ipfs/go-ipfs-util v0.0.2 // indirect
+	github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
+	github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
+	github.com/ipfs/go-ipns v0.3.0 // indirect
+	github.com/ipfs/go-log v1.0.5 // indirect
+	github.com/ipfs/go-log/v2 v2.5.1 // indirect
+	github.com/ipfs/go-metrics-interface v0.0.1 // indirect
+	github.com/ipfs/go-verifcid v0.0.2 // indirect
+	github.com/ipld/go-car v0.5.0 // indirect
+	github.com/ipld/go-codec-dagpb v1.5.0 // indirect
+	github.com/ipld/go-ipld-prime v0.19.0 // indirect
+	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
+	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
+	github.com/libp2p/go-libp2p v0.23.4 // indirect
+	github.com/libp2p/go-openssl v0.1.0 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mattn/go-pointer v0.0.1 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
+	github.com/mr-tron/base58 v1.2.0 // indirect
+	github.com/multiformats/go-base32 v0.1.0 // indirect
+	github.com/multiformats/go-base36 v0.2.0 // indirect
+	github.com/multiformats/go-multiaddr v0.8.0 // indirect
+	github.com/multiformats/go-multibase v0.1.1 // indirect
+	github.com/multiformats/go-multicodec v0.7.0 // indirect
+	github.com/multiformats/go-multihash v0.2.1 // indirect
+	github.com/multiformats/go-varint v0.0.7 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/polydawn/refmt v0.89.0 // indirect
+	github.com/prometheus/client_golang v1.13.0 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
+	github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
+	github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
+	go.opentelemetry.io/otel v1.12.0 // indirect
+	go.opentelemetry.io/otel/trace v1.12.0 // indirect
+	go.uber.org/atomic v1.10.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+	go.uber.org/zap v1.24.0 // indirect
+	golang.org/x/crypto v0.5.0 // indirect
+	golang.org/x/exp v0.0.0-20230129154200-a960b3787bd2 // indirect
+	golang.org/x/net v0.5.0 // indirect
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
+	golang.org/x/sys v0.4.0 // indirect
+	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	lukechampine.com/blake3 v1.1.7 // indirect
+)
+
+replace github.com/ipfs/go-libipfs => ../../..