From 56861d6b07b98e52b6b4a0497c8f05b6a3bcb25f Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 22 Sep 2023 19:56:21 +0200 Subject: [PATCH 01/63] docs(readme): improved header --- README.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 45edb5953..eec46e0e3 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,24 @@

-Boxo logo -
-BOXO: IPFS SDK for GO +
+ Boxo logo +
+ BOXO: IPFS SDK for GO +

+

A set of libraries for building IPFS applications and implementations in GO.

-
+

+ Official Part of IPFS Project + Discourse Forum + Matrix + ci + coverage + GitHub release + godoc reference +

-[![Go Test](https://github.com/ipfs/boxo/actions/workflows/go-test.yml/badge.svg)](https://github.com/ipfs/boxo/actions/workflows/go-test.yml) -[![Go Docs](https://img.shields.io/badge/godoc-reference-blue.svg)](https://pkg.go.dev/github.com/ipfs/boxo) -[![codecov](https://codecov.io/gh/ipfs/boxo/branch/main/graph/badge.svg?token=9eG7d8fbCB)](https://codecov.io/gh/ipfs/boxo) +
@@ -146,7 +155,7 @@ TODO: fill this in. New issues and PRs to this repo are usually looked at on a The exhaustive list is https://github.com/ipfs/boxo/network/dependents. Some notable projects include: 1. [Kubo](https://github.com/ipfs/kubo), an IPFS implementation in Go 2. [Lotus](https://github.com/filecoin-project/lotus), a Filecoin implementation in Go -3. [Bifrost Gateway](https://github.com/protocol/bifrost-gateway), a dedicated IPFS gateway +3. [Bifrost Gateway](https://github.com/ipfs/bifrost-gateway), a dedicated IPFS gateway 4. [ipfs-check](https://github.com/ipfs-shipyard/ipfs-check), checks IPFS data availability ### Governance and Access From c34473b4a186f1c1bf3c5430b04bafd7a37b338b Mon Sep 17 00:00:00 2001 From: Jorropo Date: Mon, 26 Jun 2023 15:13:24 +0200 Subject: [PATCH 02/63] util: use subtle for XORing bytes Fixes #377 --- util/util.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/util/util.go b/util/util.go index ffcab2f33..74cf1aaff 100644 --- a/util/util.go +++ b/util/util.go @@ -3,6 +3,7 @@ package util import ( + "crypto/subtle" "errors" "io" "math/rand" @@ -150,9 +151,9 @@ func IsValidHash(s string) bool { // XOR takes two byte slices, XORs them together, returns the resulting slice. func XOR(a, b []byte) []byte { + _ = b[len(a)-1] // keeping same behaviour as previously but this looks like a bug + c := make([]byte, len(a)) - for i := 0; i < len(a); i++ { - c[i] = a[i] ^ b[i] - } + subtle.XORBytes(c, a, b) return c } From 76f8e3f4135881e842c0078d55409276010d8ad4 Mon Sep 17 00:00:00 2001 From: Laurent Senta Date: Mon, 2 Oct 2023 10:11:17 -0400 Subject: [PATCH 03/63] [skip changelog] ci: gateway-conformance v0.4 (#476) ci: gateway-conformance v0.4 --- .github/workflows/gateway-conformance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gateway-conformance.yml b/.github/workflows/gateway-conformance.yml index a2a3734f6..f45e9aedf 100644 --- a/.github/workflows/gateway-conformance.yml +++ b/.github/workflows/gateway-conformance.yml @@ -16,7 +16,7 @@ jobs: steps: # 1. Download the gateway-conformance fixtures - name: Download gateway-conformance fixtures - uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.3 + uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.4 with: output: fixtures merged: true @@ -40,7 +40,7 @@ jobs: # 4. Run the gateway-conformance tests - name: Run gateway-conformance tests - uses: ipfs/gateway-conformance/.github/actions/test@v0.3 + uses: ipfs/gateway-conformance/.github/actions/test@v0.4 with: gateway-url: http://127.0.0.1:8040 json: output.json From c28c847582f0512d7f4a0e25b45aebae2ca7ca04 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 2 Oct 2023 14:26:47 +0000 Subject: [PATCH 04/63] feat(gateway)!: return HeadResponse instead of files.Node, no multi-range Co-authored-by: Henrique Dias Co-authored-by: Marcin Rataj --- CHANGELOG.md | 5 + gateway/blocks_backend.go | 65 +++- gateway/gateway.go | 82 ++++- gateway/gateway_test.go | 4 +- gateway/handler.go | 55 +--- gateway/handler_block.go | 12 +- gateway/handler_codec.go | 32 +- gateway/handler_defaults.go | 106 +++--- gateway/handler_unixfs__redirects.go | 34 +- gateway/handler_unixfs_dir.go | 45 ++- gateway/handler_unixfs_file.go | 41 +-- gateway/metrics.go | 2 +- gateway/serve_http_content.go | 472 +++++++++++++++++++++++++++ gateway/utilities_test.go | 2 +- 14 files changed, 781 insertions(+), 176 deletions(-) create mode 100644 gateway/serve_http_content.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8565c5676..bb864a5a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,11 @@ The following emojis are used to highlight certain changes: ### Changed +* `boxo/gateway` + * 🛠 The `IPFSBackend` interface was updated to make the responses of the + `Head` method more explicit. It now returns a `HeadResponse` instead of a + `files.Node`. + ### Removed ### Fixed diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index 208c92062..fc0685d62 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -153,11 +153,34 @@ func (bb *BlocksBackend) Get(ctx context.Context, path ImmutablePath, ranges ... return md, nil, err } + // Only a single range is supported in responses to HTTP Range Requests. + // When more than one is passed in the Range header, this library will + // return a response for the first one and ignores remaining ones. + var ra *ByteRange + if len(ranges) > 0 { + ra = &ranges[0] + } + rootCodec := nd.Cid().Prefix().GetCodec() + // This covers both Raw blocks and terminal IPLD codecs like dag-cbor and dag-json // Note: while only cbor, json, dag-cbor, and dag-json are currently supported by gateways this could change + // Note: For the raw codec we return just the relevant range rather than the entire block if rootCodec != uint64(mc.DagPb) { - return md, NewGetResponseFromFile(files.NewBytesFile(nd.RawData())), nil + f := files.NewBytesFile(nd.RawData()) + + fileSize, err := f.Size() + if err != nil { + return ContentPathMetadata{}, nil, err + } + + if rootCodec == uint64(mc.Raw) { + if err := seekToRangeStart(f, ra); err != nil { + return ContentPathMetadata{}, nil, err + } + } + + return md, NewGetResponseFromReader(f, fileSize), nil } // This code path covers full graph, single file/directory, and range requests @@ -179,10 +202,23 @@ func (bb *BlocksBackend) Get(ctx context.Context, path ImmutablePath, ranges ... if sz < 0 { return ContentPathMetadata{}, nil, fmt.Errorf("directory cumulative DAG size cannot be negative") } - return md, NewGetResponseFromDirectoryListing(uint64(sz), dir.EnumLinksAsync(ctx)), nil + return md, NewGetResponseFromDirectoryListing(uint64(sz), dir.EnumLinksAsync(ctx), nil), nil } if file, ok := f.(files.File); ok { - return md, NewGetResponseFromFile(file), nil + fileSize, err := f.Size() + if err != nil { + return ContentPathMetadata{}, nil, err + } + + if err := seekToRangeStart(file, ra); err != nil { + return ContentPathMetadata{}, nil, err + } + + if s, ok := f.(*files.Symlink); ok { + return md, NewGetResponseFromSymlink(s, fileSize), nil + } + + return md, NewGetResponseFromReader(file, fileSize), nil } return ContentPathMetadata{}, nil, fmt.Errorf("data was not a valid file or directory: %w", ErrInternalServerError) // TODO: should there be a gateway invalid content type to abstract over the various IPLD error types? @@ -211,7 +247,7 @@ func (bb *BlocksBackend) GetBlock(ctx context.Context, path ImmutablePath) (Cont return md, files.NewBytesFile(nd.RawData()), nil } -func (bb *BlocksBackend) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { +func (bb *BlocksBackend) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { md, nd, err := bb.getNode(ctx, path) if err != nil { return md, nil, err @@ -219,7 +255,7 @@ func (bb *BlocksBackend) Head(ctx context.Context, path ImmutablePath) (ContentP rootCodec := nd.Cid().Prefix().GetCodec() if rootCodec != uint64(mc.DagPb) { - return md, files.NewBytesFile(nd.RawData()), nil + return md, NewHeadResponseForFile(files.NewBytesFile(nd.RawData()), int64(len(nd.RawData()))), nil } // TODO: We're not handling non-UnixFS dag-pb. There's a bit of a discrepancy @@ -229,7 +265,24 @@ func (bb *BlocksBackend) Head(ctx context.Context, path ImmutablePath) (ContentP return ContentPathMetadata{}, nil, err } - return md, fileNode, nil + sz, err := fileNode.Size() + if err != nil { + return ContentPathMetadata{}, nil, err + } + + if _, ok := fileNode.(files.Directory); ok { + return md, NewHeadResponseForDirectory(sz), nil + } + + if _, ok := fileNode.(*files.Symlink); ok { + return md, NewHeadResponseForSymlink(sz), nil + } + + if f, ok := fileNode.(files.File); ok { + return md, NewHeadResponseForFile(f, sz), nil + } + + return ContentPathMetadata{}, nil, fmt.Errorf("unsupported UnixFS file type") } // emptyRoot is a CAR root with the empty identity CID. CAR files are recommended diff --git a/gateway/gateway.go b/gateway/gateway.go index 19b801bba..089dd236c 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -260,21 +260,74 @@ type ByteRange struct { } type GetResponse struct { - bytes files.File + bytes io.ReadCloser + bytesSize int64 + symlink *files.Symlink directoryMetadata *directoryMetadata } +func (r *GetResponse) Close() error { + if r.bytes != nil { + return r.bytes.Close() + } + if r.symlink != nil { + return r.symlink.Close() + } + if r.directoryMetadata != nil { + if r.directoryMetadata.closeFn == nil { + return nil + } + return r.directoryMetadata.closeFn() + } + // Should be unreachable + return nil +} + +var _ io.Closer = (*GetResponse)(nil) + type directoryMetadata struct { dagSize uint64 entries <-chan unixfs.LinkResult + closeFn func() error +} + +func NewGetResponseFromReader(file io.ReadCloser, fullFileSize int64) *GetResponse { + return &GetResponse{bytes: file, bytesSize: fullFileSize} +} + +func NewGetResponseFromSymlink(symlink *files.Symlink, size int64) *GetResponse { + return &GetResponse{symlink: symlink, bytesSize: size} } -func NewGetResponseFromFile(file files.File) *GetResponse { - return &GetResponse{bytes: file} +func NewGetResponseFromDirectoryListing(dagSize uint64, entries <-chan unixfs.LinkResult, closeFn func() error) *GetResponse { + return &GetResponse{directoryMetadata: &directoryMetadata{dagSize: dagSize, entries: entries, closeFn: closeFn}} } -func NewGetResponseFromDirectoryListing(dagSize uint64, entries <-chan unixfs.LinkResult) *GetResponse { - return &GetResponse{directoryMetadata: &directoryMetadata{dagSize, entries}} +type HeadResponse struct { + bytesSize int64 + startingBytes io.ReadCloser + isFile bool + isSymLink bool + isDir bool +} + +func (r *HeadResponse) Close() error { + if r.startingBytes != nil { + return r.startingBytes.Close() + } + return nil +} + +func NewHeadResponseForFile(startingBytes io.ReadCloser, size int64) *HeadResponse { + return &HeadResponse{startingBytes: startingBytes, isFile: true, bytesSize: size} +} + +func NewHeadResponseForSymlink(symlinkSize int64) *HeadResponse { + return &HeadResponse{isSymLink: true, bytesSize: symlinkSize} +} + +func NewHeadResponseForDirectory(dagSize int64) *HeadResponse { + return &HeadResponse{isDir: true, bytesSize: dagSize} } // IPFSBackend is the required set of functionality used to implement the IPFS @@ -305,6 +358,9 @@ type IPFSBackend interface { // file will still need magic bytes from the very beginning for content // type sniffing). // - A range request for a directory currently holds no semantic meaning. + // - For non-UnixFS (and non-raw data) such as terminal IPLD dag-cbor/json, etc. blocks the returned response + // bytes should be the complete block and returned as an [io.ReadSeekCloser] starting at the beginning of the + // block rather than as an [io.ReadCloser] that starts at the beginning of the range request. // // [HTTP Byte Ranges]: https://httpwg.org/specs/rfc9110.html#rfc.section.14.1.2 Get(context.Context, ImmutablePath, ...ByteRange) (ContentPathMetadata, *GetResponse, error) @@ -316,12 +372,16 @@ type IPFSBackend interface { // GetBlock returns a single block of data GetBlock(context.Context, ImmutablePath) (ContentPathMetadata, files.File, error) - // Head returns a file or directory depending on what the path is that has been requested. - // For UnixFS files should return a file which has the correct file size and either returns the ContentType in ContentPathMetadata or - // enough data (e.g. 3kiB) such that the content type can be determined by sniffing. - // For all other data types returning just size information is sufficient - // TODO: give function more explicit return types - Head(context.Context, ImmutablePath) (ContentPathMetadata, files.Node, error) + // Head returns a [HeadResponse] depending on what the path is that has been requested. + // For UnixFS files (and raw blocks) should return the size of the file and either set the ContentType in + // ContentPathMetadata or send back a reader from the beginning of the file with enough data (e.g. 3kiB) such that + // the content type can be determined by sniffing. + // + // For UnixFS directories and symlinks only setting the size and type are necessary. + // + // For all other data types (e.g. (DAG-)CBOR/JSON blocks) returning the size information as a file while setting + // the content-type is sufficient. + Head(context.Context, ImmutablePath) (ContentPathMetadata, *HeadResponse, error) // ResolvePath resolves the path using UnixFS resolver. If the path does not // exist due to a missing link, it should return an error of type: diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 98996acb3..d041cad55 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -719,7 +719,7 @@ func (mb *errorMockBackend) GetBlock(ctx context.Context, path ImmutablePath) (C return ContentPathMetadata{}, nil, mb.err } -func (mb *errorMockBackend) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { +func (mb *errorMockBackend) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { return ContentPathMetadata{}, nil, mb.err } @@ -803,7 +803,7 @@ func (mb *panicMockBackend) GetBlock(ctx context.Context, immutablePath Immutabl panic("i am panicking") } -func (mb *panicMockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { +func (mb *panicMockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { panic("i am panicking") } diff --git a/gateway/handler.go b/gateway/handler.go index ecf505617..af20e2b6e 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -92,13 +92,19 @@ func NewHandler(c Config, backend IPFSBackend) http.Handler { return newHandlerWithMetrics(&c, backend) } -// serveContent replies to the request using the content in the provided ReadSeeker +// serveContent replies to the request using the content in the provided Reader // and returns the status code written and any error encountered during a write. -// It wraps http.serveContent which takes care of If-None-Match+Etag, +// It wraps httpServeContent (a close clone of http.ServeContent) which takes care of If-None-Match+Etag, // Content-Length and range requests. -func serveContent(w http.ResponseWriter, req *http.Request, name string, modtime time.Time, content io.ReadSeeker) (int, bool, error) { +// +// Notes: +// 1. For HEAD requests the io.Reader may be nil/undefined +// 2. When the io.Reader is needed it must start at the beginning of the first Range Request component if it exists +// 3. Only a single HTTP Range Request is supported, if more than one are requested only the first will be honored +// 4. The Content-Type header must already be set +func serveContent(w http.ResponseWriter, req *http.Request, modtime time.Time, size int64, content io.Reader) (int, bool, error) { ew := &errRecordingResponseWriter{ResponseWriter: w} - http.ServeContent(ew, req, name, modtime, content) + httpServeContent(ew, req, modtime, size, content) // When we calculate some metrics we want a flag that lets us to ignore // errors and 304 Not Modified, and only care when requested data @@ -554,40 +560,6 @@ func etagMatch(ifNoneMatchHeader string, etagsToCheck ...string) bool { return false } -// scanETag determines if a syntactically valid ETag is present at s. If so, -// the ETag and remaining text after consuming ETag is returned. Otherwise, -// it returns "", "". -// (This is the same logic as one executed inside of http.ServeContent) -func scanETag(s string) (etag string, remain string) { - s = textproto.TrimString(s) - start := 0 - if strings.HasPrefix(s, "W/") { - start = 2 - } - if len(s[start:]) < 2 || s[start] != '"' { - return "", "" - } - // ETag is either W/"text" or "text". - // See RFC 7232 2.3. - for i := start + 1; i < len(s); i++ { - c := s[i] - switch { - // Character values allowed in ETags. - case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80: - case c == '"': - return s[:i+1], s[i+1:] - default: - return "", "" - } - } - return "", "" -} - -// etagWeakMatch reports whether a and b match using weak ETag comparison. -func etagWeakMatch(a, b string) bool { - return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/") -} - // getEtag generates an ETag value based on an HTTP Request, a CID and a response // format. This function DOES NOT generate ETags for CARs or IPNS Records. func getEtag(r *http.Request, cid cid.Cid, responseFormat string) string { @@ -776,6 +748,13 @@ func (i *handler) handleWebRequestErrors(w http.ResponseWriter, r *http.Request, return ImmutablePath{}, false } + // If the error is not an IPLD traversal error then we should not be looking for _redirects or legacy 404s + if !isErrNotFound(err) { + err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) + i.webError(w, r, err, http.StatusInternalServerError) + return ImmutablePath{}, false + } + // If we have origin isolation (subdomain gw, DNSLink website), // and response type is UnixFS (default for website hosting) // we can leverage the presence of an _redirects file and apply rules defined there. diff --git a/gateway/handler_block.go b/gateway/handler_block.go index dbff9a7ad..9d2a93b38 100644 --- a/gateway/handler_block.go +++ b/gateway/handler_block.go @@ -38,9 +38,19 @@ func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *h w.Header().Set("Content-Type", rawResponseFormat) w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) + sz, err := data.Size() + if err != nil { + i.handleRequestErrors(w, r, rq.contentPath, err) + return false + } + + if !i.seekToStartOfFirstRange(w, r, data) { + return false + } + // ServeContent will take care of // If-None-Match+Etag, Content-Length and range requests - _, dataSent, _ := serveContent(w, r, name, modtime, data) + _, dataSent, _ := serveContent(w, r, modtime, sz, data) if dataSent { // Update metrics diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index 007a52fda..97dfaad0a 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -69,10 +69,16 @@ func (i *handler) serveCodec(ctx context.Context, w http.ResponseWriter, r *http defer data.Close() setIpfsRootsHeader(w, rq, &pathMetadata) - return i.renderCodec(ctx, w, r, rq, data) + + blockSize, err := data.Size() + if !i.handleRequestErrors(w, r, rq.contentPath, err) { + return false + } + + return i.renderCodec(ctx, w, r, rq, blockSize, data) } -func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData, blockData io.ReadSeekCloser) bool { +func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *http.Request, rq *requestData, blockSize int64, blockData io.ReadSeekCloser) bool { resolvedPath := rq.pathMetadata.LastSegment ctx, span := spanTrace(ctx, "Handler.RenderCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", rq.responseFormat))) defer span.End() @@ -105,7 +111,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt // Set HTTP headers (for caching, etc). Etag will be replaced if handled by serveCodecHTML. modtime := addCacheControlHeaders(w, r, rq.contentPath, resolvedPath.Cid(), responseContentType) - name := setCodecContentDisposition(w, r, resolvedPath, responseContentType) + _ = setCodecContentDisposition(w, r, resolvedPath, responseContentType) w.Header().Set("Content-Type", responseContentType) w.Header().Set("X-Content-Type-Options", "nosniff") @@ -121,7 +127,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt } else { // This covers CIDs with codec 'json' and 'cbor' as those do not have // an explicit requested content type. - return i.serveCodecRaw(ctx, w, r, blockData, rq.contentPath, name, modtime, rq.begin) + return i.serveCodecRaw(ctx, w, r, blockSize, blockData, rq.contentPath, modtime, rq.begin) } } @@ -131,7 +137,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt if ok { for _, skipCodec := range skipCodecs { if skipCodec == cidCodec { - return i.serveCodecRaw(ctx, w, r, blockData, rq.contentPath, name, modtime, rq.begin) + return i.serveCodecRaw(ctx, w, r, blockSize, blockData, rq.contentPath, modtime, rq.begin) } } } @@ -149,7 +155,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt return i.serveCodecConverted(ctx, w, r, blockCid, blockData, rq.contentPath, toCodec, modtime, rq.begin) } -func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, blockCid cid.Cid, blockData io.ReadSeekCloser, resolvedPath ipath.Resolved, contentPath ipath.Path) bool { +func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, blockCid cid.Cid, blockData io.Reader, resolvedPath ipath.Resolved, contentPath ipath.Path) bool { // WithHostname may have constructed an IPFS (or IPNS) path using the Host header. // In this case, we need the original path for constructing the redirect. requestURI, err := url.ParseRequestURI(r.RequestURI) @@ -207,7 +213,7 @@ func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r * // parseNode does a best effort attempt to parse this request's block such that // a preview can be displayed in the gateway. If something fails along the way, // returns nil, therefore not displaying the preview. -func parseNode(blockCid cid.Cid, blockData io.ReadSeekCloser) *assets.ParsedNode { +func parseNode(blockCid cid.Cid, blockData io.Reader) *assets.ParsedNode { codec := blockCid.Prefix().Codec decoder, err := multicodec.LookupDecoder(codec) if err != nil { @@ -229,10 +235,14 @@ func parseNode(blockCid cid.Cid, blockData io.ReadSeekCloser) *assets.ParsedNode } // serveCodecRaw returns the raw block without any conversion -func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, blockData io.ReadSeekCloser, contentPath ipath.Path, name string, modtime, begin time.Time) bool { +func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, blockSize int64, blockData io.ReadSeekCloser, contentPath ipath.Path, modtime, begin time.Time) bool { // ServeContent will take care of - // If-None-Match+Etag, Content-Length and range requests - _, dataSent, _ := serveContent(w, r, name, modtime, blockData) + // If-None-Match+Etag, Content-Length and setting range request headers after we've already seeked to the start of + // the first range + if !i.seekToStartOfFirstRange(w, r, blockData) { + return false + } + _, dataSent, _ := serveContent(w, r, modtime, blockSize, blockData) if dataSent { // Update metrics @@ -243,7 +253,7 @@ func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *h } // serveCodecConverted returns payload converted to codec specified in toCodec -func (i *handler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, blockCid cid.Cid, blockData io.ReadSeekCloser, contentPath ipath.Path, toCodec mc.Code, modtime, begin time.Time) bool { +func (i *handler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, blockCid cid.Cid, blockData io.ReadCloser, contentPath ipath.Path, toCodec mc.Code, modtime, begin time.Time) bool { codec := blockCid.Prefix().Codec decoder, err := multicodec.LookupDecoder(codec) if err != nil { diff --git a/gateway/handler_defaults.go b/gateway/handler_defaults.go index de31c1fc1..22c397ade 100644 --- a/gateway/handler_defaults.go +++ b/gateway/handler_defaults.go @@ -4,12 +4,12 @@ import ( "context" "errors" "fmt" + "io" "net/http" "net/textproto" "strconv" "strings" - "github.com/ipfs/boxo/files" mc "github.com/multiformats/go-multicodec" "go.opentelemetry.io/otel/attribute" @@ -21,25 +21,23 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h defer span.End() var ( - pathMetadata ContentPathMetadata - bytesResponse files.File - isDirectoryHeadRequest bool - directoryMetadata *directoryMetadata - err error - ranges []ByteRange + pathMetadata ContentPathMetadata + err error + ranges []ByteRange + headResp *HeadResponse + getResp *GetResponse ) switch r.Method { case http.MethodHead: - var data files.Node - pathMetadata, data, err = i.backend.Head(ctx, rq.mostlyResolvedPath()) + pathMetadata, headResp, err = i.backend.Head(ctx, rq.mostlyResolvedPath()) if err != nil { if isWebRequest(rq.responseFormat) { forwardedPath, continueProcessing := i.handleWebRequestErrors(w, r, rq.mostlyResolvedPath(), rq.immutablePath, rq.contentPath, err, rq.logger) if !continueProcessing { return false } - pathMetadata, data, err = i.backend.Head(ctx, forwardedPath) + pathMetadata, headResp, err = i.backend.Head(ctx, forwardedPath) if err != nil { err = fmt.Errorf("failed to resolve %s: %w", debugStr(rq.contentPath.String()), err) i.webError(w, r, err, http.StatusInternalServerError) @@ -51,30 +49,21 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h } } } - defer data.Close() - if _, ok := data.(files.Directory); ok { - isDirectoryHeadRequest = true - } else if f, ok := data.(files.File); ok { - bytesResponse = f - } else { - i.webError(w, r, fmt.Errorf("unsupported response type"), http.StatusInternalServerError) - return false - } + defer headResp.Close() case http.MethodGet: rangeHeader := r.Header.Get("Range") if rangeHeader != "" { // TODO: Add tests for range parsing - ranges, err = parseRange(rangeHeader) + ranges, err = parseRangeWithoutLength(rangeHeader) if err != nil { i.webError(w, r, fmt.Errorf("invalid range request: %w", err), http.StatusBadRequest) return false } } - var getResp *GetResponse // TODO: passing only resolved path here, instead of contentPath is // harming content routing. Knowing original immutableContentPath will - // allow backend to find providers for parents, even when internal + // allow backend to find providers for parents, even when internal // CIDs are not announced, and will provide better key for caching // related DAGs. pathMetadata, getResp, err = i.backend.Get(ctx, rq.mostlyResolvedPath(), ranges...) @@ -96,13 +85,7 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h } } } - if getResp.bytes != nil { - bytesResponse = getResp.bytes - defer bytesResponse.Close() - } else { - directoryMetadata = getResp.directoryMetadata - } - + defer getResp.Close() default: // This shouldn't be possible to reach which is why it is a 500 rather than 4XX error i.webError(w, r, fmt.Errorf("invalid method: cannot use this HTTP method with the given request"), http.StatusInternalServerError) @@ -114,27 +97,58 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h resolvedPath := pathMetadata.LastSegment switch mc.Code(resolvedPath.Cid().Prefix().Codec) { case mc.Json, mc.DagJson, mc.Cbor, mc.DagCbor: - if bytesResponse == nil { // This should never happen - i.webError(w, r, fmt.Errorf("decoding error: data not usable as a file"), http.StatusInternalServerError) - return false - } rq.logger.Debugw("serving codec", "path", rq.contentPath) - return i.renderCodec(r.Context(), w, r, rq, bytesResponse) + var blockSize int64 + var dataToRender io.ReadSeekCloser + if headResp != nil { + blockSize = headResp.bytesSize + dataToRender = nil + } else { + blockSize = getResp.bytesSize + dataAsReadSeekCloser, ok := getResp.bytes.(io.ReadSeekCloser) + if !ok { + i.webError(w, r, fmt.Errorf("expected returned non-UnixFS data to be seekable"), http.StatusInternalServerError) + } + dataToRender = dataAsReadSeekCloser + } + + return i.renderCodec(r.Context(), w, r, rq, blockSize, dataToRender) default: rq.logger.Debugw("serving unixfs", "path", rq.contentPath) ctx, span := spanTrace(ctx, "Handler.ServeUnixFS", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() - // Handling Unixfs file - if bytesResponse != nil { - rq.logger.Debugw("serving unixfs file", "path", rq.contentPath) - return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, bytesResponse, pathMetadata.ContentType, rq.begin) - } - - // Handling Unixfs directory - if directoryMetadata != nil || isDirectoryHeadRequest { - rq.logger.Debugw("serving unixfs directory", "path", rq.contentPath) - return i.serveDirectory(ctx, w, r, resolvedPath, rq.contentPath, isDirectoryHeadRequest, directoryMetadata, ranges, rq.begin, rq.logger) + // Handle UnixFS HEAD requests + if headResp != nil { + if headResp.isFile { + rq.logger.Debugw("serving unixfs file", "path", rq.contentPath) + return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, headResp.bytesSize, headResp.startingBytes, false, true, pathMetadata.ContentType, rq.begin) + } else if headResp.isSymLink { + rq.logger.Debugw("serving unixfs file", "path", rq.contentPath) + return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, headResp.bytesSize, nil, true, true, pathMetadata.ContentType, rq.begin) + } else if headResp.isDir { + rq.logger.Debugw("serving unixfs directory", "path", rq.contentPath) + return i.serveDirectory(ctx, w, r, resolvedPath, rq.contentPath, true, nil, ranges, rq.begin, rq.logger) + } + } else { + if getResp.bytes != nil { + rq.logger.Debugw("serving unixfs file", "path", rq.contentPath) + rangeRequestStartsAtZero := true + if len(ranges) > 0 { + ra := ranges[0] + if ra.From != 0 { + rangeRequestStartsAtZero = false + } + } + return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, getResp.bytesSize, getResp.bytes, false, rangeRequestStartsAtZero, pathMetadata.ContentType, rq.begin) + } else if getResp.symlink != nil { + rq.logger.Debugw("serving unixfs file", "path", rq.contentPath) + // Note: this ignores range requests against symlinks + return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, getResp.bytesSize, getResp.symlink, true, true, pathMetadata.ContentType, rq.begin) + } else if getResp.directoryMetadata != nil { + rq.logger.Debugw("serving unixfs directory", "path", rq.contentPath) + return i.serveDirectory(ctx, w, r, resolvedPath, rq.contentPath, false, getResp.directoryMetadata, ranges, rq.begin, rq.logger) + } } i.webError(w, r, fmt.Errorf("unsupported UnixFS type"), http.StatusInternalServerError) @@ -142,8 +156,8 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h } } -// parseRange parses a Range header string as per RFC 7233. -func parseRange(s string) ([]ByteRange, error) { +// parseRangeWithoutLength parses a Range header string as per RFC 7233. +func parseRangeWithoutLength(s string) ([]ByteRange, error) { if s == "" { return nil, nil // header not present } diff --git a/gateway/handler_unixfs__redirects.go b/gateway/handler_unixfs__redirects.go index a96b87d36..978f55647 100644 --- a/gateway/handler_unixfs__redirects.go +++ b/gateway/handler_unixfs__redirects.go @@ -11,8 +11,6 @@ import ( "go.uber.org/zap" ipath "github.com/ipfs/boxo/coreiface/path" - "github.com/ipfs/boxo/files" - redirects "github.com/ipfs/go-ipfs-redirects-file" ) @@ -160,12 +158,12 @@ func (i *handler) getRedirectRules(r *http.Request, redirectsPath ImmutablePath) } return false, nil, err } + defer redirectsFileGetResp.Close() if redirectsFileGetResp.bytes == nil { return false, nil, fmt.Errorf(" _redirects is not a file") } f := redirectsFileGetResp.bytes - defer f.Close() // Parse redirect rules from file redirectRules, err := redirects.Parse(f) @@ -186,19 +184,16 @@ func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPat if err != nil { return err } + defer getresp.Close() if getresp.bytes == nil { return fmt.Errorf("could not convert node for %d page to file", status) } content4xxFile := getresp.bytes - defer content4xxFile.Close() content4xxCid := pathMetadata.LastSegment.Cid() - size, err := content4xxFile.Size() - if err != nil { - return fmt.Errorf("could not get size of %d page", status) - } + size := getresp.bytesSize logger.Debugf("using _redirects: custom %d file at %q", status, content4xxPath) w.Header().Set("Content-Type", "text/html") @@ -224,29 +219,24 @@ func hasOriginIsolation(r *http.Request) bool { // This is provided only for backward-compatibility, until websites migrate // to 404s managed via _redirects file (https://github.com/ipfs/specs/pull/290) func (i *handler) serveLegacy404IfPresent(w http.ResponseWriter, r *http.Request, imPath ImmutablePath, logger *zap.SugaredLogger) bool { - resolved404File, ctype, err := i.searchUpTreeFor404(r, imPath) + resolved404File, resolved404FileSize, ctype, err := i.searchUpTreeFor404(r, imPath) if err != nil { return false } defer resolved404File.Close() - size, err := resolved404File.Size() - if err != nil { - return false - } - logger.Debugw("using pretty 404 file", "path", imPath) w.Header().Set("Content-Type", ctype) - w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) + w.Header().Set("Content-Length", strconv.FormatInt(resolved404FileSize, 10)) w.WriteHeader(http.StatusNotFound) - _, err = io.CopyN(w, resolved404File, size) + _, err = io.CopyN(w, resolved404File, resolved404FileSize) return err == nil } -func (i *handler) searchUpTreeFor404(r *http.Request, imPath ImmutablePath) (files.File, string, error) { +func (i *handler) searchUpTreeFor404(r *http.Request, imPath ImmutablePath) (io.ReadCloser, int64, string, error) { filename404, ctype, err := preferred404Filename(r.Header.Values("Accept")) if err != nil { - return nil, "", err + return nil, 0, "", err } pathComponents := strings.Split(imPath.String(), "/") @@ -267,12 +257,14 @@ func (i *handler) searchUpTreeFor404(r *http.Request, imPath ImmutablePath) (fil continue } if getResp.bytes == nil { - return nil, "", fmt.Errorf("found a pretty 404 but it was not a file") + // Close the response here if not returning bytes, otherwise it's the caller's responsibility to close the io.ReadCloser + getResp.Close() + return nil, 0, "", fmt.Errorf("found a pretty 404 but it was not a file") } - return getResp.bytes, ctype, nil + return getResp.bytes, getResp.bytesSize, ctype, nil } - return nil, "", fmt.Errorf("no pretty 404 in any parent folder") + return nil, 0, "", fmt.Errorf("no pretty 404 in any parent folder") } func preferred404Filename(acceptHeaders []string) (string, string, error) { diff --git a/gateway/handler_unixfs_dir.go b/gateway/handler_unixfs_dir.go index 2808cfdc4..1ece9c96d 100644 --- a/gateway/handler_unixfs_dir.go +++ b/gateway/handler_unixfs_dir.go @@ -3,6 +3,7 @@ package gateway import ( "context" "fmt" + "io" "net/http" "net/url" gopath "path" @@ -59,42 +60,60 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } // Check if directory has index.html, if so, serveFile - idxPath := ipath.Join(contentPath, "index.html") - imIndexPath, err := NewImmutablePath(ipath.Join(resolvedPath, "index.html")) + appendIndexHtml := func(p ipath.Path) ipath.Path { + basePath := p.String() + if basePath[len(basePath)-1] != '/' { + basePath += "/" + } + return ipath.New(basePath + "index.html") + } + + idxPath := appendIndexHtml(contentPath) + imIndexPath, err := NewImmutablePath(appendIndexHtml(resolvedPath)) if err != nil { i.webError(w, r, err, http.StatusInternalServerError) return false } // TODO: could/should this all be skipped to have HEAD requests just return html content type and save the complexity? If so can we skip the above code as well? - var idxFile files.File + var idxFileBytes io.ReadCloser + var idxFileSize int64 + var returnRangeStartsAtZero bool if isHeadRequest { - var idx files.Node - _, idx, err = i.backend.Head(ctx, imIndexPath) + var idxHeadResp *HeadResponse + _, idxHeadResp, err = i.backend.Head(ctx, imIndexPath) if err == nil { - f, ok := idx.(files.File) - if !ok { + defer idxHeadResp.Close() + if !idxHeadResp.isFile { i.webError(w, r, fmt.Errorf("%q could not be read: %w", imIndexPath, files.ErrNotReader), http.StatusUnprocessableEntity) return false } - idxFile = f + returnRangeStartsAtZero = true + idxFileBytes = idxHeadResp.startingBytes + idxFileSize = idxHeadResp.bytesSize } } else { - var getResp *GetResponse - _, getResp, err = i.backend.Get(ctx, imIndexPath, ranges...) + var idxGetResp *GetResponse + _, idxGetResp, err = i.backend.Get(ctx, imIndexPath, ranges...) if err == nil { - if getResp.bytes == nil { + defer idxGetResp.Close() + if idxGetResp.bytes == nil { i.webError(w, r, fmt.Errorf("%q could not be read: %w", imIndexPath, files.ErrNotReader), http.StatusUnprocessableEntity) return false } - idxFile = getResp.bytes + if len(ranges) > 0 { + ra := ranges[0] + returnRangeStartsAtZero = ra.From == 0 + } + idxFileBytes = idxGetResp.bytes + idxFileSize = idxGetResp.bytesSize } } if err == nil { logger.Debugw("serving index.html file", "path", idxPath) // write to request - success := i.serveFile(ctx, w, r, resolvedPath, idxPath, idxFile, "text/html", begin) + success := i.serveFile(ctx, w, r, resolvedPath, idxPath, idxFileSize, idxFileBytes, false, returnRangeStartsAtZero, "text/html", begin) if success { i.unixfsDirIndexGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) } diff --git a/gateway/handler_unixfs_file.go b/gateway/handler_unixfs_file.go index cd924e5aa..8375bfcd2 100644 --- a/gateway/handler_unixfs_file.go +++ b/gateway/handler_unixfs_file.go @@ -1,6 +1,7 @@ package gateway import ( + "bytes" "context" "fmt" "io" @@ -12,14 +13,13 @@ import ( "github.com/gabriel-vasile/mimetype" ipath "github.com/ipfs/boxo/coreiface/path" - "github.com/ipfs/boxo/files" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) // serveFile returns data behind a file along with HTTP headers based on // the file itself, its CID and the contentPath used for accessing it. -func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, file files.File, fileContentType string, begin time.Time) bool { +func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, fileSize int64, fileBytes io.ReadCloser, isSymlink bool, returnRangeStartsAtZero bool, fileContentType string, begin time.Time) bool { _, span := spanTrace(ctx, "Handler.ServeFile", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() @@ -29,14 +29,7 @@ func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http. // Set Content-Disposition name := addContentDispositionHeader(w, r, contentPath) - // Prepare size value for Content-Length HTTP header (set inside of http.ServeContent) - size, err := file.Size() - if err != nil { - http.Error(w, "cannot serve files with unknown sizes", http.StatusBadGateway) - return false - } - - if size == 0 { + if fileSize == 0 { // We override null files to 200 to avoid issues with fragment caching reverse proxies. // Also whatever you are asking for, it's cheaper to just give you the complete file (nothing). // TODO: remove this if clause once https://github.com/golang/go/issues/54794 is fixed in two latest releases of go @@ -45,16 +38,11 @@ func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http. return true } - // Lazy seeker enables efficient range-requests and HTTP HEAD responses - content := &lazySeeker{ - size: size, - reader: file, - } - + var content io.Reader = fileBytes // Calculate deterministic value for Content-Type HTTP header // (we prefer to do it here, rather than using implicit sniffing in http.ServeContent) var ctype string - if _, isSymlink := file.(*files.Symlink); isSymlink { + if isSymlink { // We should be smarter about resolving symlinks but this is the // "most correct" we can be without doing that. ctype = "inode/symlink" @@ -63,21 +51,24 @@ func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http. if ctype == "" { ctype = fileContentType } - if ctype == "" { + if ctype == "" && returnRangeStartsAtZero { // uses https://github.com/gabriel-vasile/mimetype library to determine the content type. // Fixes https://github.com/ipfs/kubo/issues/7252 - mimeType, err := mimetype.DetectReader(content) + + // We read from a TeeReader into a buffer and then put the buffer in front of the original reader to + // simulate the behavior of being able to read from the start and then seek back to the beginning while + // only having a Reader and not a ReadSeeker + var buf bytes.Buffer + tr := io.TeeReader(fileBytes, &buf) + + mimeType, err := mimetype.DetectReader(tr) if err != nil { http.Error(w, fmt.Sprintf("cannot detect content-type: %s", err.Error()), http.StatusInternalServerError) return false } ctype = mimeType.String() - _, err = content.Seek(0, io.SeekStart) - if err != nil { - http.Error(w, "seeker can't seek", http.StatusInternalServerError) - return false - } + content = io.MultiReader(&buf, fileBytes) } // Strip the encoding from the HTML Content-Type header and let the // browser figure it out. @@ -93,7 +84,7 @@ func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http. // ServeContent will take care of // If-None-Match+Etag, Content-Length and range requests - _, dataSent, _ := serveContent(w, r, name, modtime, content) + _, dataSent, _ := serveContent(w, r, modtime, fileSize, content) // Was response successful? if dataSent { diff --git a/gateway/metrics.go b/gateway/metrics.go index 69e81425f..6035c74b5 100644 --- a/gateway/metrics.go +++ b/gateway/metrics.go @@ -96,7 +96,7 @@ func (b *ipfsBackendWithMetrics) GetBlock(ctx context.Context, path ImmutablePat return md, n, err } -func (b *ipfsBackendWithMetrics) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { +func (b *ipfsBackendWithMetrics) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { begin := time.Now() name := "IPFSBackend.Head" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("path", path.String()))) diff --git a/gateway/serve_http_content.go b/gateway/serve_http_content.go new file mode 100644 index 000000000..2bb27ae04 --- /dev/null +++ b/gateway/serve_http_content.go @@ -0,0 +1,472 @@ +package gateway + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/textproto" + "strconv" + "strings" + "time" +) + +// errNoOverlap is returned by serveContent's parseRange if first-byte-pos of +// all of the byte-range-spec values is greater than the content size. +var errNoOverlap = errors.New("invalid range: failed to overlap") + +func headerGetExact(h http.Header, key string) string { + if v := h[key]; len(v) > 0 { + return v[0] + } + return "" +} + +// httpServeContent replies to the request using the content in the +// provided Reader. +// +// The main benefit of httpServeContent over io.Copy is that it handles Range requests properly, +// handles If-Match, If-Unmodified-Since, If-None-Match, If-Modified-Since, and If-Range requests. +// +// If modtime is not the zero time or Unix epoch, ServeContent +// includes it in a Last-Modified header in the response. If the +// request includes an If-Modified-Since header, ServeContent uses +// modtime to decide whether the content needs to be sent at all. +// +// If the caller has set w's ETag header formatted per RFC 7232, section 2.3, +// ServeContent uses it to handle requests using If-Match, If-None-Match, or If-Range. +// +// Notable differences from http.ServeContent +// 1. Takes an io.Reader instead of an io.ReaderSeeker +// 2. Requires the size to be passed in explicitly instead of discovered via Seeker behavior +// 3. Only handles a single HTTP Range, if multiple are requested it returns the first +// 4. The passed io.Reader must start at wherever the HTTP Range Request will start +// 4. Requires the Content-Type header to already be set +// 5. Does not require the name to be passed in for content sniffing +// 6. content may be nil for HEAD requests +func httpServeContent(w http.ResponseWriter, r *http.Request, modtime time.Time, size int64, content io.Reader) { + setLastModified(w, modtime) + done, rangeReq := checkPreconditions(w, r, modtime) + if done { + return + } + + code := http.StatusOK + + // handle Content-Range header. + sendSize := size + if size >= 0 { + ranges, err := parseRange(rangeReq, size) + if err != nil { + if err == errNoOverlap { + w.Header().Set("Content-Range", fmt.Sprintf("bytes */%d", size)) + } + http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) + return + } + if sumRangesSize(ranges) > size { + // The total number of bytes in all the ranges + // is larger than the size of the file by + // itself, so this is probably an attack, or a + // dumb client. Ignore the range request. + ranges = nil + } + + // We only support a single range request, if more than one is submitted we just send back the first + if len(ranges) > 0 { + ra := ranges[0] + // RFC 7233, Section 4.1: + // "If a single part is being transferred, the server + // generating the 206 response MUST generate a + // Content-Range header field, describing what range + // of the selected representation is enclosed, and a + // payload consisting of the range. + // ... + // A server MUST NOT generate a multipart response to + // a request for a single range, since a client that + // does not request multiple parts might not support + // multipart responses." + + sendSize = ra.length + code = http.StatusPartialContent + w.Header().Set("Content-Range", ra.contentRange(size)) + } + + w.Header().Set("Accept-Ranges", "bytes") + if w.Header().Get("Content-Encoding") == "" { + w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) + } + } + + w.WriteHeader(code) + + if r.Method != "HEAD" { + var sendContent io.Reader = content + io.CopyN(w, sendContent, sendSize) + } +} + +// scanETag determines if a syntactically valid ETag is present at s. If so, +// the ETag and remaining text after consuming ETag is returned. Otherwise, +// it returns "", "". +func scanETag(s string) (etag string, remain string) { + s = textproto.TrimString(s) + start := 0 + if strings.HasPrefix(s, "W/") { + start = 2 + } + if len(s[start:]) < 2 || s[start] != '"' { + return "", "" + } + // ETag is either W/"text" or "text". + // See RFC 7232 2.3. + for i := start + 1; i < len(s); i++ { + c := s[i] + switch { + // Character values allowed in ETags. + case c == 0x21 || c >= 0x23 && c <= 0x7E || c >= 0x80: + case c == '"': + return s[:i+1], s[i+1:] + default: + return "", "" + } + } + return "", "" +} + +// etagStrongMatch reports whether a and b match using strong ETag comparison. +// Assumes a and b are valid ETags. +func etagStrongMatch(a, b string) bool { + return a == b && a != "" && a[0] == '"' +} + +// etagWeakMatch reports whether a and b match using weak ETag comparison. +// Assumes a and b are valid ETags. +func etagWeakMatch(a, b string) bool { + return strings.TrimPrefix(a, "W/") == strings.TrimPrefix(b, "W/") +} + +// condResult is the result of an HTTP request precondition check. +// See https://tools.ietf.org/html/rfc7232 section 3. +type condResult int + +const ( + condNone condResult = iota + condTrue + condFalse +) + +func checkIfMatch(w http.ResponseWriter, r *http.Request) condResult { + im := r.Header.Get("If-Match") + if im == "" { + return condNone + } + for { + im = textproto.TrimString(im) + if len(im) == 0 { + break + } + if im[0] == ',' { + im = im[1:] + continue + } + if im[0] == '*' { + return condTrue + } + etag, remain := scanETag(im) + if etag == "" { + break + } + if etagStrongMatch(etag, headerGetExact(w.Header(), "Etag")) { + return condTrue + } + im = remain + } + + return condFalse +} + +func checkIfUnmodifiedSince(r *http.Request, modtime time.Time) condResult { + ius := r.Header.Get("If-Unmodified-Since") + if ius == "" || isZeroTime(modtime) { + return condNone + } + t, err := http.ParseTime(ius) + if err != nil { + return condNone + } + + // The Last-Modified header truncates sub-second precision so + // the modtime needs to be truncated too. + modtime = modtime.Truncate(time.Second) + if modtime.Before(t) || modtime.Equal(t) { + return condTrue + } + return condFalse +} + +func checkIfNoneMatch(w http.ResponseWriter, r *http.Request) condResult { + inm := headerGetExact(r.Header, "If-None-Match") + if inm == "" { + return condNone + } + buf := inm + for { + buf = textproto.TrimString(buf) + if len(buf) == 0 { + break + } + if buf[0] == ',' { + buf = buf[1:] + continue + } + if buf[0] == '*' { + return condFalse + } + etag, remain := scanETag(buf) + if etag == "" { + break + } + if etagWeakMatch(etag, headerGetExact(w.Header(), "Etag")) { + return condFalse + } + buf = remain + } + return condTrue +} + +func checkIfModifiedSince(r *http.Request, modtime time.Time) condResult { + if r.Method != "GET" && r.Method != "HEAD" { + return condNone + } + ims := r.Header.Get("If-Modified-Since") + if ims == "" || isZeroTime(modtime) { + return condNone + } + t, err := http.ParseTime(ims) + if err != nil { + return condNone + } + // The Last-Modified header truncates sub-second precision so + // the modtime needs to be truncated too. + modtime = modtime.Truncate(time.Second) + if modtime.Before(t) || modtime.Equal(t) { + return condFalse + } + return condTrue +} + +func checkIfRange(w http.ResponseWriter, r *http.Request, modtime time.Time) condResult { + if r.Method != "GET" && r.Method != "HEAD" { + return condNone + } + ir := headerGetExact(r.Header, "If-Range") + if ir == "" { + return condNone + } + etag, _ := scanETag(ir) + if etag != "" { + if etagStrongMatch(etag, w.Header().Get("Etag")) { + return condTrue + } else { + return condFalse + } + } + // The If-Range value is typically the ETag value, but it may also be + // the modtime date. See golang.org/issue/8367. + if modtime.IsZero() { + return condFalse + } + t, err := http.ParseTime(ir) + if err != nil { + return condFalse + } + if t.Unix() == modtime.Unix() { + return condTrue + } + return condFalse +} + +// isZeroTime reports whether t is obviously unspecified (either zero or Unix()=0). +func isZeroTime(t time.Time) bool { + return t.IsZero() || t.Equal(unixEpochTime) +} + +func setLastModified(w http.ResponseWriter, modtime time.Time) { + if !isZeroTime(modtime) { + w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat)) + } +} + +func writeNotModified(w http.ResponseWriter) { + // RFC 7232 section 4.1: + // a sender SHOULD NOT generate representation metadata other than the + // above listed fields unless said metadata exists for the purpose of + // guiding cache updates (e.g., Last-Modified might be useful if the + // response does not have an ETag field). + h := w.Header() + delete(h, "Content-Type") + delete(h, "Content-Length") + delete(h, "Content-Encoding") + if h.Get("Etag") != "" { + delete(h, "Last-Modified") + } + w.WriteHeader(http.StatusNotModified) +} + +// checkPreconditions evaluates request preconditions and reports whether a precondition +// resulted in sending StatusNotModified or StatusPreconditionFailed. +func checkPreconditions(w http.ResponseWriter, r *http.Request, modtime time.Time) (done bool, rangeHeader string) { + // This function carefully follows RFC 7232 section 6. + ch := checkIfMatch(w, r) + if ch == condNone { + ch = checkIfUnmodifiedSince(r, modtime) + } + if ch == condFalse { + w.WriteHeader(http.StatusPreconditionFailed) + return true, "" + } + switch checkIfNoneMatch(w, r) { + case condFalse: + if r.Method == "GET" || r.Method == "HEAD" { + writeNotModified(w) + return true, "" + } else { + w.WriteHeader(http.StatusPreconditionFailed) + return true, "" + } + case condNone: + if checkIfModifiedSince(r, modtime) == condFalse { + writeNotModified(w) + return true, "" + } + } + + rangeHeader = headerGetExact(r.Header, "Range") + if rangeHeader != "" && checkIfRange(w, r, modtime) == condFalse { + rangeHeader = "" + } + return false, rangeHeader +} + +// httpRange specifies the byte range to be sent to the client. +type httpRange struct { + start, length int64 +} + +func (r httpRange) contentRange(size int64) string { + return fmt.Sprintf("bytes %d-%d/%d", r.start, r.start+r.length-1, size) +} + +// parseRange parses a Range header string as per RFC 7233. +// errNoOverlap is returned if none of the ranges overlap. +func parseRange(s string, size int64) ([]httpRange, error) { + if s == "" { + return nil, nil // header not present + } + const b = "bytes=" + if !strings.HasPrefix(s, b) { + return nil, errors.New("invalid range") + } + var ranges []httpRange + noOverlap := false + for _, ra := range strings.Split(s[len(b):], ",") { + ra = textproto.TrimString(ra) + if ra == "" { + continue + } + start, end, ok := strings.Cut(ra, "-") + if !ok { + return nil, errors.New("invalid range") + } + start, end = textproto.TrimString(start), textproto.TrimString(end) + var r httpRange + if start == "" { + // If no start is specified, end specifies the + // range start relative to the end of the file, + // and we are dealing with + // which has to be a non-negative integer as per + // RFC 7233 Section 2.1 "Byte-Ranges". + if end == "" || end[0] == '-' { + return nil, errors.New("invalid range") + } + i, err := strconv.ParseInt(end, 10, 64) + if i < 0 || err != nil { + return nil, errors.New("invalid range") + } + if i > size { + i = size + } + r.start = size - i + r.length = size - r.start + } else { + i, err := strconv.ParseInt(start, 10, 64) + if err != nil || i < 0 { + return nil, errors.New("invalid range") + } + if i >= size { + // If the range begins after the size of the content, + // then it does not overlap. + noOverlap = true + continue + } + r.start = i + if end == "" { + // If no end is specified, range extends to end of the file. + r.length = size - r.start + } else { + i, err := strconv.ParseInt(end, 10, 64) + if err != nil || r.start > i { + return nil, errors.New("invalid range") + } + if i >= size { + i = size - 1 + } + r.length = i - r.start + 1 + } + } + ranges = append(ranges, r) + } + if noOverlap && len(ranges) == 0 { + // The specified ranges did not overlap with the content. + return nil, errNoOverlap + } + return ranges, nil +} + +func sumRangesSize(ranges []httpRange) (size int64) { + for _, ra := range ranges { + size += ra.length + } + return +} + +// seekToStartOfFirstRange seeks to the start of the first Range if the request is an HTTP Range Request +func (i *handler) seekToStartOfFirstRange(w http.ResponseWriter, r *http.Request, data io.Seeker) bool { + rangeHeader := r.Header.Get("Range") + if rangeHeader != "" { + ranges, err := parseRangeWithoutLength(rangeHeader) + if err != nil { + i.webError(w, r, fmt.Errorf("invalid range request: %w", err), http.StatusBadRequest) + return false + } + if len(ranges) > 0 { + ra := &ranges[0] + err = seekToRangeStart(data, ra) + if err != nil { + i.webError(w, r, fmt.Errorf("could not seek to location in range request: %w", err), http.StatusBadRequest) + return false + } + } + } + return true +} + +func seekToRangeStart(data io.Seeker, ra *ByteRange) error { + if ra != nil && ra.From != 0 { + if _, err := data.Seek(int64(ra.From), io.SeekStart); err != nil { + return err + } + } + return nil +} diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index 27ba43a14..9d58d8c11 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -145,7 +145,7 @@ func (mb *mockBackend) GetBlock(ctx context.Context, immutablePath ImmutablePath return mb.gw.GetBlock(ctx, immutablePath) } -func (mb *mockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { +func (mb *mockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { return mb.gw.Head(ctx, immutablePath) } From 5ae64f335dd4ccedcb5b63108ccc8fe8e4408843 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 4 Oct 2023 10:13:25 +0200 Subject: [PATCH 05/63] chore: bump examples boxo version --- examples/README.md | 6 +++--- examples/go.mod | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/README.md b/examples/README.md index 438766e4c..1c63f087d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,8 +1,8 @@ -# boxo examples and tutorials +# Boxo Examples and Tutorials -In this folder, you can find some examples to help you get started using boxo and its associated libraries in your applications. +In this folder, you can find some examples to help you get started using Boxo and its associated libraries in your applications. -Let us know if you find any issue or if you want to contribute and add a new tutorial, feel welcome to submit a pr, thank you! +Let us know if you find any issue or if you want to contribute and add a new tutorial, feel welcome to submit a PR, thank you! ## Examples and Tutorials diff --git a/examples/go.mod b/examples/go.mod index 495c2455a..790573a69 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -3,7 +3,7 @@ module github.com/ipfs/boxo/examples go 1.20 require ( - github.com/ipfs/boxo v0.8.0 + github.com/ipfs/boxo v0.13.1 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 From 4295a4dd31f5242ae12f295228061097b591aab9 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 4 Oct 2023 13:13:57 +0200 Subject: [PATCH 06/63] feat(routing): export routing/http/client.Client --- CHANGELOG.md | 2 ++ routing/http/client/client.go | 34 +++++++++++++++--------------- routing/http/client/client_test.go | 2 +- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb864a5a7..2afccc4b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ The following emojis are used to highlight certain changes: * 🛠 The `IPFSBackend` interface was updated to make the responses of the `Head` method more explicit. It now returns a `HeadResponse` instead of a `files.Node`. +* `boxo/routing/http/client.Client` is now exported. This means you can now pass + it around functions, or add it to a struct if you want. ### Removed diff --git a/routing/http/client/client.go b/routing/http/client/client.go index 4a0d29b33..39dd698cb 100644 --- a/routing/http/client/client.go +++ b/routing/http/client/client.go @@ -28,7 +28,7 @@ import ( ) var ( - _ contentrouter.Client = &client{} + _ contentrouter.Client = &Client{} logger = logging.Logger("routing/http/client") defaultHTTPClient = &http.Client{ Transport: &ResponseBodyLimitedTransport{ @@ -45,7 +45,7 @@ const ( mediaTypeIPNSRecord = "application/vnd.ipfs.ipns-record" ) -type client struct { +type Client struct { baseURL string httpClient httpClient clock clock.Clock @@ -65,28 +65,28 @@ type client struct { // version sent a request var defaultUserAgent = moduleVersion() -var _ contentrouter.Client = &client{} +var _ contentrouter.Client = &Client{} type httpClient interface { Do(req *http.Request) (*http.Response, error) } -type Option func(*client) +type Option func(*Client) func WithIdentity(identity crypto.PrivKey) Option { - return func(c *client) { + return func(c *Client) { c.identity = identity } } func WithHTTPClient(h httpClient) Option { - return func(c *client) { + return func(c *Client) { c.httpClient = h } } func WithUserAgent(ua string) Option { - return func(c *client) { + return func(c *Client) { if ua == "" { return } @@ -103,7 +103,7 @@ func WithUserAgent(ua string) Option { } func WithProviderInfo(peerID peer.ID, addrs []multiaddr.Multiaddr) Option { - return func(c *client) { + return func(c *Client) { c.peerID = peerID for _, a := range addrs { c.addrs = append(c.addrs, types.Multiaddr{Multiaddr: a}) @@ -112,15 +112,15 @@ func WithProviderInfo(peerID peer.ID, addrs []multiaddr.Multiaddr) Option { } func WithStreamResultsRequired() Option { - return func(c *client) { + return func(c *Client) { c.accepts = mediaTypeNDJSON } } // New creates a content routing API client. // The Provider and identity parameters are option. If they are nil, the [client.ProvideBitswap] method will not function. -func New(baseURL string, opts ...Option) (*client, error) { - client := &client{ +func New(baseURL string, opts ...Option) (*Client, error) { + client := &Client{ baseURL: baseURL, httpClient: defaultHTTPClient, clock: clock.New(), @@ -160,7 +160,7 @@ func (c *measuringIter[T]) Close() error { return c.Iter.Close() } -func (c *client) FindProviders(ctx context.Context, key cid.Cid) (providers iter.ResultIter[types.Record], err error) { +func (c *Client) FindProviders(ctx context.Context, key cid.Cid) (providers iter.ResultIter[types.Record], err error) { // TODO test measurements m := newMeasurement("FindProviders") @@ -237,7 +237,7 @@ func (c *client) FindProviders(ctx context.Context, key cid.Cid) (providers iter // Deprecated: protocol-agnostic provide is being worked on in [IPIP-378]: // // [IPIP-378]: https://github.com/ipfs/specs/pull/378 -func (c *client) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error) { +func (c *Client) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error) { if c.identity == nil { return 0, errors.New("cannot provide Bitswap records without an identity") } @@ -283,7 +283,7 @@ func (c *client) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Du // ProvideAsync makes a provide request to a delegated router // //lint:ignore SA1019 // ignore staticcheck -func (c *client) provideSignedBitswapRecord(ctx context.Context, bswp *types.WriteBitswapRecord) (time.Duration, error) { +func (c *Client) provideSignedBitswapRecord(ctx context.Context, bswp *types.WriteBitswapRecord) (time.Duration, error) { //lint:ignore SA1019 // ignore staticcheck req := jsontypes.WriteProvidersRequest{Providers: []types.Record{bswp}} @@ -332,7 +332,7 @@ func (c *client) provideSignedBitswapRecord(ctx context.Context, bswp *types.Wri return 0, nil } -func (c *client) FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) { +func (c *Client) FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) { m := newMeasurement("FindPeers") url := c.baseURL + "/routing/v1/peers/" + peer.ToCid(pid).String() @@ -405,7 +405,7 @@ func (c *client) FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultI return &measuringIter[iter.Result[types.Record]]{Iter: it, ctx: ctx, m: m}, nil } -func (c *client) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { +func (c *Client) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { url := c.baseURL + "/routing/v1/ipns/" + name.String() httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) @@ -443,7 +443,7 @@ func (c *client) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, err return record, nil } -func (c *client) PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { +func (c *Client) PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { url := c.baseURL + "/routing/v1/ipns/" + name.String() rawRecord, err := ipns.MarshalRecord(record) diff --git a/routing/http/client/client_test.go b/routing/http/client/client_test.go index 95683bc3f..3822862dd 100644 --- a/routing/http/client/client_test.go +++ b/routing/http/client/client_test.go @@ -67,7 +67,7 @@ type testDeps struct { server *httptest.Server peerID peer.ID addrs []multiaddr.Multiaddr - client *client + client *Client } type recordingHandler struct { From b3886904df5d92a6d0b4b4b41b0e09779c2f4d57 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 4 Oct 2023 13:52:21 +0200 Subject: [PATCH 07/63] feat(examples): routing v1 client cli --- examples/README.md | 1 + examples/go.mod | 1 + examples/go.sum | 4 + .../delegated-routing-client/.gitignore | 1 + .../delegated-routing-client/README.md | 71 +++++++++ .../routing/delegated-routing-client/main.go | 148 ++++++++++++++++++ .../delegated-routing-client/main_test.go | 96 ++++++++++++ 7 files changed, 322 insertions(+) create mode 100644 examples/routing/delegated-routing-client/.gitignore create mode 100644 examples/routing/delegated-routing-client/README.md create mode 100644 examples/routing/delegated-routing-client/main.go create mode 100644 examples/routing/delegated-routing-client/main_test.go diff --git a/examples/README.md b/examples/README.md index 1c63f087d..c0795436d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -9,3 +9,4 @@ Let us know if you find any issue or if you want to contribute and add a new tut - [Fetching a UnixFS file by CID](./unixfs-file-cid) - [Gateway backed by a CAR file](./gateway/car) - [Gateway backed by a remote blockstore and IPNS resolver](./gateway/proxy) +- [Delegated Routing V1 Command Line Client](./routing/delegated-routing-client/) diff --git a/examples/go.mod b/examples/go.mod index 790573a69..24bcbe929 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -125,6 +125,7 @@ require ( github.com/quic-go/quic-go v0.38.0 // indirect github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect + github.com/samber/lo v1.36.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect diff --git a/examples/go.sum b/examples/go.sum index eaec6f954..93ee14dd3 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -216,6 +216,7 @@ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -478,6 +479,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/samber/lo v1.36.0 h1:4LaOxH1mHnbDGhTVE0i1z8v/lWaQW8AIfOD3HU4mSaw= +github.com/samber/lo v1.36.0/go.mod h1:HLeWcJRRyLKp3+/XBJvOrerCQn9mhdKMHyd7IRlgeQ8= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/component v0.0.0-20170202220835-f88ec8f54cc4/go.mod h1:XhFIlyj5a1fBNx5aJTbKoIq0mNaPvOagO+HjB3EtxrY= github.com/shurcooL/events v0.0.0-20181021180414-410e4ca65f48/go.mod h1:5u70Mqkb5O5cxEA8nxTsgrgLehJeAw6Oc4Ab1c/P1HM= @@ -530,6 +533,7 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= +github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ= github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM= diff --git a/examples/routing/delegated-routing-client/.gitignore b/examples/routing/delegated-routing-client/.gitignore new file mode 100644 index 000000000..2555457e2 --- /dev/null +++ b/examples/routing/delegated-routing-client/.gitignore @@ -0,0 +1 @@ +delegated-routing-client \ No newline at end of file diff --git a/examples/routing/delegated-routing-client/README.md b/examples/routing/delegated-routing-client/README.md new file mode 100644 index 000000000..3e34e4531 --- /dev/null +++ b/examples/routing/delegated-routing-client/README.md @@ -0,0 +1,71 @@ +# Delegated Routing V1 Command Line Client + +This is an example of how to use the Delegated Routing V1 HTTP client from Boxo. +In this package, we build a small command line tool that allows you to connect to +a Routing V1 endpoint and fetch content providers, peer information, as well as +IPNS records for a certain IPNS name. + +## Build + +```bash +> go build -o delegated-routing-client +``` + +## Usage + +First, you will need a HTTP endpoint compatible with [Delegated Routing V1 Specification][Specification]. +For that, you can potentially use [Kubo], which supports [exposing][kubo-conf] +a `/routing/v1` endpoint. For the commands below, we assume the HTTP server that +provides the endpoint is `http://127.0.0.1:8080`. + +### Find CID Providers + +To find providers, provide the flag `-cid` with the [CID] of the content you're looking for: + +```console +$ ./delegated-routing-client -e http://127.0.0.1:8080 -cid bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4 + +12D3KooWEfL19QqRGGLraaAYw1XA3dtDdVRYaHt6jymFxcuQo3Zm + Protocols: [] + Addresses: [/ip4/163.47.51.218/tcp/28131] +12D3KooWK53GAx2g2UUYfJHHjxDbVLeDgGxNMHXDWeJa5KgMhTD2 + Protocols: [] + Addresses: [/ip4/195.167.147.43/udp/8888/quic /ip4/195.167.147.43/tcp/8888] +12D3KooWCpr8kACTRLKrPy4LPpSX7LXvKQ7eYqTmY8CBvgK5HZgB + Protocols: [] + Addresses: [/ip4/163.47.49.234/tcp/28102] +12D3KooWC9L4RjPGgqpzBUBkcVpKjJYofCkC5i5QdQftg1LdsFb2 + Protocols: [] + Addresses: [/ip4/198.244.201.187/tcp/4001] +``` + +### Find Peer Information + +To find a peer, provide the flag `-peer` with the [Peer ID] of the peer you're looking for: + + +```console +$ ./delegated-routing-client -e http://127.0.0.1:8080 -peer 12D3KooWC9L4RjPGgqpzBUBkcVpKjJYofCkC5i5QdQftg1LdsFb2 + +12D3KooWC9L4RjPGgqpzBUBkcVpKjJYofCkC5i5QdQftg1LdsFb2 + Protocols: [] + Addresses: [/ip4/198.244.201.187/tcp/4001] +``` + +### Get an IPNS Record + +To find an IPNS record, provide the flag `-ipns` with the [IPNS Name] you're trying to find a record for: + +```console +$ ./delegated-routing-client -e http://127.0.0.1:8080 -ipns /ipns/k51qzi5uqu5diuz0h5tjqama8qbmyxusvqz2hfgn5go5l07l9k2ubqa09m7toe + +/ipns/k51qzi5uqu5diuz0h5tjqama8qbmyxusvqz2hfgn5go5l07l9k2ubqa09m7toe + Value: /ipfs/QmUGMoVz62ZARyxkrdEiwmFZanTwVWLLu6EAWvbWHNcwR8 +``` + +[Specification]: https://specs.ipfs.tech/routing/http-routing-v1/ +[Kubo]: https://github.com/ipfs/kubo +[kubo-conf]: https://github.com/ipfs/kubo/blob/master/docs/config.md#gatewayexposeroutingapi +[CID]: https://docs.ipfs.tech/concepts/content-addressing/#what-is-a-cid +[Peer ID]: https://docs.libp2p.io/concepts/fundamentals/peers/#peer-id +[IPNS Name]: https://specs.ipfs.tech/ipns/ipns-record/#ipns-name diff --git a/examples/routing/delegated-routing-client/main.go b/examples/routing/delegated-routing-client/main.go new file mode 100644 index 000000000..0b586d4bf --- /dev/null +++ b/examples/routing/delegated-routing-client/main.go @@ -0,0 +1,148 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "log" + "os" + "time" + + "github.com/ipfs/boxo/ipns" + "github.com/ipfs/boxo/routing/http/client" + "github.com/ipfs/boxo/routing/http/types" + "github.com/ipfs/boxo/routing/http/types/iter" + "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/peer" +) + +func main() { + gatewayUrlPtr := flag.String("e", "", "routing v1 endpoint to use") + timeoutPtr := flag.Int("t", 10, "timeout in seconds for lookup") + cidPtr := flag.String("cid", "", "cid to find") + pidPtr := flag.String("peer", "", "peer to find") + namePtr := flag.String("ipns", "", "ipns name to retrieve record for") + flag.Parse() + + if err := run(os.Stdout, *gatewayUrlPtr, *cidPtr, *pidPtr, *namePtr, *timeoutPtr); err != nil { + log.Fatal(err) + } +} + +func run(w io.Writer, gatewayURL, cidStr, pidStr, nameStr string, timeoutSeconds int) error { + // Creates a new Delegated Routing V1 client. + client, err := client.New(gatewayURL) + if err != nil { + return err + } + + timeout := time.Duration(timeoutSeconds) * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + if cidStr != "" { + return findProviders(w, ctx, client, cidStr) + } else if pidStr != "" { + return findPeers(w, ctx, client, pidStr) + } else if nameStr != "" { + return findIPNS(w, ctx, client, nameStr) + } else { + return errors.New("cid or peer must be provided") + } +} + +func findProviders(w io.Writer, ctx context.Context, client *client.Client, cidStr string) error { + // Parses the given CID to lookup the providers for. + contentCid, err := cid.Parse(cidStr) + if err != nil { + return err + } + + // Ask for providers providing the given content CID. + recordsIter, err := client.FindProviders(ctx, contentCid) + if err != nil { + return err + } + defer recordsIter.Close() + return printIter(w, recordsIter) +} + +func findPeers(w io.Writer, ctx context.Context, client *client.Client, pidStr string) error { + // Parses the given Peer ID to lookup the information for. + pid, err := peer.Decode(pidStr) + if err != nil { + return err + } + + // Ask for information about the peer with the given peer ID. + recordsIter, err := client.FindPeers(ctx, pid) + if err != nil { + return err + } + defer recordsIter.Close() + return printIter(w, recordsIter) +} + +func printIter(w io.Writer, iter iter.ResultIter[types.Record]) error { + // The response is streamed. Alternatively, you could use [iter.ReadAll] + // to fetch all the results all at once, instead of iterating as they are + // streamed. + for iter.Next() { + res := iter.Val() + + // Check for error, but do not complain if we exceeded the timeout. We are + // expecting that to happen: we explicitly defined a timeout. + if res.Err != nil { + if !errors.Is(res.Err, context.DeadlineExceeded) { + return res.Err + } + + return nil + } + + switch res.Val.GetSchema() { + case types.SchemaPeer: + record := res.Val.(*types.PeerRecord) + fmt.Fprintln(w, record.ID) + fmt.Fprintln(w, "\tProtocols:", record.Protocols) + fmt.Fprintln(w, "\tAddresses:", record.Addrs) + default: + // You may not want to fail here, it's up to you. You can just handle + // the schemas you want, or that you know, but not fail. + log.Printf("unrecognized schema: %s", res.Val.GetSchema()) + } + } + + return nil +} + +func findIPNS(w io.Writer, ctx context.Context, client *client.Client, nameStr string) error { + // Parses the given name string to get a record for. + name, err := ipns.NameFromString(nameStr) + if err != nil { + return err + } + + // Fetch an IPNS record for the given name. [client.Client.GetIPNS] verifies + // if the retrieved record is valid against the given name, and errors otherwise. + record, err := client.GetIPNS(ctx, name) + if err != nil { + return err + } + + fmt.Fprintf(w, "/ipns/%s\n", name) + v, err := record.Value() + if err != nil { + return err + } + + // Since [client.Client.GetIPNS] verifies if the retrieved record is valid, we + // do not need to verify it again. However, if you were not using this specific + // client, but using some other tool, you should always validate the IPNS Record + // using the [ipns.Validate] or [ipns.ValidateWithName] functions. + fmt.Fprintln(w, "\tSignature: VALID") + fmt.Fprintln(w, "\tValue:", v.String()) + return nil +} diff --git a/examples/routing/delegated-routing-client/main_test.go b/examples/routing/delegated-routing-client/main_test.go new file mode 100644 index 000000000..7ad813f13 --- /dev/null +++ b/examples/routing/delegated-routing-client/main_test.go @@ -0,0 +1,96 @@ +package main + +import ( + "bytes" + "crypto/rand" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/ipns" + ipfspath "github.com/ipfs/boxo/path" + "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFindProviders(t *testing.T) { + cidStr := "bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4" + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/routing/v1/providers/"+cidStr { + w.Header().Set("Content-Type", "application/x-ndjson") + w.Write([]byte(`{"Schema":"peer","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Addrs":["/ip4/111.222.222.111/tcp/5734"],"Protocols":["transport-bitswap"]}` + "\n")) + w.Write([]byte(`{"Schema":"peer","ID":"12D3KooWB6RAWgcmHAP7TGEGK7utV2ZuqSzX1DNjRa97TtJ7139n","Addrs":["/ip4/127.0.0.1/tcp/5734"],"Protocols":["transport-horse"]}` + "\n")) + } + })) + t.Cleanup(ts.Close) + + out := &bytes.Buffer{} + err := run(out, ts.URL, cidStr, "", "", 1) + assert.Contains(t, out.String(), "12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn\n\tProtocols: [transport-bitswap]\n\tAddresses: [/ip4/111.222.222.111/tcp/5734]\n") + assert.Contains(t, out.String(), "12D3KooWB6RAWgcmHAP7TGEGK7utV2ZuqSzX1DNjRa97TtJ7139n\n\tProtocols: [transport-horse]\n\tAddresses: [/ip4/127.0.0.1/tcp/5734]\n") + assert.NoError(t, err) +} + +func TestFindPeers(t *testing.T) { + pidStr := "bafzaajaiaejcbkboq2tin6dkdc2vinbbn2dgowzn3u5izpjwxejheogw23scafkz" + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/routing/v1/peers/"+pidStr { + w.Header().Set("Content-Type", "application/x-ndjson") + w.Write([]byte(`{"Schema":"peer","ID":"12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn","Addrs":["/ip4/111.222.222.111/tcp/5734"],"Protocols":["transport-bitswap"]}` + "\n")) + } + })) + t.Cleanup(ts.Close) + + out := &bytes.Buffer{} + err := run(out, ts.URL, "", pidStr, "", 1) + assert.Contains(t, out.String(), "12D3KooWM8sovaEGU1bmiWGWAzvs47DEcXKZZTuJnpQyVTkRs2Vn\n\tProtocols: [transport-bitswap]\n\tAddresses: [/ip4/111.222.222.111/tcp/5734]\n") + assert.NoError(t, err) +} + +func TestGetIPNS(t *testing.T) { + name, rec := makeNameAndRecord(t) + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/routing/v1/ipns/"+name.String() { + w.Header().Set("Content-Type", "application/vnd.ipfs.ipns-record") + w.Write(rec) + } + })) + t.Cleanup(ts.Close) + + out := &bytes.Buffer{} + err := run(out, ts.URL, "", "", name.String(), 1) + assert.Contains(t, out.String(), fmt.Sprintf("/ipns/%s\n\tSignature: VALID\n\tValue: /ipfs/bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4\n", name.String())) + assert.NoError(t, err) +} + +func makeNameAndRecord(t *testing.T) (ipns.Name, []byte) { + sk, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + pid, err := peer.IDFromPrivateKey(sk) + require.NoError(t, err) + + cid, err := cid.Decode("bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") + require.NoError(t, err) + + path := path.IpfsPath(cid) + eol := time.Now().Add(time.Hour * 48) + ttl := time.Second * 20 + + record, err := ipns.NewRecord(sk, ipfspath.FromString(path.String()), 1, eol, ttl) + require.NoError(t, err) + + rawRecord, err := ipns.MarshalRecord(record) + require.NoError(t, err) + + return ipns.NameFromPeer(pid), rawRecord +} From 85c180e26664367f79de453c592020b4f279669f Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Fri, 6 Oct 2023 16:04:23 +0200 Subject: [PATCH 08/63] feat(path)!: consolidated path libraries (#334) --- CHANGELOG.md | 11 + coreiface/block.go | 5 +- coreiface/coreapi.go | 9 +- coreiface/dht.go | 2 +- coreiface/key.go | 2 +- coreiface/name.go | 5 +- coreiface/object.go | 17 +- coreiface/path/path.go | 199 ---------- coreiface/pin.go | 6 +- coreiface/tests/block.go | 37 +- coreiface/tests/dag.go | 13 +- coreiface/tests/name.go | 21 +- coreiface/tests/object.go | 14 +- coreiface/tests/path.go | 213 ++++------ coreiface/tests/pin.go | 89 +++-- coreiface/tests/routing.go | 2 +- coreiface/tests/unixfs.go | 28 +- coreiface/unixfs.go | 11 +- .../delegated-routing-client/main_test.go | 7 +- gateway/assets/assets.go | 5 +- gateway/blocks_backend.go | 151 ++++--- gateway/errors.go | 11 +- gateway/gateway.go | 55 +-- gateway/gateway_test.go | 76 ++-- gateway/handler.go | 76 ++-- gateway/handler_block.go | 2 +- gateway/handler_car.go | 5 +- gateway/handler_car_test.go | 4 +- gateway/handler_codec.go | 29 +- gateway/handler_codec_test.go | 9 +- gateway/handler_defaults.go | 2 +- gateway/handler_ipns_record.go | 3 +- gateway/handler_tar.go | 2 +- gateway/handler_unixfs__redirects.go | 81 ++-- gateway/handler_unixfs_dir.go | 30 +- gateway/handler_unixfs_dir_test.go | 15 +- gateway/handler_unixfs_file.go | 6 +- gateway/hostname_test.go | 6 +- gateway/metrics.go | 16 +- gateway/utilities_test.go | 31 +- ipns/README.md | 5 +- ipns/name.go | 10 + ipns/name_test.go | 11 + ipns/record.go | 12 +- ipns/record_test.go | 19 +- ipns/validation_test.go | 16 +- mfs/mfs_test.go | 6 +- mfs/ops.go | 10 +- namesys/base.go | 2 +- namesys/cache.go | 8 +- namesys/dns.go | 32 +- namesys/namesys.go | 17 +- namesys/namesys_test.go | 15 +- namesys/publisher.go | 2 +- namesys/publisher_test.go | 7 +- namesys/republisher/repub.go | 3 +- namesys/republisher/repub_test.go | 16 +- namesys/resolve/resolve.go | 18 +- namesys/resolve_test.go | 24 +- namesys/routing.go | 2 +- path/error.go | 23 +- path/error_test.go | 6 +- path/internal/tracing.go | 13 - path/path.go | 304 +++++++------- path/path_test.go | 375 +++++++++++++----- path/resolver/resolver.go | 192 +++------ path/resolver/resolver_test.go | 164 ++++---- routing/http/client/client_test.go | 7 +- .../http/contentrouter/contentrouter_test.go | 7 +- routing/http/server/server_test.go | 7 +- 70 files changed, 1298 insertions(+), 1341 deletions(-) delete mode 100644 coreiface/path/path.go delete mode 100644 path/internal/tracing.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2afccc4b5..111044407 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,17 @@ The following emojis are used to highlight certain changes: `files.Node`. * `boxo/routing/http/client.Client` is now exported. This means you can now pass it around functions, or add it to a struct if you want. +* 🛠 The `path` package has been massively refactored. With this refactor, we have + condensed the different path-related and/or Kubo-specific packages under a single generic one. Therefore, there + are many breaking changes. Please consult the [documentation](https://pkg.go.dev/github.com/ipfs/boxo/path) + for more details on how to use the new package. + * Note: content paths created with `boxo/path` are automatically normalized: + - Replace multiple slashes with a single slash. + - Eliminate each `.` path name element (the current directory). + - Eliminate each inner `..` path name element (the parent directory) along with the non-`..` element that precedes it. + - Eliminate `..` elements that begin a rooted path: that is, replace "`/..`" by "`/`" at the beginning of a path. +* 🛠 The signature of `CoreAPI.ResolvePath` in `coreiface` has changed to now return + the remainder segments as a second return value, matching the signature of `resolver.ResolveToLastNode`. ### Removed diff --git a/coreiface/block.go b/coreiface/block.go index dbe31e9f8..cdd5fcee2 100644 --- a/coreiface/block.go +++ b/coreiface/block.go @@ -4,9 +4,8 @@ import ( "context" "io" - path "github.com/ipfs/boxo/coreiface/path" - "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/path" ) // BlockStat contains information about a block @@ -15,7 +14,7 @@ type BlockStat interface { Size() int // Path returns path to the block - Path() path.Resolved + Path() path.ImmutablePath } // BlockAPI specifies the interface to the block layer diff --git a/coreiface/coreapi.go b/coreiface/coreapi.go index 7276a3f60..25e54a37b 100644 --- a/coreiface/coreapi.go +++ b/coreiface/coreapi.go @@ -5,9 +5,8 @@ package iface import ( "context" - path "github.com/ipfs/boxo/coreiface/path" - "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/path" ipld "github.com/ipfs/go-ipld-format" ) @@ -47,8 +46,10 @@ type CoreAPI interface { // Routing returns an implementation of Routing API Routing() RoutingAPI - // ResolvePath resolves the path using Unixfs resolver - ResolvePath(context.Context, path.Path) (path.Resolved, error) + // ResolvePath resolves the path using UnixFS resolver, and returns the resolved + // immutable path, and the remainder of the path segments that cannot be resolved + // within UnixFS. + ResolvePath(context.Context, path.Path) (path.ImmutablePath, []string, error) // ResolveNode resolves the path (if not resolved already) using Unixfs // resolver, gets and returns the resolved Node diff --git a/coreiface/dht.go b/coreiface/dht.go index 93027a406..d9418ebfc 100644 --- a/coreiface/dht.go +++ b/coreiface/dht.go @@ -3,7 +3,7 @@ package iface import ( "context" - "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/coreiface/options" diff --git a/coreiface/key.go b/coreiface/key.go index 118fe2e4f..4a1cbae80 100644 --- a/coreiface/key.go +++ b/coreiface/key.go @@ -3,7 +3,7 @@ package iface import ( "context" - "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/coreiface/options" diff --git a/coreiface/name.go b/coreiface/name.go index 8c3e8e89a..f832033ef 100644 --- a/coreiface/name.go +++ b/coreiface/name.go @@ -4,10 +4,9 @@ import ( "context" "errors" - path "github.com/ipfs/boxo/coreiface/path" - "github.com/ipfs/boxo/ipns" - "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/ipns" + "github.com/ipfs/boxo/path" ) var ErrResolveFailed = errors.New("could not resolve name") diff --git a/coreiface/object.go b/coreiface/object.go index d983fa49b..4a73f22ea 100644 --- a/coreiface/object.go +++ b/coreiface/object.go @@ -4,9 +4,8 @@ import ( "context" "io" - path "github.com/ipfs/boxo/coreiface/path" - "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" @@ -60,11 +59,11 @@ type ObjectChange struct { // Before holds the link path before the change. Note that when a link is // added, this will be nil. - Before path.Resolved + Before path.ImmutablePath // After holds the link path after the change. Note that when a link is // removed, this will be nil. - After path.Resolved + After path.ImmutablePath } // ObjectAPI specifies the interface to MerkleDAG and contains useful utilities @@ -74,7 +73,7 @@ type ObjectAPI interface { New(context.Context, ...options.ObjectNewOption) (ipld.Node, error) // Put imports the data into merkledag - Put(context.Context, io.Reader, ...options.ObjectPutOption) (path.Resolved, error) + Put(context.Context, io.Reader, ...options.ObjectPutOption) (path.ImmutablePath, error) // Get returns the node for the path Get(context.Context, path.Path) (ipld.Node, error) @@ -91,16 +90,16 @@ type ObjectAPI interface { // AddLink adds a link under the specified path. child path can point to a // subdirectory within the patent which must be present (can be overridden // with WithCreate option). - AddLink(ctx context.Context, base path.Path, name string, child path.Path, opts ...options.ObjectAddLinkOption) (path.Resolved, error) + AddLink(ctx context.Context, base path.Path, name string, child path.Path, opts ...options.ObjectAddLinkOption) (path.ImmutablePath, error) // RmLink removes a link from the node - RmLink(ctx context.Context, base path.Path, link string) (path.Resolved, error) + RmLink(ctx context.Context, base path.Path, link string) (path.ImmutablePath, error) // AppendData appends data to the node - AppendData(context.Context, path.Path, io.Reader) (path.Resolved, error) + AppendData(context.Context, path.Path, io.Reader) (path.ImmutablePath, error) // SetData sets the data contained in the node - SetData(context.Context, path.Path, io.Reader) (path.Resolved, error) + SetData(context.Context, path.Path, io.Reader) (path.ImmutablePath, error) // Diff returns a set of changes needed to transform the first object into the // second. diff --git a/coreiface/path/path.go b/coreiface/path/path.go deleted file mode 100644 index c26b8692b..000000000 --- a/coreiface/path/path.go +++ /dev/null @@ -1,199 +0,0 @@ -package path - -import ( - "strings" - - ipfspath "github.com/ipfs/boxo/path" - cid "github.com/ipfs/go-cid" -) - -// Path is a generic wrapper for paths used in the API. A path can be resolved -// to a CID using one of Resolve functions in the API. -// -// Paths must be prefixed with a valid prefix: -// -// * /ipfs - Immutable unixfs path (files) -// * /ipld - Immutable ipld path (data) -// * /ipns - Mutable names. Usually resolves to one of the immutable paths -// TODO: /local (MFS) -type Path interface { - // String returns the path as a string. - String() string - - // Namespace returns the first component of the path. - // - // For example path "/ipfs/QmHash", calling Namespace() will return "ipfs" - // - // Calling this method on invalid paths (IsValid() != nil) will result in - // empty string - Namespace() string - - // Mutable returns false if the data pointed to by this path in guaranteed - // to not change. - // - // Note that resolved mutable path can be immutable. - Mutable() bool - - // IsValid checks if this path is a valid ipfs Path, returning nil iff it is - // valid - IsValid() error -} - -// Resolved is a path which was resolved to the last resolvable node. -// ResolvedPaths are guaranteed to return nil from `IsValid` -type Resolved interface { - // Cid returns the CID of the node referenced by the path. Remainder of the - // path is guaranteed to be within the node. - // - // Examples: - // If you have 3 linked objects: QmRoot -> A -> B: - // - // cidB := {"foo": {"bar": 42 }} - // cidA := {"B": {"/": cidB }} - // cidRoot := {"A": {"/": cidA }} - // - // And resolve paths: - // - // * "/ipfs/${cidRoot}" - // * Calling Cid() will return `cidRoot` - // * Calling Root() will return `cidRoot` - // * Calling Remainder() will return `` - // - // * "/ipfs/${cidRoot}/A" - // * Calling Cid() will return `cidA` - // * Calling Root() will return `cidRoot` - // * Calling Remainder() will return `` - // - // * "/ipfs/${cidRoot}/A/B/foo" - // * Calling Cid() will return `cidB` - // * Calling Root() will return `cidRoot` - // * Calling Remainder() will return `foo` - // - // * "/ipfs/${cidRoot}/A/B/foo/bar" - // * Calling Cid() will return `cidB` - // * Calling Root() will return `cidRoot` - // * Calling Remainder() will return `foo/bar` - Cid() cid.Cid - - // Root returns the CID of the root object of the path - // - // Example: - // If you have 3 linked objects: QmRoot -> A -> B, and resolve path - // "/ipfs/QmRoot/A/B", the Root method will return the CID of object QmRoot - // - // For more examples see the documentation of Cid() method - Root() cid.Cid - - // Remainder returns unresolved part of the path - // - // Example: - // If you have 2 linked objects: QmRoot -> A, where A is a CBOR node - // containing the following data: - // - // {"foo": {"bar": 42 }} - // - // When resolving "/ipld/QmRoot/A/foo/bar", Remainder will return "foo/bar" - // - // For more examples see the documentation of Cid() method - Remainder() string - - Path -} - -// path implements coreiface.Path -type path struct { - path string -} - -// resolvedPath implements coreiface.resolvedPath -type resolvedPath struct { - path - cid cid.Cid - root cid.Cid - remainder string -} - -// Join appends provided segments to the base path -func Join(base Path, a ...string) Path { - s := strings.Join(append([]string{base.String()}, a...), "/") - return &path{path: s} -} - -// IpfsPath creates new /ipfs path from the provided CID -func IpfsPath(c cid.Cid) Resolved { - return &resolvedPath{ - path: path{"/ipfs/" + c.String()}, - cid: c, - root: c, - remainder: "", - } -} - -// IpldPath creates new /ipld path from the provided CID -func IpldPath(c cid.Cid) Resolved { - return &resolvedPath{ - path: path{"/ipld/" + c.String()}, - cid: c, - root: c, - remainder: "", - } -} - -// New parses string path to a Path -func New(p string) Path { - if pp, err := ipfspath.ParsePath(p); err == nil { - p = pp.String() - } - - return &path{path: p} -} - -// NewResolvedPath creates new Resolved path. This function performs no checks -// and is intended to be used by resolver implementations. Incorrect inputs may -// cause panics. Handle with care. -func NewResolvedPath(ipath ipfspath.Path, c cid.Cid, root cid.Cid, remainder string) Resolved { - return &resolvedPath{ - path: path{ipath.String()}, - cid: c, - root: root, - remainder: remainder, - } -} - -func (p *path) String() string { - return p.path -} - -func (p *path) Namespace() string { - ip, err := ipfspath.ParsePath(p.path) - if err != nil { - return "" - } - - if len(ip.Segments()) < 1 { - panic("path without namespace") // this shouldn't happen under any scenario - } - return ip.Segments()[0] -} - -func (p *path) Mutable() bool { - // TODO: MFS: check for /local - return p.Namespace() == "ipns" -} - -func (p *path) IsValid() error { - _, err := ipfspath.ParsePath(p.path) - return err -} - -func (p *resolvedPath) Cid() cid.Cid { - return p.cid -} - -func (p *resolvedPath) Root() cid.Cid { - return p.root -} - -func (p *resolvedPath) Remainder() string { - return p.remainder -} diff --git a/coreiface/pin.go b/coreiface/pin.go index 6b97c6ca5..057516d08 100644 --- a/coreiface/pin.go +++ b/coreiface/pin.go @@ -3,7 +3,7 @@ package iface import ( "context" - path "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/coreiface/options" ) @@ -11,7 +11,7 @@ import ( // Pin holds information about pinned resource type Pin interface { // Path to the pinned object - Path() path.Resolved + Path() path.ImmutablePath // Type of the pin Type() string @@ -35,7 +35,7 @@ type PinStatus interface { // BadPinNode is a node that has been marked as bad by Pin.Verify type BadPinNode interface { // Path is the path of the node - Path() path.Resolved + Path() path.ImmutablePath // Err is the reason why the node has been marked as bad Err() error diff --git a/coreiface/tests/block.go b/coreiface/tests/block.go index 5dcb16e4f..6e254063e 100644 --- a/coreiface/tests/block.go +++ b/coreiface/tests/block.go @@ -9,9 +9,8 @@ import ( coreiface "github.com/ipfs/boxo/coreiface" opt "github.com/ipfs/boxo/coreiface/options" - "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/path" ipld "github.com/ipfs/go-ipld-format" - mh "github.com/multiformats/go-multihash" ) @@ -68,8 +67,8 @@ func (tp *TestSuite) TestBlockPut(t *testing.T) { t.Fatal(err) } - if res.Path().Cid().String() != rawCid { - t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + if res.Path().RootCid().String() != rawCid { + t.Errorf("got wrong cid: %s", res.Path().RootCid().String()) } } @@ -88,8 +87,8 @@ func (tp *TestSuite) TestBlockPutFormatDagCbor(t *testing.T) { t.Fatal(err) } - if res.Path().Cid().String() != cborCid { - t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + if res.Path().RootCid().String() != cborCid { + t.Errorf("got wrong cid: %s", res.Path().RootCid().String()) } } @@ -108,8 +107,8 @@ func (tp *TestSuite) TestBlockPutFormatDagPb(t *testing.T) { t.Fatal(err) } - if res.Path().Cid().String() != pbCid { - t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + if res.Path().RootCid().String() != pbCid { + t.Errorf("got wrong cid: %s", res.Path().RootCid().String()) } } @@ -128,8 +127,8 @@ func (tp *TestSuite) TestBlockPutFormatV0(t *testing.T) { t.Fatal(err) } - if res.Path().Cid().String() != pbCidV0 { - t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + if res.Path().RootCid().String() != pbCidV0 { + t.Errorf("got wrong cid: %s", res.Path().RootCid().String()) } } @@ -146,8 +145,8 @@ func (tp *TestSuite) TestBlockPutCidCodecDagCbor(t *testing.T) { t.Fatal(err) } - if res.Path().Cid().String() != cborCid { - t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + if res.Path().RootCid().String() != cborCid { + t.Errorf("got wrong cid: %s", res.Path().RootCid().String()) } } @@ -164,8 +163,8 @@ func (tp *TestSuite) TestBlockPutCidCodecDagPb(t *testing.T) { t.Fatal(err) } - if res.Path().Cid().String() != pbCid { - t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + if res.Path().RootCid().String() != pbCid { + t.Errorf("got wrong cid: %s", res.Path().RootCid().String()) } } @@ -187,8 +186,8 @@ func (tp *TestSuite) TestBlockPutHash(t *testing.T) { t.Fatal(err) } - if res.Path().Cid().String() != cborKCid { - t.Errorf("got wrong cid: %s", res.Path().Cid().String()) + if res.Path().RootCid().String() != cborKCid { + t.Errorf("got wrong cid: %s", res.Path().RootCid().String()) } } @@ -219,13 +218,13 @@ func (tp *TestSuite) TestBlockGet(t *testing.T) { t.Error("didn't get correct data back") } - p := path.New("/ipfs/" + res.Path().Cid().String()) + p := path.FromCid(res.Path().RootCid()) - rp, err := api.ResolvePath(ctx, p) + rp, _, err := api.ResolvePath(ctx, p) if err != nil { t.Fatal(err) } - if rp.Cid().String() != res.Path().Cid().String() { + if rp.RootCid().String() != res.Path().RootCid().String() { t.Error("paths didn't match") } } diff --git a/coreiface/tests/dag.go b/coreiface/tests/dag.go index b9a03c8f4..a106788d6 100644 --- a/coreiface/tests/dag.go +++ b/coreiface/tests/dag.go @@ -3,13 +3,11 @@ package tests import ( "context" "math" - gopath "path" "strings" "testing" - path "github.com/ipfs/boxo/coreiface/path" - coreiface "github.com/ipfs/boxo/coreiface" + "github.com/ipfs/boxo/path" ipldcbor "github.com/ipfs/go-ipld-cbor" ipld "github.com/ipfs/go-ipld-format" @@ -113,14 +111,17 @@ func (tp *TestSuite) TestDagPath(t *testing.T) { t.Fatal(err) } - p := path.New(gopath.Join(nd.Cid().String(), "lnk")) + p, err := path.Join(path.FromCid(nd.Cid()), "lnk") + if err != nil { + t.Fatal(err) + } - rp, err := api.ResolvePath(ctx, p) + rp, _, err := api.ResolvePath(ctx, p) if err != nil { t.Fatal(err) } - ndd, err := api.Dag().Get(ctx, rp.Cid()) + ndd, err := api.Dag().Get(ctx, rp.RootCid()) if err != nil { t.Fatal(err) } diff --git a/coreiface/tests/name.go b/coreiface/tests/name.go index 74d88edff..2b6b7ec49 100644 --- a/coreiface/tests/name.go +++ b/coreiface/tests/name.go @@ -4,15 +4,14 @@ import ( "context" "io" "math/rand" - gopath "path" "testing" "time" coreiface "github.com/ipfs/boxo/coreiface" opt "github.com/ipfs/boxo/coreiface/options" - path "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/ipns" + "github.com/ipfs/boxo/path" "github.com/stretchr/testify/require" ) @@ -35,10 +34,6 @@ func addTestObject(ctx context.Context, api coreiface.CoreAPI) (path.Path, error return api.Unixfs().Add(ctx, files.NewReaderFile(&io.LimitedReader{R: rnd, N: 4092})) } -func appendPath(p path.Path, sub string) path.Path { - return path.New(gopath.Join(p.String(), sub)) -} - func (tp *TestSuite) TestPublishResolve(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -68,7 +63,10 @@ func (tp *TestSuite) TestPublishResolve(t *testing.T) { t.Run("publishPath", func(t *testing.T) { api, p := init() - name, err := api.Name().Publish(ctx, appendPath(p, "/test")) + p, err := path.Join(p, "/test") + require.NoError(t, err) + + name, err := api.Name().Publish(ctx, p) require.NoError(t, err) self, err := api.Key().Self(ctx) @@ -77,7 +75,7 @@ func (tp *TestSuite) TestPublishResolve(t *testing.T) { resPath, err := api.Name().Resolve(ctx, name.String(), ropts...) require.NoError(t, err) - require.Equal(t, p.String()+"/test", resPath.String()) + require.Equal(t, p.String(), resPath.String()) }) t.Run("revolvePath", func(t *testing.T) { @@ -96,7 +94,10 @@ func (tp *TestSuite) TestPublishResolve(t *testing.T) { t.Run("publishRevolvePath", func(t *testing.T) { api, p := init() - name, err := api.Name().Publish(ctx, appendPath(p, "/a")) + p, err := path.Join(p, "/a") + require.NoError(t, err) + + name, err := api.Name().Publish(ctx, p) require.NoError(t, err) self, err := api.Key().Self(ctx) @@ -105,7 +106,7 @@ func (tp *TestSuite) TestPublishResolve(t *testing.T) { resPath, err := api.Name().Resolve(ctx, name.String()+"/b", ropts...) require.NoError(t, err) - require.Equal(t, p.String()+"/a/b", resPath.String()) + require.Equal(t, p.String()+"/b", resPath.String()) }) } diff --git a/coreiface/tests/object.go b/coreiface/tests/object.go index 63c218eb3..5c6ba828c 100644 --- a/coreiface/tests/object.go +++ b/coreiface/tests/object.go @@ -166,7 +166,7 @@ func (tp *TestSuite) TestObjectLinks(t *testing.T) { t.Fatal(err) } - p2, err := api.Object().Put(ctx, strings.NewReader(`{"Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`"}]}`)) + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Links":[{"Name":"bar", "Hash":"`+p1.RootCid().String()+`"}]}`)) if err != nil { t.Fatal(err) } @@ -180,7 +180,7 @@ func (tp *TestSuite) TestObjectLinks(t *testing.T) { t.Errorf("unexpected number of links: %d", len(links)) } - if links[0].Cid.String() != p1.Cid().String() { + if links[0].Cid.String() != p1.RootCid().String() { t.Fatal("cids didn't batch") } @@ -202,7 +202,7 @@ func (tp *TestSuite) TestObjectStat(t *testing.T) { t.Fatal(err) } - p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.RootCid().String()+`", "Size":3}]}`)) if err != nil { t.Fatal(err) } @@ -212,7 +212,7 @@ func (tp *TestSuite) TestObjectStat(t *testing.T) { t.Fatal(err) } - if stat.Cid.String() != p2.Cid().String() { + if stat.Cid.String() != p2.RootCid().String() { t.Error("unexpected stat.Cid") } @@ -250,7 +250,7 @@ func (tp *TestSuite) TestObjectAddLink(t *testing.T) { t.Fatal(err) } - p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.RootCid().String()+`", "Size":3}]}`)) if err != nil { t.Fatal(err) } @@ -291,7 +291,7 @@ func (tp *TestSuite) TestObjectAddLinkCreate(t *testing.T) { t.Fatal(err) } - p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.RootCid().String()+`", "Size":3}]}`)) if err != nil { t.Fatal(err) } @@ -340,7 +340,7 @@ func (tp *TestSuite) TestObjectRmLink(t *testing.T) { t.Fatal(err) } - p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.Cid().String()+`", "Size":3}]}`)) + p2, err := api.Object().Put(ctx, strings.NewReader(`{"Data":"bazz", "Links":[{"Name":"bar", "Hash":"`+p1.RootCid().String()+`", "Size":3}]}`)) if err != nil { t.Fatal(err) } diff --git a/coreiface/tests/path.go b/coreiface/tests/path.go index 06f3aa1f8..116aed2e7 100644 --- a/coreiface/tests/path.go +++ b/coreiface/tests/path.go @@ -2,17 +2,26 @@ package tests import ( "context" + "fmt" "math" "strings" "testing" - "github.com/ipfs/boxo/coreiface/path" - "github.com/ipfs/boxo/coreiface/options" - + "github.com/ipfs/boxo/path" + "github.com/ipfs/go-cid" ipldcbor "github.com/ipfs/go-ipld-cbor" + "github.com/stretchr/testify/require" ) +func newIPLDPath(t *testing.T, cid cid.Cid) path.ImmutablePath { + p, err := path.NewPath(fmt.Sprintf("/%s/%s", path.IPLDNamespace, cid.String())) + require.NoError(t, err) + im, err := path.NewImmutablePath(p) + require.NoError(t, err) + return im +} + func (tp *TestSuite) TestPath(t *testing.T) { t.Run("TestMutablePath", tp.TestMutablePath) t.Run("TestPathRemainder", tp.TestPathRemainder) @@ -25,173 +34,115 @@ func (tp *TestSuite) TestPath(t *testing.T) { func (tp *TestSuite) TestMutablePath(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + api, err := tp.makeAPI(t, ctx) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) blk, err := api.Block().Put(ctx, strings.NewReader(`foo`)) - if err != nil { - t.Fatal(err) - } - - if blk.Path().Mutable() { - t.Error("expected /ipld path to be immutable") - } - - // get self /ipns path - - if api.Key() == nil { - t.Fatal(".Key not implemented") - } + require.NoError(t, err) + require.False(t, blk.Path().Mutable()) + require.NotNil(t, api.Key()) keys, err := api.Key().List(ctx) - if err != nil { - t.Fatal(err) - } - - if !keys[0].Path().Mutable() { - t.Error("expected self /ipns path to be mutable") - } + require.NoError(t, err) + require.True(t, keys[0].Path().Mutable()) } func (tp *TestSuite) TestPathRemainder(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - api, err := tp.makeAPI(t, ctx) - if err != nil { - t.Fatal(err) - } - if api.Dag() == nil { - t.Fatal(".Dag not implemented") - } + api, err := tp.makeAPI(t, ctx) + require.NoError(t, err) + require.NotNil(t, api.Dag()) nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"bar": "baz"}}`), math.MaxUint64, -1) - if err != nil { - t.Fatal(err) - } - - if err := api.Dag().Add(ctx, nd); err != nil { - t.Fatal(err) - } - - rp1, err := api.ResolvePath(ctx, path.New(nd.String()+"/foo/bar")) - if err != nil { - t.Fatal(err) - } - - if rp1.Remainder() != "foo/bar" { - t.Error("expected to get path remainder") - } + require.NoError(t, err) + + err = api.Dag().Add(ctx, nd) + require.NoError(t, err) + + p, err := path.Join(path.FromCid(nd.Cid()), "foo", "bar") + require.NoError(t, err) + + _, remainder, err := api.ResolvePath(ctx, p) + require.NoError(t, err) + require.Equal(t, "/foo/bar", path.SegmentsToString(remainder...)) } func (tp *TestSuite) TestEmptyPathRemainder(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - api, err := tp.makeAPI(t, ctx) - if err != nil { - t.Fatal(err) - } - if api.Dag() == nil { - t.Fatal(".Dag not implemented") - } + api, err := tp.makeAPI(t, ctx) + require.NoError(t, err) + require.NotNil(t, api.Dag()) nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"bar": "baz"}}`), math.MaxUint64, -1) - if err != nil { - t.Fatal(err) - } - - if err := api.Dag().Add(ctx, nd); err != nil { - t.Fatal(err) - } - - rp1, err := api.ResolvePath(ctx, path.New(nd.Cid().String())) - if err != nil { - t.Fatal(err) - } - - if rp1.Remainder() != "" { - t.Error("expected the resolved path to not have a remainder") - } + require.NoError(t, err) + + err = api.Dag().Add(ctx, nd) + require.NoError(t, err) + + _, remainder, err := api.ResolvePath(ctx, path.FromCid(nd.Cid())) + require.NoError(t, err) + require.Empty(t, remainder) } func (tp *TestSuite) TestInvalidPathRemainder(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - api, err := tp.makeAPI(t, ctx) - if err != nil { - t.Fatal(err) - } - if api.Dag() == nil { - t.Fatal(".Dag not implemented") - } + api, err := tp.makeAPI(t, ctx) + require.NoError(t, err) + require.NotNil(t, api.Dag()) nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"bar": "baz"}}`), math.MaxUint64, -1) - if err != nil { - t.Fatal(err) - } - - if err := api.Dag().Add(ctx, nd); err != nil { - t.Fatal(err) - } - - _, err = api.ResolvePath(ctx, path.New("/ipld/"+nd.Cid().String()+"/bar/baz")) - if err == nil || !strings.Contains(err.Error(), `no link named "bar"`) { - t.Fatalf("unexpected error: %s", err) - } + require.NoError(t, err) + + err = api.Dag().Add(ctx, nd) + require.NoError(t, err) + + p, err := path.Join(newIPLDPath(t, nd.Cid()), "/bar/baz") + require.NoError(t, err) + + _, _, err = api.ResolvePath(ctx, p) + require.NotNil(t, err) + require.ErrorContains(t, err, `no link named "bar"`) } func (tp *TestSuite) TestPathRoot(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - api, err := tp.makeAPI(t, ctx) - if err != nil { - t.Fatal(err) - } - if api.Block() == nil { - t.Fatal(".Block not implemented") - } + api, err := tp.makeAPI(t, ctx) + require.NoError(t, err) + require.NotNil(t, api.Block()) blk, err := api.Block().Put(ctx, strings.NewReader(`foo`), options.Block.Format("raw")) - if err != nil { - t.Fatal(err) - } - - if api.Dag() == nil { - t.Fatal(".Dag not implemented") - } - - nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"/": "`+blk.Path().Cid().String()+`"}}`), math.MaxUint64, -1) - if err != nil { - t.Fatal(err) - } - - if err := api.Dag().Add(ctx, nd); err != nil { - t.Fatal(err) - } - - rp, err := api.ResolvePath(ctx, path.New("/ipld/"+nd.Cid().String()+"/foo")) - if err != nil { - t.Fatal(err) - } - - if rp.Root().String() != nd.Cid().String() { - t.Error("unexpected path root") - } - - if rp.Cid().String() != blk.Path().Cid().String() { - t.Error("unexpected path cid") - } + require.NoError(t, err) + require.NotNil(t, api.Dag()) + + nd, err := ipldcbor.FromJSON(strings.NewReader(`{"foo": {"/": "`+blk.Path().RootCid().String()+`"}}`), math.MaxUint64, -1) + require.NoError(t, err) + + err = api.Dag().Add(ctx, nd) + require.NoError(t, err) + + p, err := path.Join(newIPLDPath(t, nd.Cid()), "/foo") + require.NoError(t, err) + + rp, _, err := api.ResolvePath(ctx, p) + require.NoError(t, err) + require.Equal(t, rp.RootCid().String(), blk.Path().RootCid().String()) } func (tp *TestSuite) TestPathJoin(t *testing.T) { - p1 := path.New("/ipfs/QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6/bar/baz") + p1, err := path.NewPath("/ipfs/QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6/bar/baz") + require.NoError(t, err) + + p2, err := path.Join(p1, "foo") + require.NoError(t, err) - if path.Join(p1, "foo").String() != "/ipfs/QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6/bar/baz/foo" { - t.Error("unexpected path") - } + require.Equal(t, "/ipfs/QmYNmQKp6SuaVrpgWRsPTgCQCnpxUYGq76YEKBXuj2N4H6/bar/baz/foo", p2.String()) } diff --git a/coreiface/tests/pin.go b/coreiface/tests/pin.go index 4b0fea01d..49499b36a 100644 --- a/coreiface/tests/pin.go +++ b/coreiface/tests/pin.go @@ -8,8 +8,7 @@ import ( iface "github.com/ipfs/boxo/coreiface" opt "github.com/ipfs/boxo/coreiface/options" - "github.com/ipfs/boxo/coreiface/path" - + "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" ipldcbor "github.com/ipfs/go-ipld-cbor" ipld "github.com/ipfs/go-ipld-format" @@ -77,7 +76,7 @@ func (tp *TestSuite) TestPinSimple(t *testing.T) { t.Errorf("unexpected pin list len: %d", len(list)) } - if list[0].Path().Cid().String() != p.Cid().String() { + if list[0].Path().RootCid().String() != p.RootCid().String() { t.Error("paths don't match") } @@ -120,12 +119,12 @@ func (tp *TestSuite) TestPinRecursive(t *testing.T) { t.Fatal(err) } - nd2, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+p0.Cid().String()+`"}}`), math.MaxUint64, -1) + nd2, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+p0.RootCid().String()+`"}}`), math.MaxUint64, -1) if err != nil { t.Fatal(err) } - nd3, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+p1.Cid().String()+`"}}`), math.MaxUint64, -1) + nd3, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+p1.RootCid().String()+`"}}`), math.MaxUint64, -1) if err != nil { t.Fatal(err) } @@ -134,12 +133,12 @@ func (tp *TestSuite) TestPinRecursive(t *testing.T) { t.Fatal(err) } - err = api.Pin().Add(ctx, path.IpldPath(nd2.Cid())) + err = api.Pin().Add(ctx, path.FromCid(nd2.Cid())) if err != nil { t.Fatal(err) } - err = api.Pin().Add(ctx, path.IpldPath(nd3.Cid()), opt.Pin.Recursive(false)) + err = api.Pin().Add(ctx, path.FromCid(nd3.Cid()), opt.Pin.Recursive(false)) if err != nil { t.Fatal(err) } @@ -162,8 +161,8 @@ func (tp *TestSuite) TestPinRecursive(t *testing.T) { t.Errorf("unexpected pin list len: %d", len(list)) } - if list[0].Path().String() != path.IpldPath(nd3.Cid()).String() { - t.Errorf("unexpected path, %s != %s", list[0].Path().String(), path.IpfsPath(nd3.Cid()).String()) + if list[0].Path().String() != path.FromCid(nd3.Cid()).String() { + t.Errorf("unexpected path, %s != %s", list[0].Path().String(), path.FromCid(nd3.Cid()).String()) } list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Recursive())) @@ -175,8 +174,8 @@ func (tp *TestSuite) TestPinRecursive(t *testing.T) { t.Errorf("unexpected pin list len: %d", len(list)) } - if list[0].Path().String() != path.IpldPath(nd2.Cid()).String() { - t.Errorf("unexpected path, %s != %s", list[0].Path().String(), path.IpldPath(nd2.Cid()).String()) + if list[0].Path().String() != path.FromCid(nd2.Cid()).String() { + t.Errorf("unexpected path, %s != %s", list[0].Path().String(), path.FromCid(nd2.Cid()).String()) } list, err = accPins(api.Pin().Ls(ctx, opt.Pin.Ls.Indirect())) @@ -188,8 +187,8 @@ func (tp *TestSuite) TestPinRecursive(t *testing.T) { t.Errorf("unexpected pin list len: %d", len(list)) } - if list[0].Path().Cid().String() != p0.Cid().String() { - t.Errorf("unexpected path, %s != %s", list[0].Path().Cid().String(), p0.Cid().String()) + if list[0].Path().RootCid().String() != p0.RootCid().String() { + t.Errorf("unexpected path, %s != %s", list[0].Path().RootCid().String(), p0.RootCid().String()) } res, err := api.Pin().Verify(ctx) @@ -259,12 +258,12 @@ func (tp *TestSuite) TestPinLsIndirect(t *testing.T) { leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "foo") - err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid())) + err = api.Pin().Add(ctx, path.FromCid(grandparent.Cid())) if err != nil { t.Fatal(err) } - err = api.Pin().Add(ctx, path.IpldPath(parent.Cid()), opt.Pin.Recursive(false)) + err = api.Pin().Add(ctx, path.FromCid(parent.Cid()), opt.Pin.Recursive(false)) if err != nil { t.Fatal(err) } @@ -293,12 +292,12 @@ func (tp *TestSuite) TestPinLsPredenceRecursiveIndirect(t *testing.T) { // Test recursive > indirect leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "recursive > indirect") - err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid())) + err = api.Pin().Add(ctx, path.FromCid(grandparent.Cid())) if err != nil { t.Fatal(err) } - err = api.Pin().Add(ctx, path.IpldPath(parent.Cid())) + err = api.Pin().Add(ctx, path.FromCid(parent.Cid())) if err != nil { t.Fatal(err) } @@ -317,12 +316,12 @@ func (tp *TestSuite) TestPinLsPrecedenceDirectIndirect(t *testing.T) { // Test direct > indirect leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "direct > indirect") - err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid())) + err = api.Pin().Add(ctx, path.FromCid(grandparent.Cid())) if err != nil { t.Fatal(err) } - err = api.Pin().Add(ctx, path.IpldPath(parent.Cid()), opt.Pin.Recursive(false)) + err = api.Pin().Add(ctx, path.FromCid(parent.Cid()), opt.Pin.Recursive(false)) if err != nil { t.Fatal(err) } @@ -341,24 +340,24 @@ func (tp *TestSuite) TestPinLsPrecedenceRecursiveDirect(t *testing.T) { // Test recursive > direct leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "recursive + direct = error") - err = api.Pin().Add(ctx, path.IpldPath(parent.Cid())) + err = api.Pin().Add(ctx, path.FromCid(parent.Cid())) if err != nil { t.Fatal(err) } - err = api.Pin().Add(ctx, path.IpldPath(parent.Cid()), opt.Pin.Recursive(false)) + err = api.Pin().Add(ctx, path.FromCid(parent.Cid()), opt.Pin.Recursive(false)) if err == nil { t.Fatal("expected error directly pinning a recursively pinned node") } assertPinTypes(t, ctx, api, []cidContainer{parent}, []cidContainer{}, []cidContainer{leaf}) - err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid()), opt.Pin.Recursive(false)) + err = api.Pin().Add(ctx, path.FromCid(grandparent.Cid()), opt.Pin.Recursive(false)) if err != nil { t.Fatal(err) } - err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid())) + err = api.Pin().Add(ctx, path.FromCid(grandparent.Cid())) if err != nil { t.Fatal(err) } @@ -376,40 +375,48 @@ func (tp *TestSuite) TestPinIsPinned(t *testing.T) { leaf, parent, grandparent := getThreeChainedNodes(t, ctx, api, "foofoo") - assertNotPinned(t, ctx, api, path.IpldPath(grandparent.Cid())) - assertNotPinned(t, ctx, api, path.IpldPath(parent.Cid())) - assertNotPinned(t, ctx, api, path.IpldPath(leaf.Cid())) + assertNotPinned(t, ctx, api, newIPLDPath(t, grandparent.Cid())) + assertNotPinned(t, ctx, api, newIPLDPath(t, parent.Cid())) + assertNotPinned(t, ctx, api, newIPLDPath(t, leaf.Cid())) - err = api.Pin().Add(ctx, path.IpldPath(parent.Cid()), opt.Pin.Recursive(true)) + err = api.Pin().Add(ctx, newIPLDPath(t, parent.Cid()), opt.Pin.Recursive(true)) if err != nil { t.Fatal(err) } - assertNotPinned(t, ctx, api, path.IpldPath(grandparent.Cid())) - assertIsPinned(t, ctx, api, path.IpldPath(parent.Cid()), "recursive") - assertIsPinned(t, ctx, api, path.IpldPath(leaf.Cid()), "indirect") + assertNotPinned(t, ctx, api, newIPLDPath(t, grandparent.Cid())) + assertIsPinned(t, ctx, api, newIPLDPath(t, parent.Cid()), "recursive") + assertIsPinned(t, ctx, api, newIPLDPath(t, leaf.Cid()), "indirect") - err = api.Pin().Add(ctx, path.IpldPath(grandparent.Cid()), opt.Pin.Recursive(false)) + err = api.Pin().Add(ctx, newIPLDPath(t, grandparent.Cid()), opt.Pin.Recursive(false)) if err != nil { t.Fatal(err) } - assertIsPinned(t, ctx, api, path.IpldPath(grandparent.Cid()), "direct") - assertIsPinned(t, ctx, api, path.IpldPath(parent.Cid()), "recursive") - assertIsPinned(t, ctx, api, path.IpldPath(leaf.Cid()), "indirect") + assertIsPinned(t, ctx, api, newIPLDPath(t, grandparent.Cid()), "direct") + assertIsPinned(t, ctx, api, newIPLDPath(t, parent.Cid()), "recursive") + assertIsPinned(t, ctx, api, newIPLDPath(t, leaf.Cid()), "indirect") } type cidContainer interface { Cid() cid.Cid } +type immutablePathCidContainer struct { + path.ImmutablePath +} + +func (i immutablePathCidContainer) Cid() cid.Cid { + return i.RootCid() +} + func getThreeChainedNodes(t *testing.T, ctx context.Context, api iface.CoreAPI, leafData string) (cidContainer, cidContainer, cidContainer) { leaf, err := api.Unixfs().Add(ctx, strFile(leafData)()) if err != nil { t.Fatal(err) } - parent, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+leaf.Cid().String()+`"}}`), math.MaxUint64, -1) + parent, err := ipldcbor.FromJSON(strings.NewReader(`{"lnk": {"/": "`+leaf.RootCid().String()+`"}}`), math.MaxUint64, -1) if err != nil { t.Fatal(err) } @@ -423,7 +430,7 @@ func getThreeChainedNodes(t *testing.T, ctx context.Context, api iface.CoreAPI, t.Fatal(err) } - return leaf, parent, grandparent + return immutablePathCidContainer{leaf}, parent, grandparent } func assertPinTypes(t *testing.T, ctx context.Context, api iface.CoreAPI, recusive, direct, indirect []cidContainer) { @@ -466,7 +473,7 @@ func assertPinCids(t *testing.T, pins []iface.Pin, cids ...cidContainer) { valid := true for _, p := range pins { - c := p.Path().Cid() + c := p.Path().RootCid() if cSet.Has(c) { cSet.Remove(c) } else { @@ -480,7 +487,7 @@ func assertPinCids(t *testing.T, pins []iface.Pin, cids ...cidContainer) { if !valid { pinStrs := make([]string, len(pins)) for i, p := range pins { - pinStrs[i] = p.Path().Cid().String() + pinStrs[i] = p.Path().RootCid().String() } pathStrs := make([]string, len(cids)) for i, c := range cids { @@ -511,13 +518,13 @@ func assertPinLsAllConsistency(t *testing.T, ctx context.Context, api iface.Core } for _, p := range allPins { - if !all.Visit(p.Path().Cid()) { + if !all.Visit(p.Path().RootCid()) { t.Fatalf("pin ls returned the same cid multiple times") } typeStr := p.Type() if typeSet, ok := typeMap[p.Type()]; ok { - typeSet.Add(p.Path().Cid()) + typeSet.Add(p.Path().RootCid()) } else { t.Fatalf("unknown pin type: %s", typeStr) } @@ -538,7 +545,7 @@ func assertPinLsAllConsistency(t *testing.T, ctx context.Context, api iface.Core t.Fatalf("returned wrong pin type: expected %s, got %s", typeStr, pinType) } - if c := p.Path().Cid(); !pinProps.Has(c) { + if c := p.Path().RootCid(); !pinProps.Has(c) { t.Fatalf("%s expected to be in pin ls all as type %s", c.String(), typeStr) } } diff --git a/coreiface/tests/routing.go b/coreiface/tests/routing.go index fd10dffcd..c56e91659 100644 --- a/coreiface/tests/routing.go +++ b/coreiface/tests/routing.go @@ -7,8 +7,8 @@ import ( iface "github.com/ipfs/boxo/coreiface" "github.com/ipfs/boxo/coreiface/options" - "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/ipns" + "github.com/ipfs/boxo/path" "github.com/stretchr/testify/require" ) diff --git a/coreiface/tests/unixfs.go b/coreiface/tests/unixfs.go index 25c3ac1b7..e0c37fce4 100644 --- a/coreiface/tests/unixfs.go +++ b/coreiface/tests/unixfs.go @@ -14,10 +14,9 @@ import ( "sync" "testing" - "github.com/ipfs/boxo/coreiface/path" - coreiface "github.com/ipfs/boxo/coreiface" "github.com/ipfs/boxo/coreiface/options" + "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/files" mdag "github.com/ipfs/boxo/ipld/merkledag" @@ -106,12 +105,12 @@ func (tp *TestSuite) TestAdd(t *testing.T) { t.Fatal(err) } - p := func(h string) path.Resolved { + p := func(h string) path.ImmutablePath { c, err := cid.Parse(h) if err != nil { t.Fatal(err) } - return path.IpfsPath(c) + return path.FromCid(c) } rf, err := os.CreateTemp(os.TempDir(), "unixfs-add-real") @@ -410,7 +409,7 @@ func (tp *TestSuite) TestAdd(t *testing.T) { } if expected[0].Path != nil && event.Path != nil { - if expected[0].Path.Cid().String() != event.Path.Cid().String() { + if expected[0].Path.RootCid().String() != event.Path.RootCid().String() { t.Errorf("Event.Hash didn't match, %s != %s", expected[0].Path, event.Path) } } else if event.Path != expected[0].Path { @@ -553,7 +552,7 @@ func (tp *TestSuite) TestAddPinned(t *testing.T) { t.Fatalf("expected 1 pin, got %d", len(pins)) } - if pins[0].Path().String() != "/ipld/QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk" { + if pins[0].Path().String() != "/ipfs/QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk" { t.Fatalf("got unexpected pin: %s", pins[0].Path().String()) } } @@ -597,7 +596,10 @@ func (tp *TestSuite) TestGetEmptyFile(t *testing.T) { t.Fatal(err) } - emptyFilePath := path.New(emptyFile) + emptyFilePath, err := path.NewPath(emptyFile) + if err != nil { + t.Fatal(err) + } r, err := api.Unixfs().Get(ctx, emptyFilePath) if err != nil { @@ -626,18 +628,18 @@ func (tp *TestSuite) TestGetDir(t *testing.T) { if err != nil { t.Fatal(err) } - p := path.IpfsPath(edir.Cid()) + p := path.FromCid(edir.Cid()) emptyDir, err := api.Object().New(ctx, options.Object.Type("unixfs-dir")) if err != nil { t.Fatal(err) } - if p.String() != path.IpfsPath(emptyDir.Cid()).String() { + if p.String() != path.FromCid(emptyDir.Cid()).String() { t.Fatalf("expected path %s, got: %s", emptyDir.Cid(), p.String()) } - r, err := api.Unixfs().Get(ctx, path.IpfsPath(emptyDir.Cid())) + r, err := api.Unixfs().Get(ctx, path.FromCid(emptyDir.Cid())) if err != nil { t.Fatal(err) } @@ -661,7 +663,7 @@ func (tp *TestSuite) TestGetNonUnixfs(t *testing.T) { t.Fatal(err) } - _, err = api.Unixfs().Get(ctx, path.IpfsPath(nd.Cid())) + _, err = api.Unixfs().Get(ctx, path.FromCid(nd.Cid())) if !strings.Contains(err.Error(), "proto: required field") { t.Fatalf("expected protobuf error, got: %s", err) } @@ -787,7 +789,7 @@ func (tp *TestSuite) TestLsEmptyDir(t *testing.T) { t.Fatal(err) } - links, err := api.Unixfs().Ls(ctx, path.IpfsPath(emptyDir.Cid())) + links, err := api.Unixfs().Ls(ctx, path.FromCid(emptyDir.Cid())) if err != nil { t.Fatal(err) } @@ -816,7 +818,7 @@ func (tp *TestSuite) TestLsNonUnixfs(t *testing.T) { t.Fatal(err) } - links, err := api.Unixfs().Ls(ctx, path.IpfsPath(nd.Cid())) + links, err := api.Unixfs().Ls(ctx, path.FromCid(nd.Cid())) if err != nil { t.Fatal(err) } diff --git a/coreiface/unixfs.go b/coreiface/unixfs.go index 606bc8e78..35e108c02 100644 --- a/coreiface/unixfs.go +++ b/coreiface/unixfs.go @@ -4,17 +4,16 @@ import ( "context" "github.com/ipfs/boxo/coreiface/options" - path "github.com/ipfs/boxo/coreiface/path" - "github.com/ipfs/boxo/files" + "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" ) type AddEvent struct { Name string - Path path.Resolved `json:",omitempty"` - Bytes int64 `json:",omitempty"` - Size string `json:",omitempty"` + Path path.ImmutablePath `json:",omitempty"` + Bytes int64 `json:",omitempty"` + Size string `json:",omitempty"` } // FileType is an enum of possible UnixFS file types. @@ -66,7 +65,7 @@ type UnixfsAPI interface { // Add imports the data from the reader into merkledag file // // TODO: a long useful comment on how to use this for many different scenarios - Add(context.Context, files.Node, ...options.UnixfsAddOption) (path.Resolved, error) + Add(context.Context, files.Node, ...options.UnixfsAddOption) (path.ImmutablePath, error) // Get returns a read-only handle to a file tree referenced by a path // diff --git a/examples/routing/delegated-routing-client/main_test.go b/examples/routing/delegated-routing-client/main_test.go index 7ad813f13..2dab7b13a 100644 --- a/examples/routing/delegated-routing-client/main_test.go +++ b/examples/routing/delegated-routing-client/main_test.go @@ -9,9 +9,8 @@ import ( "testing" "time" - "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/ipns" - ipfspath "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" @@ -82,11 +81,11 @@ func makeNameAndRecord(t *testing.T) (ipns.Name, []byte) { cid, err := cid.Decode("bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") require.NoError(t, err) - path := path.IpfsPath(cid) + path := path.FromCid(cid) eol := time.Now().Add(time.Hour * 48) ttl := time.Second * 20 - record, err := ipns.NewRecord(sk, ipfspath.FromString(path.String()), 1, eol, ttl) + record, err := ipns.NewRecord(sk, path, 1, eol, ttl) require.NoError(t, err) rawRecord, err := ipns.MarshalRecord(record) diff --git a/gateway/assets/assets.go b/gateway/assets/assets.go index 3c0265f0c..4a629c366 100644 --- a/gateway/assets/assets.go +++ b/gateway/assets/assets.go @@ -10,8 +10,7 @@ import ( "strings" "github.com/cespare/xxhash/v2" - - ipfspath "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" ) //go:embed *.html *.css @@ -132,7 +131,7 @@ type Breadcrumb struct { func Breadcrumbs(urlPath string, dnslinkOrigin bool) []Breadcrumb { var ret []Breadcrumb - p, err := ipfspath.ParsePath(urlPath) + p, err := path.NewPath(urlPath) if err != nil { // No assets.Breadcrumbs, fallback to bare Path in template return ret diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index fc0685d62..6868d3247 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -7,13 +7,11 @@ import ( "fmt" "io" "net/http" - gopath "path" "strings" "github.com/ipfs/boxo/blockservice" blockstore "github.com/ipfs/boxo/blockstore" nsopts "github.com/ipfs/boxo/coreiface/options/namesys" - ifacepath "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/fetcher" bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/files" @@ -22,7 +20,7 @@ import ( uio "github.com/ipfs/boxo/ipld/unixfs/io" "github.com/ipfs/boxo/namesys" "github.com/ipfs/boxo/namesys/resolve" - ipfspath "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/path/resolver" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" @@ -147,7 +145,7 @@ func NewBlocksBackend(blockService blockservice.BlockService, opts ...BlocksBack }, nil } -func (bb *BlocksBackend) Get(ctx context.Context, path ImmutablePath, ranges ...ByteRange) (ContentPathMetadata, *GetResponse, error) { +func (bb *BlocksBackend) Get(ctx context.Context, path path.ImmutablePath, ranges ...ByteRange) (ContentPathMetadata, *GetResponse, error) { md, nd, err := bb.getNode(ctx, path) if err != nil { return md, nil, err @@ -224,7 +222,7 @@ func (bb *BlocksBackend) Get(ctx context.Context, path ImmutablePath, ranges ... return ContentPathMetadata{}, nil, fmt.Errorf("data was not a valid file or directory: %w", ErrInternalServerError) // TODO: should there be a gateway invalid content type to abstract over the various IPLD error types? } -func (bb *BlocksBackend) GetAll(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { +func (bb *BlocksBackend) GetAll(ctx context.Context, path path.ImmutablePath) (ContentPathMetadata, files.Node, error) { md, nd, err := bb.getNode(ctx, path) if err != nil { return md, nil, err @@ -238,7 +236,7 @@ func (bb *BlocksBackend) GetAll(ctx context.Context, path ImmutablePath) (Conten return md, n, nil } -func (bb *BlocksBackend) GetBlock(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.File, error) { +func (bb *BlocksBackend) GetBlock(ctx context.Context, path path.ImmutablePath) (ContentPathMetadata, files.File, error) { md, nd, err := bb.getNode(ctx, path) if err != nil { return md, nil, err @@ -247,7 +245,7 @@ func (bb *BlocksBackend) GetBlock(ctx context.Context, path ImmutablePath) (Cont return md, files.NewBytesFile(nd.RawData()), nil } -func (bb *BlocksBackend) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { +func (bb *BlocksBackend) Head(ctx context.Context, path path.ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { md, nd, err := bb.getNode(ctx, path) if err != nil { return md, nil, err @@ -290,7 +288,7 @@ func (bb *BlocksBackend) Head(ctx context.Context, path ImmutablePath) (ContentP // https://ipld.io/specs/transport/car/carv1/#number-of-roots var emptyRoot = []cid.Cid{cid.MustParse("bafkqaaa")} -func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { +func (bb *BlocksBackend) GetCAR(ctx context.Context, p path.ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { pathMetadata, err := bb.ResolvePath(ctx, p) if err != nil { rootCid, err := cid.Decode(strings.Split(p.String(), "/")[2]) @@ -314,21 +312,19 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params Car // Setup the UnixFS resolver. f := newNodeGetterFetcherSingleUseFactory(ctx, blockGetter) pathResolver := resolver.NewBasicResolver(f) - ip := ipfspath.FromString(p.String()) - _, _, err = pathResolver.ResolveToLastNode(ctx, ip) + _, _, err = pathResolver.ResolveToLastNode(ctx, p) if isErrNotFound(err) { return ContentPathMetadata{ PathSegmentRoots: nil, - LastSegment: ifacepath.NewResolvedPath(ip, rootCid, rootCid, ""), + LastSegment: path.FromCid(rootCid), ContentType: "", }, io.NopCloser(&buf), nil } return ContentPathMetadata{}, nil, err } - contentPathStr := p.String() - if !strings.HasPrefix(contentPathStr, "/ipfs/") { + if p.Namespace() != path.IPFSNamespace { return ContentPathMetadata{}, nil, fmt.Errorf("path does not have /ipfs/ prefix") } @@ -336,7 +332,7 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params Car go func() { cw, err := storage.NewWritable( w, - []cid.Cid{pathMetadata.LastSegment.Cid()}, + []cid.Cid{pathMetadata.LastSegment.RootCid()}, car.WriteAsCarV1(true), car.AllowDuplicatePuts(params.Duplicates.Bool()), ) @@ -363,7 +359,7 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params Car // TODO: support selectors passed as request param: https://github.com/ipfs/kubo/issues/8769 // TODO: this is very slow if blocks are remote due to linear traversal. Do we need deterministic traversals here? - carWriteErr := walkGatewaySimpleSelector(ctx, ipfspath.Path(contentPathStr), params, &lsys, pathResolver) + carWriteErr := walkGatewaySimpleSelector(ctx, p, params, &lsys, pathResolver) // io.PipeWriter.CloseWithError always returns nil. _ = w.CloseWithError(carWriteErr) @@ -373,7 +369,7 @@ func (bb *BlocksBackend) GetCAR(ctx context.Context, p ImmutablePath, params Car } // walkGatewaySimpleSelector walks the subgraph described by the path and terminal element parameters -func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params CarParams, lsys *ipld.LinkSystem, pathResolver resolver.Resolver) error { +func walkGatewaySimpleSelector(ctx context.Context, p path.ImmutablePath, params CarParams, lsys *ipld.LinkSystem, pathResolver resolver.Resolver) error { // First resolve the path since we always need to. lastCid, remainder, err := pathResolver.ResolveToLastNode(ctx, p) if err != nil { @@ -541,18 +537,19 @@ func walkGatewaySimpleSelector(ctx context.Context, p ipfspath.Path, params CarP } } -func (bb *BlocksBackend) getNode(ctx context.Context, path ImmutablePath) (ContentPathMetadata, format.Node, error) { - roots, lastSeg, err := bb.getPathRoots(ctx, path) +func (bb *BlocksBackend) getNode(ctx context.Context, path path.ImmutablePath) (ContentPathMetadata, format.Node, error) { + roots, lastSeg, remainder, err := bb.getPathRoots(ctx, path) if err != nil { return ContentPathMetadata{}, nil, err } md := ContentPathMetadata{ - PathSegmentRoots: roots, - LastSegment: lastSeg, + PathSegmentRoots: roots, + LastSegment: lastSeg, + LastSegmentRemainder: remainder, } - lastRoot := lastSeg.Cid() + lastRoot := lastSeg.RootCid() nd, err := bb.dagService.Get(ctx, lastRoot) if err != nil { @@ -562,7 +559,7 @@ func (bb *BlocksBackend) getNode(ctx context.Context, path ImmutablePath) (Conte return md, nd, err } -func (bb *BlocksBackend) getPathRoots(ctx context.Context, contentPath ImmutablePath) ([]cid.Cid, ifacepath.Resolved, error) { +func (bb *BlocksBackend) getPathRoots(ctx context.Context, contentPath path.ImmutablePath) ([]cid.Cid, path.ImmutablePath, []string, error) { /* These are logical roots where each CID represent one path segment and resolves to either a directory or the root block of a file. @@ -586,56 +583,50 @@ func (bb *BlocksBackend) getPathRoots(ctx context.Context, contentPath Immutable contentPathStr := contentPath.String() pathSegments := strings.Split(contentPathStr[6:], "/") sp.WriteString(contentPathStr[:5]) // /ipfs or /ipns - var lastPath ifacepath.Resolved + var ( + lastPath path.ImmutablePath + remainder []string + ) for _, root := range pathSegments { if root == "" { continue } sp.WriteString("/") sp.WriteString(root) - resolvedSubPath, err := bb.resolvePath(ctx, ifacepath.New(sp.String())) + p, err := path.NewPath(sp.String()) + if err != nil { + return nil, nil, nil, err + } + resolvedSubPath, remainderSubPath, err := bb.resolvePath(ctx, p) if err != nil { // TODO: should we be more explicit here and is this part of the IPFSBackend contract? // The issue here was that we returned datamodel.ErrWrongKind instead of this resolver error if isErrNotFound(err) { - return nil, nil, resolver.ErrNoLink{Name: root, Node: lastPath.Cid()} + return nil, nil, nil, &resolver.ErrNoLink{Name: root, Node: lastPath.RootCid()} } - return nil, nil, err + return nil, nil, nil, err } lastPath = resolvedSubPath - pathRoots = append(pathRoots, lastPath.Cid()) + remainder = remainderSubPath + pathRoots = append(pathRoots, lastPath.RootCid()) } pathRoots = pathRoots[:len(pathRoots)-1] - return pathRoots, lastPath, nil + return pathRoots, lastPath, remainder, nil } -func (bb *BlocksBackend) ResolveMutable(ctx context.Context, p ifacepath.Path) (ImmutablePath, error) { - err := p.IsValid() - if err != nil { - return ImmutablePath{}, err - } - - ipath := ipfspath.Path(p.String()) - switch ipath.Segments()[0] { - case "ipns": - ipath, err = resolve.ResolveIPNS(ctx, bb.namesys, ipath) - if err != nil { - return ImmutablePath{}, err - } - imPath, err := NewImmutablePath(ifacepath.New(ipath.String())) +func (bb *BlocksBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { + switch p.Namespace() { + case path.IPNSNamespace: + p, err := resolve.ResolveIPNS(ctx, bb.namesys, p) if err != nil { - return ImmutablePath{}, err - } - return imPath, nil - case "ipfs": - imPath, err := NewImmutablePath(ifacepath.New(ipath.String())) - if err != nil { - return ImmutablePath{}, err + return nil, err } - return imPath, nil + return path.NewImmutablePath(p) + case path.IPFSNamespace: + return path.NewImmutablePath(p) default: - return ImmutablePath{}, NewErrorStatusCode(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) + return nil, NewErrorStatusCode(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) } } @@ -659,73 +650,75 @@ func (bb *BlocksBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, return bb.routing.GetValue(ctx, "/ipns/"+string(id)) } -func (bb *BlocksBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (ifacepath.Path, error) { +func (bb *BlocksBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) { if bb.namesys != nil { p, err := bb.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) if err == namesys.ErrResolveRecursion { err = nil } - return ifacepath.New(p.String()), err + return p, err } return nil, NewErrorStatusCode(errors.New("not implemented"), http.StatusNotImplemented) } -func (bb *BlocksBackend) IsCached(ctx context.Context, p ifacepath.Path) bool { - rp, err := bb.resolvePath(ctx, p) +func (bb *BlocksBackend) IsCached(ctx context.Context, p path.Path) bool { + rp, _, err := bb.resolvePath(ctx, p) if err != nil { return false } - has, _ := bb.blockStore.Has(ctx, rp.Cid()) + has, _ := bb.blockStore.Has(ctx, rp.RootCid()) return has } -func (bb *BlocksBackend) ResolvePath(ctx context.Context, path ImmutablePath) (ContentPathMetadata, error) { - roots, lastSeg, err := bb.getPathRoots(ctx, path) +func (bb *BlocksBackend) ResolvePath(ctx context.Context, path path.ImmutablePath) (ContentPathMetadata, error) { + roots, lastSeg, remainder, err := bb.getPathRoots(ctx, path) if err != nil { return ContentPathMetadata{}, err } md := ContentPathMetadata{ - PathSegmentRoots: roots, - LastSegment: lastSeg, + PathSegmentRoots: roots, + LastSegment: lastSeg, + LastSegmentRemainder: remainder, } return md, nil } -func (bb *BlocksBackend) resolvePath(ctx context.Context, p ifacepath.Path) (ifacepath.Resolved, error) { - if _, ok := p.(ifacepath.Resolved); ok { - return p.(ifacepath.Resolved), nil +func (bb *BlocksBackend) resolvePath(ctx context.Context, p path.Path) (path.ImmutablePath, []string, error) { + var err error + if p.Namespace() == path.IPNSNamespace { + p, err = resolve.ResolveIPNS(ctx, bb.namesys, p) + if err != nil { + return nil, nil, err + } } - err := p.IsValid() - if err != nil { - return nil, err + if p.Namespace() != path.IPFSNamespace { + return nil, nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace()) } - ipath := ipfspath.Path(p.String()) - if ipath.Segments()[0] == "ipns" { - ipath, err = resolve.ResolveIPNS(ctx, bb.namesys, ipath) - if err != nil { - return nil, err - } + imPath, err := path.NewImmutablePath(p) + if err != nil { + return nil, nil, err } - if ipath.Segments()[0] != "ipfs" { - return nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace()) + node, remainder, err := bb.resolver.ResolveToLastNode(ctx, imPath) + if err != nil { + return nil, nil, err } - node, rest, err := bb.resolver.ResolveToLastNode(ctx, ipath) + p, err = path.Join(path.FromCid(node), remainder...) if err != nil { - return nil, err + return nil, nil, err } - root, err := cid.Parse(ipath.Segments()[1]) + imPath, err = path.NewImmutablePath(p) if err != nil { - return nil, err + return nil, nil, err } - return ifacepath.NewResolvedPath(ipath, node, root, gopath.Join(rest...)), nil + return imPath, remainder, nil } type nodeGetterToCarExporer struct { diff --git a/gateway/errors.go b/gateway/errors.go index a39bd2a01..08b532e82 100644 --- a/gateway/errors.go +++ b/gateway/errors.go @@ -179,15 +179,14 @@ func webError(w http.ResponseWriter, r *http.Request, c *Config, err error, defa // isErrNotFound returns true for IPLD errors that should return 4xx errors (e.g. the path doesn't exist, the data is // the wrong type, etc.), rather than issues with just finding and retrieving the data. func isErrNotFound(err error) bool { + if errors.Is(err, &resolver.ErrNoLink{}) { + return true + } + // Checks if err is of a type that does not implement the .Is interface and // cannot be directly compared to. Therefore, errors.Is cannot be used. for { - _, ok := err.(resolver.ErrNoLink) - if ok { - return true - } - - _, ok = err.(datamodel.ErrWrongKind) + _, ok := err.(datamodel.ErrWrongKind) if ok { return true } diff --git a/gateway/gateway.go b/gateway/gateway.go index 089dd236c..bfafa48e1 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -9,10 +9,10 @@ import ( "strconv" "strings" - "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/gateway/assets" "github.com/ipfs/boxo/ipld/unixfs" + "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" ) @@ -93,38 +93,6 @@ type PublicGateway struct { DeserializedResponses bool } -// ImmutablePath represents a [path.Path] that is not mutable. -// -// TODO: Is this what we want for ImmutablePath? -type ImmutablePath struct { - p path.Path -} - -func NewImmutablePath(p path.Path) (ImmutablePath, error) { - if p.Mutable() { - return ImmutablePath{}, fmt.Errorf("path cannot be mutable") - } - return ImmutablePath{p: p}, nil -} - -func (i ImmutablePath) String() string { - return i.p.String() -} - -func (i ImmutablePath) Namespace() string { - return i.p.Namespace() -} - -func (i ImmutablePath) Mutable() bool { - return false -} - -func (i ImmutablePath) IsValid() error { - return i.p.IsValid() -} - -var _ path.Path = (*ImmutablePath)(nil) - type CarParams struct { Range *DagByteRange Scope DagScope @@ -241,9 +209,10 @@ func (d DuplicateBlocksPolicy) String() string { } type ContentPathMetadata struct { - PathSegmentRoots []cid.Cid - LastSegment path.Resolved - ContentType string // Only used for UnixFS requests + PathSegmentRoots []cid.Cid + LastSegment path.ImmutablePath + LastSegmentRemainder []string + ContentType string // Only used for UnixFS requests } // ByteRange describes a range request within a UnixFS file. "From" and "To" mostly @@ -363,14 +332,14 @@ type IPFSBackend interface { // block rather than as an [io.ReadCloser] that starts at the beginning of the range request. // // [HTTP Byte Ranges]: https://httpwg.org/specs/rfc9110.html#rfc.section.14.1.2 - Get(context.Context, ImmutablePath, ...ByteRange) (ContentPathMetadata, *GetResponse, error) + Get(context.Context, path.ImmutablePath, ...ByteRange) (ContentPathMetadata, *GetResponse, error) // GetAll returns a UnixFS file or directory depending on what the path is that has been requested. Directories should // include all content recursively. - GetAll(context.Context, ImmutablePath) (ContentPathMetadata, files.Node, error) + GetAll(context.Context, path.ImmutablePath) (ContentPathMetadata, files.Node, error) // GetBlock returns a single block of data - GetBlock(context.Context, ImmutablePath) (ContentPathMetadata, files.File, error) + GetBlock(context.Context, path.ImmutablePath) (ContentPathMetadata, files.File, error) // Head returns a [HeadResponse] depending on what the path is that has been requested. // For UnixFS files (and raw blocks) should return the size of the file and either set the ContentType in @@ -381,16 +350,16 @@ type IPFSBackend interface { // // For all other data types (e.g. (DAG-)CBOR/JSON blocks) returning the size information as a file while setting // the content-type is sufficient. - Head(context.Context, ImmutablePath) (ContentPathMetadata, *HeadResponse, error) + Head(context.Context, path.ImmutablePath) (ContentPathMetadata, *HeadResponse, error) // ResolvePath resolves the path using UnixFS resolver. If the path does not // exist due to a missing link, it should return an error of type: // NewErrorResponse(fmt.Errorf("no link named %q under %s", name, cid), http.StatusNotFound) - ResolvePath(context.Context, ImmutablePath) (ContentPathMetadata, error) + ResolvePath(context.Context, path.ImmutablePath) (ContentPathMetadata, error) // GetCAR returns a CAR file for the given immutable path. It returns an error // if there was an issue before the CAR streaming begins. - GetCAR(context.Context, ImmutablePath, CarParams) (ContentPathMetadata, io.ReadCloser, error) + GetCAR(context.Context, path.ImmutablePath, CarParams) (ContentPathMetadata, io.ReadCloser, error) // IsCached returns whether or not the path exists locally. IsCached(context.Context, path.Path) bool @@ -404,7 +373,7 @@ type IPFSBackend interface { // // For example, given a mapping from `/ipns/dnslink.tld -> /ipns/ipns-id/mydirectory` and `/ipns/ipns-id` to // `/ipfs/some-cid`, the result of passing `/ipns/dnslink.tld/myfile` would be `/ipfs/some-cid/mydirectory/myfile`. - ResolveMutable(context.Context, path.Path) (ImmutablePath, error) + ResolveMutable(context.Context, path.Path) (path.ImmutablePath, error) // GetDNSLinkRecord returns the DNSLink TXT record for the provided FQDN. // Unlike ResolvePath, it does not perform recursive resolution. It only diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index d041cad55..775a4e24b 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -9,10 +9,9 @@ import ( "testing" "time" - ipath "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/namesys" - path "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/path/resolver" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" @@ -26,14 +25,23 @@ func TestGatewayGet(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - k, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), "subdir", "fnord")) + p, err := path.Join(path.FromCid(root), "subdir", "fnord") require.NoError(t, err) - backend.namesys["/ipns/example.com"] = path.FromCid(k.Cid()) - backend.namesys["/ipns/working.example.com"] = path.FromString(k.String()) - backend.namesys["/ipns/double.example.com"] = path.FromString("/ipns/working.example.com") - backend.namesys["/ipns/triple.example.com"] = path.FromString("/ipns/double.example.com") - backend.namesys["/ipns/broken.example.com"] = path.FromString("/ipns/" + k.Cid().String()) + k, err := backend.resolvePathNoRootsReturned(ctx, p) + require.NoError(t, err) + + mustMakeDNSLinkPath := func(domain string) path.Path { + p, err := path.NewPath("/ipns/" + domain) + require.NoError(t, err) + return p + } + + backend.namesys["/ipns/example.com"] = path.FromCid(k.RootCid()) + backend.namesys["/ipns/working.example.com"] = k + backend.namesys["/ipns/double.example.com"] = mustMakeDNSLinkPath("working.example.com") + backend.namesys["/ipns/triple.example.com"] = mustMakeDNSLinkPath("double.example.com") + backend.namesys["/ipns/broken.example.com"] = mustMakeDNSLinkPath(k.RootCid().String()) // We picked .man because: // 1. It's a valid TLD. // 2. Go treats it as the file extension for "man" files (even though @@ -41,7 +49,7 @@ func TestGatewayGet(t *testing.T) { // // Unfortunately, this may not work on all platforms as file type // detection is platform dependent. - backend.namesys["/ipns/example.man"] = path.FromString(k.String()) + backend.namesys["/ipns/example.man"] = k for _, test := range []struct { host string @@ -50,10 +58,10 @@ func TestGatewayGet(t *testing.T) { text string }{ {"127.0.0.1:8080", "/", http.StatusNotFound, "404 page not found\n"}, - {"127.0.0.1:8080", "/ipfs", http.StatusBadRequest, "invalid path \"/ipfs/\": not enough path components\n"}, - {"127.0.0.1:8080", "/ipns", http.StatusBadRequest, "invalid path \"/ipns/\": not enough path components\n"}, - {"127.0.0.1:8080", "/" + k.Cid().String(), http.StatusNotFound, "404 page not found\n"}, - {"127.0.0.1:8080", "/ipfs/this-is-not-a-cid", http.StatusBadRequest, "invalid path \"/ipfs/this-is-not-a-cid\": invalid CID: invalid cid: illegal base32 data at input byte 3\n"}, + {"127.0.0.1:8080", "/ipfs", http.StatusBadRequest, "invalid path \"/ipfs/\": path does not have enough components\n"}, + {"127.0.0.1:8080", "/ipns", http.StatusBadRequest, "invalid path \"/ipns/\": path does not have enough components\n"}, + {"127.0.0.1:8080", "/" + k.RootCid().String(), http.StatusNotFound, "404 page not found\n"}, + {"127.0.0.1:8080", "/ipfs/this-is-not-a-cid", http.StatusBadRequest, "invalid path \"/ipfs/this-is-not-a-cid\": invalid cid: illegal base32 data at input byte 3\n"}, {"127.0.0.1:8080", k.String(), http.StatusOK, "fnord"}, {"127.0.0.1:8080", "/ipns/nxdomain.example.com", http.StatusInternalServerError, "failed to resolve /ipns/nxdomain.example.com: " + namesys.ErrResolveFailed.Error() + "\n"}, {"127.0.0.1:8080", "/ipns/%0D%0A%0D%0Ahello", http.StatusInternalServerError, "failed to resolve /ipns/\\r\\n\\r\\nhello: " + namesys.ErrResolveFailed.Error() + "\n"}, @@ -64,7 +72,7 @@ func TestGatewayGet(t *testing.T) { {"working.example.com", "/", http.StatusOK, "fnord"}, {"double.example.com", "/", http.StatusOK, "fnord"}, {"triple.example.com", "/", http.StatusOK, "fnord"}, - {"working.example.com", k.String(), http.StatusNotFound, "failed to resolve /ipns/working.example.com" + k.String() + ": no link named \"ipfs\" under " + k.Cid().String() + "\n"}, + {"working.example.com", k.String(), http.StatusNotFound, "failed to resolve /ipns/working.example.com" + k.String() + ": no link named \"ipfs\" under " + k.RootCid().String() + "\n"}, {"broken.example.com", "/", http.StatusInternalServerError, "failed to resolve /ipns/broken.example.com/: " + namesys.ErrResolveFailed.Error() + "\n"}, {"broken.example.com", k.String(), http.StatusInternalServerError, "failed to resolve /ipns/broken.example.com" + k.String() + ": " + namesys.ErrResolveFailed.Error() + "\n"}, // This test case ensures we don't treat the TLD as a file extension. @@ -707,43 +715,43 @@ type errorMockBackend struct { err error } -func (mb *errorMockBackend) Get(ctx context.Context, path ImmutablePath, getRange ...ByteRange) (ContentPathMetadata, *GetResponse, error) { +func (mb *errorMockBackend) Get(ctx context.Context, path path.ImmutablePath, getRange ...ByteRange) (ContentPathMetadata, *GetResponse, error) { return ContentPathMetadata{}, nil, mb.err } -func (mb *errorMockBackend) GetAll(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { +func (mb *errorMockBackend) GetAll(ctx context.Context, path path.ImmutablePath) (ContentPathMetadata, files.Node, error) { return ContentPathMetadata{}, nil, mb.err } -func (mb *errorMockBackend) GetBlock(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.File, error) { +func (mb *errorMockBackend) GetBlock(ctx context.Context, path path.ImmutablePath) (ContentPathMetadata, files.File, error) { return ContentPathMetadata{}, nil, mb.err } -func (mb *errorMockBackend) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { +func (mb *errorMockBackend) Head(ctx context.Context, path path.ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { return ContentPathMetadata{}, nil, mb.err } -func (mb *errorMockBackend) GetCAR(ctx context.Context, path ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { +func (mb *errorMockBackend) GetCAR(ctx context.Context, path path.ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { return ContentPathMetadata{}, nil, mb.err } -func (mb *errorMockBackend) ResolveMutable(ctx context.Context, path ipath.Path) (ImmutablePath, error) { - return ImmutablePath{}, mb.err +func (mb *errorMockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { + return nil, mb.err } func (mb *errorMockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { return nil, mb.err } -func (mb *errorMockBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (ipath.Path, error) { +func (mb *errorMockBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) { return nil, mb.err } -func (mb *errorMockBackend) IsCached(ctx context.Context, p ipath.Path) bool { +func (mb *errorMockBackend) IsCached(ctx context.Context, p path.Path) bool { return false } -func (mb *errorMockBackend) ResolvePath(ctx context.Context, path ImmutablePath) (ContentPathMetadata, error) { +func (mb *errorMockBackend) ResolvePath(ctx context.Context, path path.ImmutablePath) (ContentPathMetadata, error) { return ContentPathMetadata{}, mb.err } @@ -763,7 +771,7 @@ func TestErrorBubblingFromBackend(t *testing.T) { } testError("500 Not Found from IPLD", &ipld.ErrNotFound{}, http.StatusInternalServerError) - testError("404 Not Found from path resolver", resolver.ErrNoLink{}, http.StatusNotFound) + testError("404 Not Found from path resolver", &resolver.ErrNoLink{}, http.StatusNotFound) testError("502 Bad Gateway", ErrBadGateway, http.StatusBadGateway) testError("504 Gateway Timeout", ErrGatewayTimeout, http.StatusGatewayTimeout) @@ -791,27 +799,27 @@ type panicMockBackend struct { panicOnHostnameHandler bool } -func (mb *panicMockBackend) Get(ctx context.Context, immutablePath ImmutablePath, ranges ...ByteRange) (ContentPathMetadata, *GetResponse, error) { +func (mb *panicMockBackend) Get(ctx context.Context, immutablePath path.ImmutablePath, ranges ...ByteRange) (ContentPathMetadata, *GetResponse, error) { panic("i am panicking") } -func (mb *panicMockBackend) GetAll(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { +func (mb *panicMockBackend) GetAll(ctx context.Context, immutablePath path.ImmutablePath) (ContentPathMetadata, files.Node, error) { panic("i am panicking") } -func (mb *panicMockBackend) GetBlock(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.File, error) { +func (mb *panicMockBackend) GetBlock(ctx context.Context, immutablePath path.ImmutablePath) (ContentPathMetadata, files.File, error) { panic("i am panicking") } -func (mb *panicMockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { +func (mb *panicMockBackend) Head(ctx context.Context, immutablePath path.ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { panic("i am panicking") } -func (mb *panicMockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { +func (mb *panicMockBackend) GetCAR(ctx context.Context, immutablePath path.ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { panic("i am panicking") } -func (mb *panicMockBackend) ResolveMutable(ctx context.Context, p ipath.Path) (ImmutablePath, error) { +func (mb *panicMockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { panic("i am panicking") } @@ -819,7 +827,7 @@ func (mb *panicMockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byt panic("i am panicking") } -func (mb *panicMockBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (ipath.Path, error) { +func (mb *panicMockBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) { // GetDNSLinkRecord is also called on the WithHostname handler. We have this option // to disable panicking here so we can test if both the regular gateway handler // and the hostname handler can handle panics. @@ -830,11 +838,11 @@ func (mb *panicMockBackend) GetDNSLinkRecord(ctx context.Context, hostname strin return nil, errors.New("not implemented") } -func (mb *panicMockBackend) IsCached(ctx context.Context, p ipath.Path) bool { +func (mb *panicMockBackend) IsCached(ctx context.Context, p path.Path) bool { panic("i am panicking") } -func (mb *panicMockBackend) ResolvePath(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, error) { +func (mb *panicMockBackend) ResolvePath(ctx context.Context, immutablePath path.ImmutablePath) (ContentPathMetadata, error) { panic("i am panicking") } diff --git a/gateway/handler.go b/gateway/handler.go index af20e2b6e..6bcd515f9 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -16,9 +16,9 @@ import ( "strings" "time" - ipath "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/gateway/assets" "github.com/ipfs/boxo/ipns" + "github.com/ipfs/boxo/path" cid "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/peer" @@ -188,12 +188,12 @@ type requestData struct { // Defined for all requests. begin time.Time logger *zap.SugaredLogger - contentPath ipath.Path + contentPath path.Path responseFormat string responseParams map[string]string // Defined for non IPNS Record requests. - immutablePath ImmutablePath + immutablePath path.ImmutablePath // Defined if resolution has already happened. pathMetadata *ContentPathMetadata @@ -202,9 +202,9 @@ type requestData struct { // mostlyResolvedPath is an opportunistic optimization that returns the mostly // resolved version of ImmutablePath available. It does not guarantee it is fully // resolved, nor that it is the original. -func (rq *requestData) mostlyResolvedPath() ImmutablePath { +func (rq *requestData) mostlyResolvedPath() path.ImmutablePath { if rq.pathMetadata != nil { - imPath, err := NewImmutablePath(rq.pathMetadata.LastSegment) + imPath, err := path.NewImmutablePath(rq.pathMetadata.LastSegment) if err != nil { // This will never happen. This error has previously been checked in // [handleIfNoneMatch] and the request will have returned 500. @@ -223,12 +223,18 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { if handleProtocolHandlerRedirect(w, r, i.config) || i.handleServiceWorkerRegistration(w, r) || - handleIpnsB58mhToCidRedirection(w, r) { + handleIpnsB58mhToCidRedirection(w, r) || + i.handleSuperfluousNamespace(w, r) { return } var success bool - contentPath := ipath.New(r.URL.Path) + contentPath, err := path.NewPath(r.URL.Path) + if err != nil { + i.webError(w, r, err, http.StatusBadRequest) + return + } + ctx := context.WithValue(r.Context(), ContentPathKey, contentPath) r = r.WithContext(ctx) @@ -238,13 +244,7 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { } }() - if i.handleOnlyIfCached(w, r, contentPath) || - i.handleSuperfluousNamespace(w, r, contentPath) { - return - } - - if err := contentPath.IsValid(); err != nil { - i.webError(w, r, err, http.StatusBadRequest) + if i.handleOnlyIfCached(w, r, contentPath) { return } @@ -292,7 +292,7 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { return } } else { - rq.immutablePath, err = NewImmutablePath(contentPath) + rq.immutablePath, err = path.NewImmutablePath(contentPath) if err != nil { err = fmt.Errorf("path was expected to be immutable, but was not %s: %w", debugStr(contentPath.String()), err) i.webError(w, r, err, http.StatusInternalServerError) @@ -371,7 +371,7 @@ func (i *handler) isDeserializedResponsePossible(r *http.Request) bool { // in the [Trustless Gateway] spec. // // [Trustless Gateway]: https://specs.ipfs.tech/http-gateways/trustless-gateway/ -func (i *handler) isTrustlessRequest(contentPath ipath.Path, responseFormat string) bool { +func (i *handler) isTrustlessRequest(contentPath path.Path, responseFormat string) bool { // Only allow "/{#1}/{#2}"-like paths. trimmedPath := strings.Trim(contentPath.String(), "/") pathComponents := strings.Split(trimmedPath, "/") @@ -379,7 +379,7 @@ func (i *handler) isTrustlessRequest(contentPath ipath.Path, responseFormat stri return false } - if contentPath.Namespace() == "ipns" { + if contentPath.Namespace() == path.IPNSNamespace { // TODO: only ipns records allowed until https://github.com/ipfs/specs/issues/369 is resolved if responseFormat != ipnsRecordResponseFormat { return false @@ -415,7 +415,7 @@ func panicHandler(w http.ResponseWriter) { } } -func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, cid cid.Cid, responseFormat string) (modtime time.Time) { +func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath path.Path, cid cid.Cid, responseFormat string) (modtime time.Time) { // Best effort attempt to set an Etag based on the CID and response format. // Setting an ETag is handled separately for CARs and IPNS records. if etag := getEtag(r, cid, responseFormat); etag != "" { @@ -452,7 +452,7 @@ func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath // - Creation of HTML links that trigger "Save As.." dialog instead of being rendered by the browser // - Overriding the filename used when saving sub-resource assets on HTML page // - providing a default filename for HTTP clients when downloading direct /ipfs/CID without any subpath -func addContentDispositionHeader(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) string { +func addContentDispositionHeader(w http.ResponseWriter, r *http.Request, contentPath path.Path) string { // URL param ?filename=cat.jpg triggers Content-Disposition: [..] filename // which impacts default name used in "Save As.." dialog name := getFilename(contentPath) @@ -470,7 +470,7 @@ func addContentDispositionHeader(w http.ResponseWriter, r *http.Request, content return name } -func getFilename(contentPath ipath.Path) string { +func getFilename(contentPath path.Path) string { s := contentPath.String() if (strings.HasPrefix(s, ipfsPathPrefix) || strings.HasPrefix(s, ipnsPathPrefix)) && strings.Count(gopath.Clean(s), "/") <= 2 { // Don't want to treat ipfs.io in /ipns/ipfs.io as a filename. @@ -520,7 +520,7 @@ func setIpfsRootsHeader(w http.ResponseWriter, rq *requestData, md *ContentPathM for _, c := range rq.pathMetadata.PathSegmentRoots { pathRoots = append(pathRoots, c.String()) } - pathRoots = append(pathRoots, rq.pathMetadata.LastSegment.Cid().String()) + pathRoots = append(pathRoots, rq.pathMetadata.LastSegment.RootCid().String()) rootCidList := strings.Join(pathRoots, ",") // convention from rfc2616#sec4.2 w.Header().Set("X-Ipfs-Roots", rootCidList) @@ -675,7 +675,7 @@ func (i *handler) handleIfNoneMatch(w http.ResponseWriter, r *http.Request, rq * if ifNoneMatch := r.Header.Get("If-None-Match"); ifNoneMatch != "" { pathMetadata, err := i.backend.ResolvePath(r.Context(), rq.immutablePath) if err != nil { - var forwardedPath ImmutablePath + var forwardedPath path.ImmutablePath var continueProcessing bool if isWebRequest(rq.responseFormat) { forwardedPath, continueProcessing = i.handleWebRequestErrors(w, r, rq.mostlyResolvedPath(), rq.immutablePath, rq.contentPath, err, rq.logger) @@ -690,7 +690,7 @@ func (i *handler) handleIfNoneMatch(w http.ResponseWriter, r *http.Request, rq * } } - pathCid := pathMetadata.LastSegment.Cid() + pathCid := pathMetadata.LastSegment.RootCid() // Checks against both file, dir listing, and dag index Etags. // This is an inexpensive check, and it happens before we do any I/O. @@ -705,7 +705,7 @@ func (i *handler) handleIfNoneMatch(w http.ResponseWriter, r *http.Request, rq * } // Check if the resolvedPath is an immutable path. - _, err = NewImmutablePath(pathMetadata.LastSegment) + _, err = path.NewImmutablePath(pathMetadata.LastSegment) if err != nil { i.webError(w, r, err, http.StatusInternalServerError) return true @@ -726,7 +726,7 @@ func isWebRequest(responseFormat string) bool { } // handleRequestErrors is used when request type is other than Web+UnixFS -func (i *handler) handleRequestErrors(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, err error) bool { +func (i *handler) handleRequestErrors(w http.ResponseWriter, r *http.Request, contentPath path.Path, err error) bool { if err == nil { return true } @@ -737,7 +737,7 @@ func (i *handler) handleRequestErrors(w http.ResponseWriter, r *http.Request, co // handleWebRequestErrors is used when request type is Web+UnixFS and err could // be a 404 (Not Found) that should be recovered via _redirects file (IPIP-290) -func (i *handler) handleWebRequestErrors(w http.ResponseWriter, r *http.Request, maybeResolvedImPath, immutableContentPath ImmutablePath, contentPath ipath.Path, err error, logger *zap.SugaredLogger) (ImmutablePath, bool) { +func (i *handler) handleWebRequestErrors(w http.ResponseWriter, r *http.Request, maybeResolvedImPath, immutableContentPath path.ImmutablePath, contentPath path.Path, err error, logger *zap.SugaredLogger) (path.ImmutablePath, bool) { if err == nil { return maybeResolvedImPath, true } @@ -745,14 +745,14 @@ func (i *handler) handleWebRequestErrors(w http.ResponseWriter, r *http.Request, if errors.Is(err, ErrServiceUnavailable) { err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) i.webError(w, r, err, http.StatusServiceUnavailable) - return ImmutablePath{}, false + return nil, false } // If the error is not an IPLD traversal error then we should not be looking for _redirects or legacy 404s if !isErrNotFound(err) { err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) i.webError(w, r, err, http.StatusInternalServerError) - return ImmutablePath{}, false + return nil, false } // If we have origin isolation (subdomain gw, DNSLink website), @@ -774,17 +774,17 @@ func (i *handler) handleWebRequestErrors(w http.ResponseWriter, r *http.Request, // follow https://docs.ipfs.tech/how-to/websites-on-ipfs/redirects-and-custom-404s/ instead. if i.serveLegacy404IfPresent(w, r, immutableContentPath, logger) { logger.Debugw("served legacy 404") - return ImmutablePath{}, false + return nil, false } err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) i.webError(w, r, err, http.StatusInternalServerError) - return ImmutablePath{}, false + return nil, false } // Detect 'Cache-Control: only-if-cached' in request and return data if it is already in the local datastore. // https://github.com/ipfs/specs/blob/main/http-gateways/PATH_GATEWAY.md#cache-control-request-header -func (i *handler) handleOnlyIfCached(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) bool { +func (i *handler) handleOnlyIfCached(w http.ResponseWriter, r *http.Request, contentPath path.Path) bool { if r.Header.Get("Cache-Control") == "only-if-cached" { if !i.backend.IsCached(r.Context(), contentPath) { if r.Method == http.MethodHead { @@ -894,23 +894,19 @@ func handleIpnsB58mhToCidRedirection(w http.ResponseWriter, r *http.Request) boo // 'intended' path is valid. This is in case gremlins were tickled // wrong way and user ended up at /ipfs/ipfs/{cid} or /ipfs/ipns/{id} // like in bafybeien3m7mdn6imm425vc2s22erzyhbvk5n3ofzgikkhmdkh5cuqbpbq :^)) -func (i *handler) handleSuperfluousNamespace(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) bool { - // If the path is valid, there's nothing to do - if pathErr := contentPath.IsValid(); pathErr == nil { - return false - } - +func (i *handler) handleSuperfluousNamespace(w http.ResponseWriter, r *http.Request) bool { // If there's no superflous namespace, there's nothing to do if !(strings.HasPrefix(r.URL.Path, "/ipfs/ipfs/") || strings.HasPrefix(r.URL.Path, "/ipfs/ipns/")) { return false } // Attempt to fix the superflous namespace - intendedPath := ipath.New(strings.TrimPrefix(r.URL.Path, "/ipfs")) - if err := intendedPath.IsValid(); err != nil { + intendedPath, err := path.NewPath(strings.TrimPrefix(r.URL.Path, "/ipfs")) + if err != nil { i.webError(w, r, fmt.Errorf("invalid ipfs path: %w", err), http.StatusBadRequest) return true } + intendedURL := intendedPath.String() if r.URL.RawQuery != "" { // we render HTML, so ensure query entries are properly escaped @@ -935,7 +931,7 @@ func (i *handler) handleSuperfluousNamespace(w http.ResponseWriter, r *http.Requ } // getTemplateGlobalData returns the global data necessary by most templates. -func (i *handler) getTemplateGlobalData(r *http.Request, contentPath ipath.Path) assets.GlobalData { +func (i *handler) getTemplateGlobalData(r *http.Request, contentPath path.Path) assets.GlobalData { // gatewayURL is used to link to other root CIDs. THis will be blank unless // subdomain or DNSLink resolution is being used for this request. var gatewayURL string diff --git a/gateway/handler_block.go b/gateway/handler_block.go index 9d2a93b38..461e306d8 100644 --- a/gateway/handler_block.go +++ b/gateway/handler_block.go @@ -22,7 +22,7 @@ func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *h setIpfsRootsHeader(w, rq, &pathMetadata) - blockCid := pathMetadata.LastSegment.Cid() + blockCid := pathMetadata.LastSegment.RootCid() // Set Content-Disposition var name string diff --git a/gateway/handler_car.go b/gateway/handler_car.go index 000e0dc9c..0b2162c4f 100644 --- a/gateway/handler_car.go +++ b/gateway/handler_car.go @@ -10,6 +10,7 @@ import ( "time" "github.com/cespare/xxhash/v2" + "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" "go.opentelemetry.io/otel/attribute" @@ -203,7 +204,7 @@ func buildContentTypeFromCarParams(params CarParams) string { return h.String() } -func getCarRootCidAndLastSegment(imPath ImmutablePath) (cid.Cid, string, error) { +func getCarRootCidAndLastSegment(imPath path.ImmutablePath) (cid.Cid, string, error) { imPathStr := imPath.String() if !strings.HasPrefix(imPathStr, "/ipfs/") { return cid.Undef, "", fmt.Errorf("path does not have /ipfs/ prefix") @@ -224,7 +225,7 @@ func getCarRootCidAndLastSegment(imPath ImmutablePath) (cid.Cid, string, error) return rootCid, lastSegment, err } -func getCarEtag(imPath ImmutablePath, params CarParams, rootCid cid.Cid) string { +func getCarEtag(imPath path.ImmutablePath, params CarParams, rootCid cid.Cid) string { data := imPath.String() if params.Scope != DagScopeAll { data += string(params.Scope) diff --git a/gateway/handler_car_test.go b/gateway/handler_car_test.go index 65777453d..efbad2b22 100644 --- a/gateway/handler_car_test.go +++ b/gateway/handler_car_test.go @@ -4,7 +4,7 @@ import ( "net/http" "testing" - "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -143,7 +143,7 @@ func TestGetCarEtag(t *testing.T) { cid, err := cid.Parse("bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") require.NoError(t, err) - imPath, err := NewImmutablePath(path.IpfsPath(cid)) + imPath, err := path.NewImmutablePath(path.FromCid(cid)) require.NoError(t, err) t.Run("Etag with entity-bytes=0:* is the same as without query param", func(t *testing.T) { diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index 97dfaad0a..c1abb1d27 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -10,8 +10,8 @@ import ( "strings" "time" - ipath "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/gateway/assets" + "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime/multicodec" "github.com/ipld/go-ipld-prime/node/basicnode" @@ -83,16 +83,17 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt ctx, span := spanTrace(ctx, "Handler.RenderCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", rq.responseFormat))) defer span.End() - blockCid := resolvedPath.Cid() + blockCid := resolvedPath.RootCid() cidCodec := mc.Code(blockCid.Prefix().Codec) responseContentType := rq.responseFormat // If the resolved path still has some remainder, return error for now. // TODO: handle this when we have IPLD Patch (https://ipld.io/specs/patch/) via HTTP PUT // TODO: (depends on https://github.com/ipfs/kubo/issues/4801 and https://github.com/ipfs/kubo/issues/4782) - if resolvedPath.Remainder() != "" { - path := strings.TrimSuffix(resolvedPath.String(), resolvedPath.Remainder()) - err := fmt.Errorf("%q of %q could not be returned: reading IPLD Kinds other than Links (CBOR Tag 42) is not implemented: try reading %q instead", resolvedPath.Remainder(), resolvedPath.String(), path) + if len(rq.pathMetadata.LastSegmentRemainder) != 0 { + remainderStr := path.SegmentsToString(rq.pathMetadata.LastSegmentRemainder...) + path := strings.TrimSuffix(resolvedPath.String(), remainderStr) + err := fmt.Errorf("%q of %q could not be returned: reading IPLD Kinds other than Links (CBOR Tag 42) is not implemented: try reading %q instead", remainderStr, resolvedPath.String(), path) i.webError(w, r, err, http.StatusNotImplemented) return false } @@ -110,7 +111,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt } // Set HTTP headers (for caching, etc). Etag will be replaced if handled by serveCodecHTML. - modtime := addCacheControlHeaders(w, r, rq.contentPath, resolvedPath.Cid(), responseContentType) + modtime := addCacheControlHeaders(w, r, rq.contentPath, resolvedPath.RootCid(), responseContentType) _ = setCodecContentDisposition(w, r, resolvedPath, responseContentType) w.Header().Set("Content-Type", responseContentType) w.Header().Set("X-Content-Type-Options", "nosniff") @@ -155,7 +156,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt return i.serveCodecConverted(ctx, w, r, blockCid, blockData, rq.contentPath, toCodec, modtime, rq.begin) } -func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, blockCid cid.Cid, blockData io.Reader, resolvedPath ipath.Resolved, contentPath ipath.Path) bool { +func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r *http.Request, blockCid cid.Cid, blockData io.Reader, resolvedPath path.ImmutablePath, contentPath path.Path) bool { // WithHostname may have constructed an IPFS (or IPNS) path using the Host header. // In this case, we need the original path for constructing the redirect. requestURI, err := url.ParseRequestURI(r.RequestURI) @@ -185,7 +186,7 @@ func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r * w.Header().Del("Content-Disposition") // Generated index requires custom Etag (output may change between Kubo versions) - dagEtag := getDagIndexEtag(resolvedPath.Cid()) + dagEtag := getDagIndexEtag(resolvedPath.RootCid()) w.Header().Set("Etag", dagEtag) // Remove Cache-Control for now to match UnixFS dir-index-html responses @@ -193,11 +194,11 @@ func (i *handler) serveCodecHTML(ctx context.Context, w http.ResponseWriter, r * // TODO: if we ever change behavior for UnixFS dir listings, same changes should be applied here w.Header().Del("Cache-Control") - cidCodec := mc.Code(resolvedPath.Cid().Prefix().Codec) + cidCodec := mc.Code(resolvedPath.RootCid().Prefix().Codec) if err := assets.DagTemplate.Execute(w, assets.DagTemplateData{ GlobalData: i.getTemplateGlobalData(r, contentPath), Path: contentPath.String(), - CID: resolvedPath.Cid().String(), + CID: resolvedPath.RootCid().String(), CodecName: cidCodec.String(), CodecHex: fmt.Sprintf("0x%x", uint64(cidCodec)), Node: parseNode(blockCid, blockData), @@ -235,7 +236,7 @@ func parseNode(blockCid cid.Cid, blockData io.Reader) *assets.ParsedNode { } // serveCodecRaw returns the raw block without any conversion -func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, blockSize int64, blockData io.ReadSeekCloser, contentPath ipath.Path, modtime, begin time.Time) bool { +func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, blockSize int64, blockData io.ReadSeekCloser, contentPath path.Path, modtime, begin time.Time) bool { // ServeContent will take care of // If-None-Match+Etag, Content-Length and setting range request headers after we've already seeked to the start of // the first range @@ -253,7 +254,7 @@ func (i *handler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *h } // serveCodecConverted returns payload converted to codec specified in toCodec -func (i *handler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, blockCid cid.Cid, blockData io.ReadCloser, contentPath ipath.Path, toCodec mc.Code, modtime, begin time.Time) bool { +func (i *handler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, blockCid cid.Cid, blockData io.ReadCloser, contentPath path.Path, toCodec mc.Code, modtime, begin time.Time) bool { codec := blockCid.Prefix().Codec decoder, err := multicodec.LookupDecoder(codec) if err != nil { @@ -298,7 +299,7 @@ func (i *handler) serveCodecConverted(ctx context.Context, w http.ResponseWriter return false } -func setCodecContentDisposition(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentType string) string { +func setCodecContentDisposition(w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, contentType string) string { var dispType, name string ext, ok := contentTypeToExtension[contentType] @@ -310,7 +311,7 @@ func setCodecContentDisposition(w http.ResponseWriter, r *http.Request, resolved if urlFilename := r.URL.Query().Get("filename"); urlFilename != "" { name = urlFilename } else { - name = resolvedPath.Cid().String() + ext + name = resolvedPath.RootCid().String() + ext } // JSON should be inlined, but ?download=true should still override diff --git a/gateway/handler_codec_test.go b/gateway/handler_codec_test.go index c79b07689..d22579027 100644 --- a/gateway/handler_codec_test.go +++ b/gateway/handler_codec_test.go @@ -7,7 +7,7 @@ import ( "net/http" "testing" - ipath "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/path" "github.com/stretchr/testify/require" ) @@ -31,10 +31,13 @@ func TestDagJsonCborPreview(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - resolvedPath, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), "subdir", "dag-cbor-document")) + p, err := path.Join(path.FromCid(root), "subdir", "dag-cbor-document") require.NoError(t, err) - cidStr := resolvedPath.Cid().String() + resolvedPath, err := backend.resolvePathNoRootsReturned(ctx, p) + require.NoError(t, err) + + cidStr := resolvedPath.RootCid().String() t.Run("path gateway normalizes to trailing slash", func(t *testing.T) { t.Parallel() diff --git a/gateway/handler_defaults.go b/gateway/handler_defaults.go index 22c397ade..e56e6c5d9 100644 --- a/gateway/handler_defaults.go +++ b/gateway/handler_defaults.go @@ -95,7 +95,7 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h setIpfsRootsHeader(w, rq, &pathMetadata) resolvedPath := pathMetadata.LastSegment - switch mc.Code(resolvedPath.Cid().Prefix().Codec) { + switch mc.Code(resolvedPath.RootCid().Prefix().Codec) { case mc.Json, mc.DagJson, mc.Cbor, mc.DagCbor: rq.logger.Debugw("serving codec", "path", rq.contentPath) var blockSize int64 diff --git a/gateway/handler_ipns_record.go b/gateway/handler_ipns_record.go index b077fa59a..93d1621f2 100644 --- a/gateway/handler_ipns_record.go +++ b/gateway/handler_ipns_record.go @@ -11,6 +11,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/ipfs/boxo/ipns" + "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -20,7 +21,7 @@ func (i *handler) serveIpnsRecord(ctx context.Context, w http.ResponseWriter, r ctx, span := spanTrace(ctx, "Handler.ServeIPNSRecord", trace.WithAttributes(attribute.String("path", rq.contentPath.String()))) defer span.End() - if rq.contentPath.Namespace() != "ipns" { + if rq.contentPath.Namespace() != path.IPNSNamespace { err := fmt.Errorf("%s is not an IPNS link", rq.contentPath.String()) i.webError(w, r, err, http.StatusBadRequest) return false diff --git a/gateway/handler_tar.go b/gateway/handler_tar.go index 784e51993..6af1d0c4d 100644 --- a/gateway/handler_tar.go +++ b/gateway/handler_tar.go @@ -28,7 +28,7 @@ func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.R defer file.Close() setIpfsRootsHeader(w, rq, &pathMetadata) - rootCid := pathMetadata.LastSegment.Cid() + rootCid := pathMetadata.LastSegment.RootCid() // Set Cache-Control and read optional Last-Modified time modtime := addCacheControlHeaders(w, r, rq.contentPath, rootCid, tarResponseFormat) diff --git a/gateway/handler_unixfs__redirects.go b/gateway/handler_unixfs__redirects.go index 978f55647..6b63a793a 100644 --- a/gateway/handler_unixfs__redirects.go +++ b/gateway/handler_unixfs__redirects.go @@ -8,10 +8,9 @@ import ( "strconv" "strings" - "go.uber.org/zap" - - ipath "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/path" redirects "github.com/ipfs/go-ipfs-redirects-file" + "go.uber.org/zap" ) // Resolving a UnixFS path involves determining if the provided `path.Path` exists and returning the `path.Resolved` @@ -36,23 +35,35 @@ import ( // // Note that for security reasons, redirect rules are only processed when the request has origin isolation. // See https://github.com/ipfs/specs/pull/290 for more information. -func (i *handler) serveRedirectsIfPresent(w http.ResponseWriter, r *http.Request, maybeResolvedImPath, immutableContentPath ImmutablePath, contentPath ipath.Path, logger *zap.SugaredLogger) (newContentPath ImmutablePath, continueProcessing bool, hadMatchingRule bool) { +func (i *handler) serveRedirectsIfPresent(w http.ResponseWriter, r *http.Request, maybeResolvedImPath, immutableContentPath path.ImmutablePath, contentPath path.Path, logger *zap.SugaredLogger) (newContentPath path.ImmutablePath, continueProcessing bool, hadMatchingRule bool) { // contentPath is the full ipfs path to the requested resource, // regardless of whether path or subdomain resolution is used. - rootPath := getRootPath(immutableContentPath) - redirectsPath := ipath.Join(rootPath, "_redirects") - imRedirectsPath, err := NewImmutablePath(redirectsPath) + rootPath, err := getRootPath(immutableContentPath) + if err != nil { + err = fmt.Errorf("trouble processing _redirects path %q: %w", immutableContentPath.String(), err) + i.webError(w, r, err, http.StatusInternalServerError) + return nil, false, true + } + + redirectsPath, err := path.Join(rootPath, "_redirects") + if err != nil { + err = fmt.Errorf("trouble processing _redirects path %q: %w", rootPath.String(), err) + i.webError(w, r, err, http.StatusInternalServerError) + return nil, false, true + } + + imRedirectsPath, err := path.NewImmutablePath(redirectsPath) if err != nil { err = fmt.Errorf("trouble processing _redirects path %q: %w", redirectsPath, err) i.webError(w, r, err, http.StatusInternalServerError) - return ImmutablePath{}, false, true + return nil, false, true } foundRedirect, redirectRules, err := i.getRedirectRules(r, imRedirectsPath) if err != nil { err = fmt.Errorf("trouble processing _redirects file at %q: %w", redirectsPath, err) i.webError(w, r, err, http.StatusInternalServerError) - return ImmutablePath{}, false, true + return nil, false, true } if foundRedirect { @@ -60,22 +71,27 @@ func (i *handler) serveRedirectsIfPresent(w http.ResponseWriter, r *http.Request if err != nil { err = fmt.Errorf("trouble processing _redirects file at %q: %w", redirectsPath, err) i.webError(w, r, err, http.StatusInternalServerError) - return ImmutablePath{}, false, true + return nil, false, true } if redirected { - return ImmutablePath{}, false, true + return nil, false, true } // 200 is treated as a rewrite, so update the path and continue if newPath != "" { // Reassign contentPath and resolvedPath since the URL was rewritten - p := ipath.New(newPath) - imPath, err := NewImmutablePath(p) + p, err := path.NewPath(newPath) + if err != nil { + err = fmt.Errorf("could not use _redirects file to %q: %w", p, err) + i.webError(w, r, err, http.StatusInternalServerError) + return nil, false, true + } + imPath, err := path.NewImmutablePath(p) if err != nil { err = fmt.Errorf("could not use _redirects file to %q: %w", p, err) i.webError(w, r, err, http.StatusInternalServerError) - return ImmutablePath{}, false, true + return nil, false, true } return imPath, true, true } @@ -85,7 +101,7 @@ func (i *handler) serveRedirectsIfPresent(w http.ResponseWriter, r *http.Request return maybeResolvedImPath, true, false } -func (i *handler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Request, immutableContentPath ImmutablePath, cPath ipath.Path, redirectRules []redirects.Rule, logger *zap.SugaredLogger) (redirected bool, newContentPath string, err error) { +func (i *handler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Request, immutableContentPath path.ImmutablePath, cPath path.Path, redirectRules []redirects.Rule, logger *zap.SugaredLogger) (redirected bool, newContentPath string, err error) { // Attempt to match a rule to the URL path, and perform the corresponding redirect or rewrite pathParts := strings.Split(immutableContentPath.String(), "/") if len(pathParts) > 3 { @@ -113,7 +129,12 @@ func (i *handler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Reques // Or 4xx if rule.Status == 404 || rule.Status == 410 || rule.Status == 451 { toPath := rootPath + rule.To - imContent4xxPath, err := NewImmutablePath(ipath.New(toPath)) + p, err := path.NewPath(toPath) + if err != nil { + return true, toPath, err + } + + imContent4xxPath, err := path.NewImmutablePath(p) if err != nil { return true, toPath, err } @@ -127,7 +148,11 @@ func (i *handler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Reques } // All paths should start with /ip(f|n)s//, so get the path after that contentRootPath := strings.Join(contentPathParts[:3], "/") - content4xxPath := ipath.New(contentRootPath + rule.To) + content4xxPath, err := path.NewPath(contentRootPath + rule.To) + if err != nil { + return true, toPath, err + } + err = i.serve4xx(w, r, imContent4xxPath, content4xxPath, rule.Status, logger) return true, toPath, err } @@ -147,7 +172,7 @@ func (i *handler) handleRedirectsFileRules(w http.ResponseWriter, r *http.Reques // getRedirectRules fetches the _redirects file corresponding to a given path and returns the rules // Returns whether _redirects was found, the rules (if they exist) and if there was an error (other than a missing _redirects) // If there is an error returns (false, nil, err) -func (i *handler) getRedirectRules(r *http.Request, redirectsPath ImmutablePath) (bool, []redirects.Rule, error) { +func (i *handler) getRedirectRules(r *http.Request, redirectsPath path.ImmutablePath) (bool, []redirects.Rule, error) { // Check for _redirects file. // Any path resolution failures are ignored and we just assume there's no _redirects file. // Note that ignoring these errors also ensures that the use of the empty CID (bafkqaaa) in tests doesn't fail. @@ -174,12 +199,12 @@ func (i *handler) getRedirectRules(r *http.Request, redirectsPath ImmutablePath) } // Returns the root CID Path for the given path -func getRootPath(path ipath.Path) ipath.Path { - parts := strings.Split(path.String(), "/") - return ipath.New(gopath.Join("/", path.Namespace(), parts[2])) +func getRootPath(p path.Path) (path.Path, error) { + parts := strings.Split(p.String(), "/") + return path.NewPath(gopath.Join("/", p.Namespace(), parts[2])) } -func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPathImPath ImmutablePath, content4xxPath ipath.Path, status int, logger *zap.SugaredLogger) error { +func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPathImPath path.ImmutablePath, content4xxPath path.Path, status int, logger *zap.SugaredLogger) error { pathMetadata, getresp, err := i.backend.Get(r.Context(), content4xxPathImPath) if err != nil { return err @@ -191,7 +216,7 @@ func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPat } content4xxFile := getresp.bytes - content4xxCid := pathMetadata.LastSegment.Cid() + content4xxCid := pathMetadata.LastSegment.RootCid() size := getresp.bytesSize @@ -218,7 +243,7 @@ func hasOriginIsolation(r *http.Request) bool { // Deprecated: legacy ipfs-404.html files are superseded by _redirects file // This is provided only for backward-compatibility, until websites migrate // to 404s managed via _redirects file (https://github.com/ipfs/specs/pull/290) -func (i *handler) serveLegacy404IfPresent(w http.ResponseWriter, r *http.Request, imPath ImmutablePath, logger *zap.SugaredLogger) bool { +func (i *handler) serveLegacy404IfPresent(w http.ResponseWriter, r *http.Request, imPath path.ImmutablePath, logger *zap.SugaredLogger) bool { resolved404File, resolved404FileSize, ctype, err := i.searchUpTreeFor404(r, imPath) if err != nil { return false @@ -233,7 +258,7 @@ func (i *handler) serveLegacy404IfPresent(w http.ResponseWriter, r *http.Request return err == nil } -func (i *handler) searchUpTreeFor404(r *http.Request, imPath ImmutablePath) (io.ReadCloser, int64, string, error) { +func (i *handler) searchUpTreeFor404(r *http.Request, imPath path.ImmutablePath) (io.ReadCloser, int64, string, error) { filename404, ctype, err := preferred404Filename(r.Header.Values("Accept")) if err != nil { return nil, 0, "", err @@ -243,11 +268,11 @@ func (i *handler) searchUpTreeFor404(r *http.Request, imPath ImmutablePath) (io. for idx := len(pathComponents); idx >= 3; idx-- { pretty404 := gopath.Join(append(pathComponents[0:idx], filename404)...) - parsed404Path := ipath.New("/" + pretty404) - if parsed404Path.IsValid() != nil { + parsed404Path, err := path.NewPath("/" + pretty404) + if err != nil { break } - imparsed404Path, err := NewImmutablePath(parsed404Path) + imparsed404Path, err := path.NewImmutablePath(parsed404Path) if err != nil { break } diff --git a/gateway/handler_unixfs_dir.go b/gateway/handler_unixfs_dir.go index 1ece9c96d..479f4e3c9 100644 --- a/gateway/handler_unixfs_dir.go +++ b/gateway/handler_unixfs_dir.go @@ -11,10 +11,9 @@ import ( "time" "github.com/dustin/go-humanize" - ipath "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/gateway/assets" - path "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" cid "github.com/ipfs/go-cid" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -24,7 +23,7 @@ import ( // serveDirectory returns the best representation of UnixFS directory // // It will return index.html if present, or generate directory listing otherwise. -func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, isHeadRequest bool, directoryMetadata *directoryMetadata, ranges []ByteRange, begin time.Time, logger *zap.SugaredLogger) bool { +func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, contentPath path.Path, isHeadRequest bool, directoryMetadata *directoryMetadata, ranges []ByteRange, begin time.Time, logger *zap.SugaredLogger) bool { ctx, span := spanTrace(ctx, "Handler.ServeDirectory", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() @@ -60,16 +59,19 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } // Check if directory has index.html, if so, serveFile - appendIndexHtml := func(p ipath.Path) ipath.Path { - basePath := p.String() - if basePath[len(basePath)-1] != '/' { - basePath += "/" - } - return ipath.New(basePath + "index.html") + idxPath, err := path.Join(contentPath, "index.html") + if err != nil { + i.webError(w, r, err, http.StatusInternalServerError) + return false + } + + indexPath, err := path.Join(resolvedPath, "index.html") + if err != nil { + i.webError(w, r, err, http.StatusInternalServerError) + return false } - idxPath := appendIndexHtml(contentPath) - imIndexPath, err := NewImmutablePath(appendIndexHtml(resolvedPath)) + imIndexPath, err := path.NewImmutablePath(indexPath) if err != nil { i.webError(w, r, err, http.StatusInternalServerError) return false @@ -132,7 +134,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * w.Header().Set("Content-Type", "text/html") // Generated dir index requires custom Etag (output may change between go-libipfs versions) - dirEtag := getDirListingEtag(resolvedPath.Cid()) + dirEtag := getDirListingEtag(resolvedPath.RootCid()) w.Header().Set("Etag", dirEtag) if r.Method == http.MethodHead { @@ -167,7 +169,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * backLink := originalURLPath // don't go further up than /ipfs/$hash/ - pathSplit := path.SplitList(contentPath.String()) + pathSplit := strings.Split(contentPath.String(), "/") switch { // skip backlink when listing a content root case len(pathSplit) == 3: // url: /ipfs/$hash @@ -187,7 +189,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } size := humanize.Bytes(directoryMetadata.dagSize) - hash := resolvedPath.Cid().String() + hash := resolvedPath.RootCid().String() globalData := i.getTemplateGlobalData(r, contentPath) // See comment above where originalUrlPath is declared. diff --git a/gateway/handler_unixfs_dir_test.go b/gateway/handler_unixfs_dir_test.go index a8ce04778..48f2625d6 100644 --- a/gateway/handler_unixfs_dir_test.go +++ b/gateway/handler_unixfs_dir_test.go @@ -6,8 +6,7 @@ import ( "net/http" "testing" - ipath "github.com/ipfs/boxo/coreiface/path" - path "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" "github.com/stretchr/testify/require" ) @@ -19,10 +18,14 @@ func TestIPNSHostnameBacklinks(t *testing.T) { defer cancel() // create /ipns/example.net/foo/ - k2, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), "foo? #<'")) + p2, err := path.Join(path.FromCid(root), "foo? #<'") + require.NoError(t, err) + k2, err := backend.resolvePathNoRootsReturned(ctx, p2) require.NoError(t, err) - k3, err := backend.resolvePathNoRootsReturned(ctx, ipath.Join(ipath.IpfsPath(root), "foo? #<'/bar")) + p3, err := path.Join(path.FromCid(root), "foo? #<'/bar") + require.NoError(t, err) + k3, err := backend.resolvePathNoRootsReturned(ctx, p3) require.NoError(t, err) backend.namesys["/ipns/example.net"] = path.FromCid(root) @@ -44,7 +47,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { require.Contains(t, s, "", "expected backlink in directory listing") require.Contains(t, s, "", "expected file in directory listing") - require.Contains(t, s, s, k2.Cid().String(), "expected hash in directory listing") + require.Contains(t, s, s, k2.RootCid().String(), "expected hash in directory listing") // make request to directory listing at root req = mustNewRequest(t, http.MethodGet, ts.URL, nil) @@ -83,5 +86,5 @@ func TestIPNSHostnameBacklinks(t *testing.T) { require.True(t, matchPathOrBreadcrumbs(s, "/ipns/example.net/foo? #<'/bar"), "expected a path in directory listing") require.Contains(t, s, "", "expected backlink in directory listing") require.Contains(t, s, "", "expected file in directory listing") - require.Contains(t, s, k3.Cid().String(), "expected hash in directory listing") + require.Contains(t, s, k3.RootCid().String(), "expected hash in directory listing") } diff --git a/gateway/handler_unixfs_file.go b/gateway/handler_unixfs_file.go index 8375bfcd2..fb1f9940c 100644 --- a/gateway/handler_unixfs_file.go +++ b/gateway/handler_unixfs_file.go @@ -12,19 +12,19 @@ import ( "time" "github.com/gabriel-vasile/mimetype" - ipath "github.com/ipfs/boxo/coreiface/path" + "github.com/ipfs/boxo/path" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" ) // serveFile returns data behind a file along with HTTP headers based on // the file itself, its CID and the contentPath used for accessing it. -func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, fileSize int64, fileBytes io.ReadCloser, isSymlink bool, returnRangeStartsAtZero bool, fileContentType string, begin time.Time) bool { +func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, contentPath path.Path, fileSize int64, fileBytes io.ReadCloser, isSymlink bool, returnRangeStartsAtZero bool, fileContentType string, begin time.Time) bool { _, span := spanTrace(ctx, "Handler.ServeFile", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() // Set Cache-Control and read optional Last-Modified time - modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.Cid(), "") + modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.RootCid(), "") // Set Content-Disposition name := addContentDispositionHeader(w, r, contentPath) diff --git a/gateway/hostname_test.go b/gateway/hostname_test.go index f7cee35a2..3a6809555 100644 --- a/gateway/hostname_test.go +++ b/gateway/hostname_test.go @@ -7,7 +7,7 @@ import ( "net/http/httptest" "testing" - path "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" cid "github.com/ipfs/go-cid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -20,8 +20,8 @@ func TestToSubdomainURL(t *testing.T) { testCID, err := cid.Decode("bafkqaglimvwgy3zakrsxg5cun5jxkyten5wwc2lokvjeycq") require.NoError(t, err) - backend.namesys["/ipns/dnslink.long-name.example.com"] = path.FromString(testCID.String()) - backend.namesys["/ipns/dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com"] = path.FromString(testCID.String()) + backend.namesys["/ipns/dnslink.long-name.example.com"] = path.FromCid(testCID) + backend.namesys["/ipns/dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com"] = path.FromCid(testCID) httpRequest := httptest.NewRequest("GET", "http://127.0.0.1:8080", nil) httpsRequest := httptest.NewRequest("GET", "https://https-request-stub.example.com", nil) httpsProxiedRequest := httptest.NewRequest("GET", "http://proxied-https-request-stub.example.com", nil) diff --git a/gateway/metrics.go b/gateway/metrics.go index 6035c74b5..bccaefef2 100644 --- a/gateway/metrics.go +++ b/gateway/metrics.go @@ -6,8 +6,8 @@ import ( "io" "time" - "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/files" + "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" prometheus "github.com/prometheus/client_golang/prometheus" "go.opentelemetry.io/otel" @@ -60,7 +60,7 @@ func (b *ipfsBackendWithMetrics) updateBackendCallMetric(name string, err error, } } -func (b *ipfsBackendWithMetrics) Get(ctx context.Context, path ImmutablePath, ranges ...ByteRange) (ContentPathMetadata, *GetResponse, error) { +func (b *ipfsBackendWithMetrics) Get(ctx context.Context, path path.ImmutablePath, ranges ...ByteRange) (ContentPathMetadata, *GetResponse, error) { begin := time.Now() name := "IPFSBackend.Get" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("path", path.String()), attribute.Int("ranges", len(ranges)))) @@ -72,7 +72,7 @@ func (b *ipfsBackendWithMetrics) Get(ctx context.Context, path ImmutablePath, ra return md, f, err } -func (b *ipfsBackendWithMetrics) GetAll(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.Node, error) { +func (b *ipfsBackendWithMetrics) GetAll(ctx context.Context, path path.ImmutablePath) (ContentPathMetadata, files.Node, error) { begin := time.Now() name := "IPFSBackend.GetAll" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("path", path.String()))) @@ -84,7 +84,7 @@ func (b *ipfsBackendWithMetrics) GetAll(ctx context.Context, path ImmutablePath) return md, n, err } -func (b *ipfsBackendWithMetrics) GetBlock(ctx context.Context, path ImmutablePath) (ContentPathMetadata, files.File, error) { +func (b *ipfsBackendWithMetrics) GetBlock(ctx context.Context, path path.ImmutablePath) (ContentPathMetadata, files.File, error) { begin := time.Now() name := "IPFSBackend.GetBlock" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("path", path.String()))) @@ -96,7 +96,7 @@ func (b *ipfsBackendWithMetrics) GetBlock(ctx context.Context, path ImmutablePat return md, n, err } -func (b *ipfsBackendWithMetrics) Head(ctx context.Context, path ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { +func (b *ipfsBackendWithMetrics) Head(ctx context.Context, path path.ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { begin := time.Now() name := "IPFSBackend.Head" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("path", path.String()))) @@ -108,7 +108,7 @@ func (b *ipfsBackendWithMetrics) Head(ctx context.Context, path ImmutablePath) ( return md, n, err } -func (b *ipfsBackendWithMetrics) ResolvePath(ctx context.Context, path ImmutablePath) (ContentPathMetadata, error) { +func (b *ipfsBackendWithMetrics) ResolvePath(ctx context.Context, path path.ImmutablePath) (ContentPathMetadata, error) { begin := time.Now() name := "IPFSBackend.ResolvePath" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("path", path.String()))) @@ -120,7 +120,7 @@ func (b *ipfsBackendWithMetrics) ResolvePath(ctx context.Context, path Immutable return md, err } -func (b *ipfsBackendWithMetrics) GetCAR(ctx context.Context, path ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { +func (b *ipfsBackendWithMetrics) GetCAR(ctx context.Context, path path.ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { begin := time.Now() name := "IPFSBackend.GetCAR" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("path", path.String()))) @@ -155,7 +155,7 @@ func (b *ipfsBackendWithMetrics) GetIPNSRecord(ctx context.Context, cid cid.Cid) return r, err } -func (b *ipfsBackendWithMetrics) ResolveMutable(ctx context.Context, path path.Path) (ImmutablePath, error) { +func (b *ipfsBackendWithMetrics) ResolveMutable(ctx context.Context, path path.Path) (path.ImmutablePath, error) { begin := time.Now() name := "IPFSBackend.ResolveMutable" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("path", path.String()))) diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index 9d58d8c11..89f955cbc 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -14,11 +14,10 @@ import ( "github.com/ipfs/boxo/blockservice" nsopts "github.com/ipfs/boxo/coreiface/options/namesys" - ipath "github.com/ipfs/boxo/coreiface/path" offline "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/namesys" - path "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" carblockstore "github.com/ipld/go-car/v2/blockstore" "github.com/libp2p/go-libp2p/core/crypto" @@ -73,7 +72,7 @@ func (m mockNamesys) Resolve(ctx context.Context, name string, opts ...nsopts.Re var ok bool value, ok = m[name] if !ok { - return "", namesys.ErrResolveFailed + return nil, namesys.ErrResolveFailed } name = value.String() } @@ -133,27 +132,27 @@ func newMockBackend(t *testing.T, fixturesFile string) (*mockBackend, cid.Cid) { }, cids[0] } -func (mb *mockBackend) Get(ctx context.Context, immutablePath ImmutablePath, ranges ...ByteRange) (ContentPathMetadata, *GetResponse, error) { +func (mb *mockBackend) Get(ctx context.Context, immutablePath path.ImmutablePath, ranges ...ByteRange) (ContentPathMetadata, *GetResponse, error) { return mb.gw.Get(ctx, immutablePath, ranges...) } -func (mb *mockBackend) GetAll(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.Node, error) { +func (mb *mockBackend) GetAll(ctx context.Context, immutablePath path.ImmutablePath) (ContentPathMetadata, files.Node, error) { return mb.gw.GetAll(ctx, immutablePath) } -func (mb *mockBackend) GetBlock(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, files.File, error) { +func (mb *mockBackend) GetBlock(ctx context.Context, immutablePath path.ImmutablePath) (ContentPathMetadata, files.File, error) { return mb.gw.GetBlock(ctx, immutablePath) } -func (mb *mockBackend) Head(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { +func (mb *mockBackend) Head(ctx context.Context, immutablePath path.ImmutablePath) (ContentPathMetadata, *HeadResponse, error) { return mb.gw.Head(ctx, immutablePath) } -func (mb *mockBackend) GetCAR(ctx context.Context, immutablePath ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { +func (mb *mockBackend) GetCAR(ctx context.Context, immutablePath path.ImmutablePath, params CarParams) (ContentPathMetadata, io.ReadCloser, error) { return mb.gw.GetCAR(ctx, immutablePath, params) } -func (mb *mockBackend) ResolveMutable(ctx context.Context, p ipath.Path) (ImmutablePath, error) { +func (mb *mockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { return mb.gw.ResolveMutable(ctx, p) } @@ -161,28 +160,28 @@ func (mb *mockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, er return nil, routing.ErrNotSupported } -func (mb *mockBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (ipath.Path, error) { +func (mb *mockBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) { if mb.namesys != nil { p, err := mb.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) if err == namesys.ErrResolveRecursion { err = nil } - return ipath.New(p.String()), err + return p, err } return nil, errors.New("not implemented") } -func (mb *mockBackend) IsCached(ctx context.Context, p ipath.Path) bool { +func (mb *mockBackend) IsCached(ctx context.Context, p path.Path) bool { return mb.gw.IsCached(ctx, p) } -func (mb *mockBackend) ResolvePath(ctx context.Context, immutablePath ImmutablePath) (ContentPathMetadata, error) { +func (mb *mockBackend) ResolvePath(ctx context.Context, immutablePath path.ImmutablePath) (ContentPathMetadata, error) { return mb.gw.ResolvePath(ctx, immutablePath) } -func (mb *mockBackend) resolvePathNoRootsReturned(ctx context.Context, ip ipath.Path) (ipath.Resolved, error) { - var imPath ImmutablePath +func (mb *mockBackend) resolvePathNoRootsReturned(ctx context.Context, ip path.Path) (path.ImmutablePath, error) { + var imPath path.ImmutablePath var err error if ip.Mutable() { imPath, err = mb.ResolveMutable(ctx, ip) @@ -190,7 +189,7 @@ func (mb *mockBackend) resolvePathNoRootsReturned(ctx context.Context, ip ipath. return nil, err } } else { - imPath, err = NewImmutablePath(ip) + imPath, err = path.NewImmutablePath(ip) if err != nil { return nil, err } diff --git a/ipns/README.md b/ipns/README.md index 502f6dded..28989e0c9 100644 --- a/ipns/README.md +++ b/ipns/README.md @@ -30,7 +30,10 @@ func main() { } // Define the path this record will point to. - path := path.FromString("/ipfs/bafkqac3jobxhgidsn5rww4yk") + path, err := path.NewPath("/ipfs/bafkqac3jobxhgidsn5rww4yk") + if err != nil { + panic(err) + } // Until when the record is valid. eol := time.Now().Add(time.Hour) diff --git a/ipns/name.go b/ipns/name.go index 2a6bbdbf4..b35e8f9d8 100644 --- a/ipns/name.go +++ b/ipns/name.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" + "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" mb "github.com/multiformats/go-multibase" @@ -133,3 +134,12 @@ func (n Name) MarshalJSON() ([]byte, error) { func (n Name) Equal(other Name) bool { return bytes.Equal(n.src, other.src) } + +// AsPath returns the IPNS Name as a [path.Path] prefixed by [path.IPNSNamespace]. +func (n Name) AsPath() path.Path { + p, err := path.NewPathFromSegments(path.IPNSNamespace, n.String()) + if err != nil { + panic(fmt.Errorf("path.NewPathFromSegments was called with invalid parameters: %w", err)) + } + return p +} diff --git a/ipns/name_test.go b/ipns/name_test.go index 4b8ccd414..c3c31878c 100644 --- a/ipns/name_test.go +++ b/ipns/name_test.go @@ -34,6 +34,16 @@ func TestName(t *testing.T) { }) } + testPath := func(t *testing.T, name, input, expected string) { + t.Run("AsPath method: "+name, func(t *testing.T) { + t.Parallel() + + name, err := NameFromString(input) + require.NoError(t, err) + require.Equal(t, expected, name.AsPath().String()) + }) + } + testMarshalJSON := func(t *testing.T, name, input, expected string) { t.Run("Marshal JSON: "+name, func(t *testing.T) { t.Parallel() @@ -66,6 +76,7 @@ func TestName(t *testing.T) { testFromCid(t, v[0], v[2], v[2]) testString(t, v[0], v[1], v[2]) testString(t, v[0], NamespacePrefix+v[1], v[2]) + testPath(t, v[0], v[1], NamespacePrefix+v[2]) testMarshalJSON(t, v[0], v[1], `"`+v[2]+`"`) testMarshalJSON(t, v[0], NamespacePrefix+v[1], `"`+v[2]+`"`) testUnmarshalJSON(t, v[0], []byte(`"`+v[2]+`"`), v[2]) diff --git a/ipns/record.go b/ipns/record.go index cdb901af0..ddb57ac42 100644 --- a/ipns/record.go +++ b/ipns/record.go @@ -85,12 +85,12 @@ func MarshalRecord(rec *Record) ([]byte, error) { func (rec *Record) Value() (path.Path, error) { value, err := rec.getBytesValue(cborValueKey) if err != nil { - return "", err + return nil, err } - p := path.FromString(string(value)) - if err := p.IsValid(); err != nil { - return "", multierr.Combine(ErrInvalidPath, err) + p, err := path.NewPath(string(value)) + if err != nil { + return nil, multierr.Combine(ErrInvalidPath, err) } return p, nil @@ -259,7 +259,7 @@ func NewRecord(sk ic.PrivKey, value path.Path, seq uint64, eol time.Time, ttl ti } if options.v1Compatibility { - pb.Value = []byte(value) + pb.Value = []byte(value.String()) typ := ipns_pb.IpnsRecord_EOL pb.ValidityType = &typ pb.Sequence = &seq @@ -306,7 +306,7 @@ func createNode(value path.Path, seq uint64, eol time.Time, ttl time.Duration) ( m := make(map[string]ipld.Node) var keys []string - m[cborValueKey] = basicnode.NewBytes([]byte(value)) + m[cborValueKey] = basicnode.NewBytes([]byte(value.String())) keys = append(keys, cborValueKey) m[cborValidityKey] = basicnode.NewBytes([]byte(util.FormatRFC3339(eol))) diff --git a/ipns/record_test.go b/ipns/record_test.go index d761ecfc5..db92a6a17 100644 --- a/ipns/record_test.go +++ b/ipns/record_test.go @@ -17,10 +17,18 @@ import ( "google.golang.org/protobuf/proto" ) -const ( - testPath = path.Path("/ipfs/bafkqac3jobxhgidsn5rww4yk") +var ( + testPath path.Path ) +func init() { + var err error + testPath, err = path.NewPath("/ipfs/bafkqac3jobxhgidsn5rww4yk") + if err != nil { + panic(err) + } +} + func mustKeyPair(t *testing.T, typ int) (ic.PrivKey, ic.PubKey, Name) { sr := util.NewTimeSeededRand() sk, pk, err := ic.GenerateKeyPairWithReader(typ, 2048, sr) @@ -195,14 +203,15 @@ func TestCBORDataSerialization(t *testing.T) { sk, _, _ := mustKeyPair(t, ic.Ed25519) eol := time.Now().Add(time.Hour) - path := path.FromString(string(append([]byte("/path/1"), 0x00))) + path, err := path.Join(testPath, string([]byte{0x00})) + require.NoError(t, err) seq := uint64(1) ttl := time.Hour rec := mustNewRecord(t, sk, path, seq, eol, ttl) builder := basicnode.Prototype__Map{}.NewBuilder() - err := dagcbor.Decode(builder, bytes.NewReader(rec.pb.GetData())) + err = dagcbor.Decode(builder, bytes.NewReader(rec.pb.GetData())) require.NoError(t, err) node := builder.Build() @@ -218,7 +227,7 @@ func TestCBORDataSerialization(t *testing.T) { case cborValueKey: b, err := v.AsBytes() require.NoError(t, err) - require.Equal(t, b, []byte(path)) + require.Equal(t, b, []byte(path.String())) case cborSequenceKey: s, err := v.AsInt() require.NoError(t, err) diff --git a/ipns/validation_test.go b/ipns/validation_test.go index 126c357e7..8195fde23 100644 --- a/ipns/validation_test.go +++ b/ipns/validation_test.go @@ -164,8 +164,14 @@ func TestValidate(t *testing.T) { v := Validator{} - rec1 := mustNewRecord(t, sk, path.FromString("/path/1"), 1, eol, 0, WithV1Compatibility(true)) - rec2 := mustNewRecord(t, sk, path.FromString("/path/2"), 2, eol, 0, WithV1Compatibility(true)) + path1, err := path.Join(testPath, "1") + require.NoError(t, err) + + path2, err := path.Join(testPath, "2") + require.NoError(t, err) + + rec1 := mustNewRecord(t, sk, path1, 1, eol, 0, WithV1Compatibility(true)) + rec2 := mustNewRecord(t, sk, path2, 2, eol, 0, WithV1Compatibility(true)) best, err := v.Select(ipnsRoutingKey, [][]byte{mustMarshal(t, rec1), mustMarshal(t, rec2)}) require.NoError(t, err) @@ -210,8 +216,10 @@ func TestValidate(t *testing.T) { sk, pk, _ := mustKeyPair(t, ic.RSA) // Create a record that is too large (value + other fields). - value := make([]byte, MaxRecordSize) - rec, err := NewRecord(sk, path.FromString(string(value)), 1, eol, 0) + path, err := path.Join(testPath, string(make([]byte, MaxRecordSize))) + require.NoError(t, err) + + rec, err := NewRecord(sk, path, 1, eol, 0) require.NoError(t, err) err = Validate(rec, pk) diff --git a/mfs/mfs_test.go b/mfs/mfs_test.go index 9ecdbffd5..2bdf82994 100644 --- a/mfs/mfs_test.go +++ b/mfs/mfs_test.go @@ -12,6 +12,7 @@ import ( "os" gopath "path" "sort" + "strings" "sync" "testing" "time" @@ -24,7 +25,6 @@ import ( ft "github.com/ipfs/boxo/ipld/unixfs" importer "github.com/ipfs/boxo/ipld/unixfs/importer" uio "github.com/ipfs/boxo/ipld/unixfs/io" - path "github.com/ipfs/boxo/path" u "github.com/ipfs/boxo/util" cid "github.com/ipfs/go-cid" @@ -59,7 +59,7 @@ func fileNodeFromReader(t *testing.T, ds ipld.DAGService, r io.Reader) ipld.Node } func mkdirP(t *testing.T, root *Directory, pth string) *Directory { - dirs := path.SplitList(pth) + dirs := strings.Split(pth, "/") cur := root for _, d := range dirs { n, err := cur.Mkdir(d) @@ -145,7 +145,7 @@ func assertFileAtPath(ds ipld.DAGService, root *Directory, expn ipld.Node, pth s return dag.ErrNotProtobuf } - parts := path.SplitList(pth) + parts := strings.Split(pth, "/") cur := root for i, d := range parts[:len(parts)-1] { next, err := cur.Child(d) diff --git a/mfs/ops.go b/mfs/ops.go index 78156dd52..b3d9f2a5e 100644 --- a/mfs/ops.go +++ b/mfs/ops.go @@ -7,8 +7,6 @@ import ( gopath "path" "strings" - path "github.com/ipfs/boxo/path" - cid "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" ) @@ -131,7 +129,7 @@ func Mkdir(r *Root, pth string, opts MkdirOpts) error { if pth == "" { return fmt.Errorf("no path given to Mkdir") } - parts := path.SplitList(pth) + parts := strings.Split(pth, "/") if parts[0] == "" { parts = parts[1:] } @@ -167,7 +165,7 @@ func Mkdir(r *Root, pth string, opts MkdirOpts) error { next, ok := fsn.(*Directory) if !ok { - return fmt.Errorf("%s was not a directory", path.Join(parts[:i])) + return fmt.Errorf("%s was not a directory", strings.Join(parts[:i], "/")) } cur = next } @@ -205,7 +203,7 @@ func Lookup(r *Root, path string) (FSNode, error) { // under the directory 'd' func DirLookup(d *Directory, pth string) (FSNode, error) { pth = strings.Trim(pth, "/") - parts := path.SplitList(pth) + parts := strings.Split(pth, "/") if len(parts) == 1 && parts[0] == "" { return d, nil } @@ -215,7 +213,7 @@ func DirLookup(d *Directory, pth string) (FSNode, error) { for i, p := range parts { chdir, ok := cur.(*Directory) if !ok { - return nil, fmt.Errorf("cannot access %s: Not a directory", path.Join(parts[:i+1])) + return nil, fmt.Errorf("cannot access %s: Not a directory", strings.Join(parts[:i+1], "/")) } child, err := chdir.Child(p) diff --git a/namesys/base.go b/namesys/base.go index 06b24bedc..6b8e41ab5 100644 --- a/namesys/base.go +++ b/namesys/base.go @@ -6,7 +6,7 @@ import ( "time" opts "github.com/ipfs/boxo/coreiface/options/namesys" - path "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" ) type onceResult struct { diff --git a/namesys/cache.go b/namesys/cache.go index 8b7f50794..51fe3149b 100644 --- a/namesys/cache.go +++ b/namesys/cache.go @@ -3,7 +3,7 @@ package namesys import ( "time" - path "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" ) func (ns *mpns) cacheGet(name string) (path.Path, bool) { @@ -16,12 +16,12 @@ func (ns *mpns) cacheGet(name string) (path.Path, bool) { } if ns.cache == nil { - return "", false + return nil, false } ientry, ok := ns.cache.Get(name) if !ok { - return "", false + return nil, false } entry, ok := ientry.(cacheEntry) @@ -36,7 +36,7 @@ func (ns *mpns) cacheGet(name string) (path.Path, bool) { ns.cache.Remove(name) - return "", false + return nil, false } func (ns *mpns) cacheSet(name string, val path.Path, ttl time.Duration) { diff --git a/namesys/dns.go b/namesys/dns.go index 6f846fcda..48becfc1c 100644 --- a/namesys/dns.go +++ b/namesys/dns.go @@ -5,11 +5,12 @@ import ( "errors" "fmt" "net" - gpath "path" + gopath "path" "strings" opts "github.com/ipfs/boxo/coreiface/options/namesys" - path "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" + "github.com/ipfs/go-cid" dns "github.com/miekg/dns" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -84,7 +85,7 @@ func (r *DNSResolver) resolveOnceAsync(ctx context.Context, name string, options appendPath := func(p path.Path) (path.Path, error) { if len(segments) > 1 { - return path.FromSegments("", strings.TrimRight(p.String(), "/"), segments[1]) + return path.Join(p, segments[1]) } return p, nil } @@ -133,7 +134,7 @@ func (r *DNSResolver) resolveOnceAsync(ctx context.Context, name string, options // dnslink, then output a more specific error message if rootResErr == ErrResolveFailed && subResErr == ErrResolveFailed { // Wrap error so that it can be tested if it is a ErrResolveFailed - err := fmt.Errorf("%w: _dnslink subdomain at %q is missing a TXT record (https://docs.ipfs.tech/concepts/dnslink/)", ErrResolveFailed, gpath.Base(name)) + err := fmt.Errorf("%w: _dnslink subdomain at %q is missing a TXT record (https://docs.ipfs.tech/concepts/dnslink/)", ErrResolveFailed, gopath.Base(name)) emitOnceResult(ctx, out, onceResult{err: err}) } return @@ -160,7 +161,7 @@ func workDomain(ctx context.Context, r *DNSResolver, name string, res chan looku } } // Could not look up any text records for name - res <- lookupRes{"", err} + res <- lookupRes{nil, err} return } @@ -173,23 +174,36 @@ func workDomain(ctx context.Context, r *DNSResolver, name string, res chan looku } // There were no TXT records with a dnslink - res <- lookupRes{"", ErrResolveFailed} + res <- lookupRes{nil, ErrResolveFailed} } func parseEntry(txt string) (path.Path, error) { - p, err := path.ParseCidToPath(txt) // bare IPFS multihashes + p, err := path.NewPath(txt) // bare IPFS multihashes if err == nil { return p, nil } + // Support legacy DNSLink entries composed by the CID only. + if cid, err := cid.Decode(txt); err == nil { + return path.FromCid(cid), nil + } + return tryParseDNSLink(txt) } func tryParseDNSLink(txt string) (path.Path, error) { parts := strings.SplitN(txt, "=", 2) if len(parts) == 2 && parts[0] == "dnslink" { - return path.ParsePath(parts[1]) + p, err := path.NewPath(parts[1]) + if err == nil { + return p, nil + } + + // Support legacy DNSLink entries composed by "dnslink={CID}". + if cid, err := cid.Decode(parts[1]); err == nil { + return path.FromCid(cid), nil + } } - return "", errors.New("not a valid dnslink entry") + return nil, errors.New("not a valid dnslink entry") } diff --git a/namesys/namesys.go b/namesys/namesys.go index df4403570..381e5bff1 100644 --- a/namesys/namesys.go +++ b/namesys/namesys.go @@ -100,7 +100,10 @@ func NewNameSystem(r routing.ValueStore, opts ...Option) (NameSystem, error) { for _, pair := range strings.Split(list, ",") { mapping := strings.SplitN(pair, ":", 2) key := mapping[0] - value := path.FromString(mapping[1]) + value, err := path.NewPath(mapping[1]) + if err != nil { + return nil, err + } staticMap[key] = value } } @@ -139,11 +142,11 @@ func (ns *mpns) Resolve(ctx context.Context, name string, options ...opts.Resolv defer span.End() if strings.HasPrefix(name, "/ipfs/") { - return path.ParsePath(name) + return path.NewPath(name) } if !strings.HasPrefix(name, "/") { - return path.ParsePath("/ipfs/" + name) + return path.NewPath("/ipfs/" + name) } return resolve(ctx, ns, name, opts.ProcessOpts(options)) @@ -154,7 +157,7 @@ func (ns *mpns) ResolveAsync(ctx context.Context, name string, options ...opts.R defer span.End() if strings.HasPrefix(name, "/ipfs/") { - p, err := path.ParsePath(name) + p, err := path.NewPath(name) res := make(chan Result, 1) res <- Result{p, err} close(res) @@ -162,7 +165,7 @@ func (ns *mpns) ResolveAsync(ctx context.Context, name string, options ...opts.R } if !strings.HasPrefix(name, "/") { - p, err := path.ParsePath("/ipfs/" + name) + p, err := path.NewPath("/ipfs/" + name) res := make(chan Result, 1) res <- Result{p, err} close(res) @@ -220,7 +223,7 @@ func (ns *mpns) resolveOnceAsync(ctx context.Context, name string, options opts. if p, ok := ns.cacheGet(cacheKey); ok { var err error if len(segments) > 3 { - p, err = path.FromSegments("", strings.TrimRight(p.String(), "/"), segments[3]) + p, err = path.Join(p, segments[3]) } span.SetAttributes(attribute.Bool("CacheHit", true)) span.RecordError(err) @@ -263,7 +266,7 @@ func (ns *mpns) resolveOnceAsync(ctx context.Context, name string, options opts. // Attach rest of the path if len(segments) > 3 { - p, err = path.FromSegments("", strings.TrimRight(p.String(), "/"), segments[3]) + p, err = path.Join(p, segments[3]) } emitOnceResult(ctx, out, onceResult{value: p, ttl: ttl, err: err}) diff --git a/namesys/namesys_test.go b/namesys/namesys_test.go index 52fce6794..9e1d9f5f6 100644 --- a/namesys/namesys_test.go +++ b/namesys/namesys_test.go @@ -17,6 +17,7 @@ import ( ci "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" + "github.com/stretchr/testify/assert" ) type mockResolver struct { @@ -31,15 +32,15 @@ func testResolution(t *testing.T, resolver Resolver, name string, depth uint, ex "expected %s with a depth of %d to have a '%s' error, but got '%s'", name, depth, expError, err)) } - if p.String() != expected { - t.Fatal(fmt.Errorf( - "%s with depth %d resolved to %s != %s", - name, depth, p.String(), expected)) + if expected == "" { + assert.Nil(t, p, "%s with depth %d", name, depth) + } else { + assert.Equal(t, p.String(), expected, "%s with depth %d", name, depth) } } func (r *mockResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult { - p, err := path.ParsePath(r.entries[name]) + p, err := path.NewPath(r.entries[name]) out := make(chan onceResult, 1) out <- onceResult{value: p, err: err} close(out) @@ -118,7 +119,7 @@ func TestPublishWithCache0(t *testing.T) { } // CID is arbitrary. - p, err := path.ParsePath("QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") + p, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") if err != nil { t.Fatal(err) } @@ -158,7 +159,7 @@ func TestPublishWithTTL(t *testing.T) { } // CID is arbitrary. - p, err := path.ParsePath("QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") + p, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") if err != nil { t.Fatal(err) } diff --git a/namesys/publisher.go b/namesys/publisher.go index c913b0bbc..9cb3ae66a 100644 --- a/namesys/publisher.go +++ b/namesys/publisher.go @@ -168,7 +168,7 @@ func (p *IpnsPublisher) updateRecord(ctx context.Context, k crypto.PrivKey, valu if err != nil { return nil, err } - if value != path.Path(p.String()) { + if value.String() != p.String() { // Don't bother incrementing the sequence number unless the // value changes. seqno++ diff --git a/namesys/publisher_test.go b/namesys/publisher_test.go index ad975f59a..536e72771 100644 --- a/namesys/publisher_test.go +++ b/namesys/publisher_test.go @@ -56,7 +56,10 @@ func testNamekeyPublisher(t *testing.T, keyType int, expectedErr error, expected } // Value - value := path.Path("ipfs/TESTING") + value, err := path.NewPath("/ipfs/bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") + if err != nil { + t.Fatal(err) + } // Seqnum seqnum := uint64(0) @@ -125,7 +128,7 @@ func TestAsyncDS(t *testing.T) { publisher := NewIpnsPublisher(rt, ds) ipnsFakeID := testutil.RandIdentityOrFatal(t) - ipnsVal, err := path.ParsePath("/ipns/foo.bar") + ipnsVal, err := path.NewPath("/ipns/foo.bar") if err != nil { t.Fatal(err) } diff --git a/namesys/republisher/repub.go b/namesys/republisher/repub.go index 87200ff5c..bb7a5e2b0 100644 --- a/namesys/republisher/repub.go +++ b/namesys/republisher/repub.go @@ -9,7 +9,6 @@ import ( keystore "github.com/ipfs/boxo/keystore" "github.com/ipfs/boxo/namesys" - "github.com/ipfs/boxo/path" "go.opentelemetry.io/otel/attribute" opts "github.com/ipfs/boxo/coreiface/options/namesys" @@ -165,7 +164,7 @@ func (rp *Republisher) republishEntry(ctx context.Context, priv ic.PrivKey) erro if prevEol.After(eol) { eol = prevEol } - err = rp.ns.Publish(ctx, priv, path.Path(p.String()), opts.PublishWithEOL(eol)) + err = rp.ns.Publish(ctx, priv, p, opts.PublishWithEOL(eol)) span.RecordError(err) return err } diff --git a/namesys/republisher/repub_test.go b/namesys/republisher/repub_test.go index d6c7b0d85..6b5d2abf0 100644 --- a/namesys/republisher/repub_test.go +++ b/namesys/republisher/repub_test.go @@ -91,7 +91,11 @@ func TestRepublish(t *testing.T) { // have one node publish a record that is valid for 1 second publisher := nodes[3] - p := path.FromString("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid + p, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid + if err != nil { + t.Fatal(err) + } + rp := namesys.NewIpnsPublisher(publisher.dht, publisher.store) name := "/ipns/" + publisher.id @@ -172,12 +176,16 @@ func TestLongEOLRepublish(t *testing.T) { // have one node publish a record that is valid for 1 second publisher := nodes[3] - p := path.FromString("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid + p, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid + if err != nil { + t.Fatal(err) + } + rp := namesys.NewIpnsPublisher(publisher.dht, publisher.store) name := "/ipns/" + publisher.id expiration := time.Now().Add(time.Hour) - err := rp.Publish(ctx, publisher.privKey, p, opts.PublishWithEOL(expiration)) + err = rp.Publish(ctx, publisher.privKey, p, opts.PublishWithEOL(expiration)) if err != nil { t.Fatal(err) } @@ -239,7 +247,7 @@ func verifyResolution(nsystems []namesys.NameSystem, key string, exp path.Path) return err } - if val != exp { + if val.String() != exp.String() { return errors.New("resolved wrong record") } } diff --git a/namesys/resolve/resolve.go b/namesys/resolve/resolve.go index b2acf0602..b01fd38f7 100644 --- a/namesys/resolve/resolve.go +++ b/namesys/resolve/resolve.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "strings" "github.com/ipfs/boxo/path" "go.opentelemetry.io/otel/attribute" @@ -22,34 +21,35 @@ var ErrNoNamesys = errors.New( func ResolveIPNS(ctx context.Context, nsys namesys.NameSystem, p path.Path) (path.Path, error) { ctx, span := namesys.StartSpan(ctx, "ResolveIPNS", trace.WithAttributes(attribute.String("Path", p.String()))) defer span.End() - if strings.HasPrefix(p.String(), "/ipns/") { + + if p.Namespace() == path.IPNSNamespace { // TODO(cryptix): we should be able to query the local cache for the path if nsys == nil { - return "", ErrNoNamesys + return nil, ErrNoNamesys } seg := p.Segments() if len(seg) < 2 || seg[1] == "" { // just "/" without further segments err := fmt.Errorf("invalid path %q: ipns path missing IPNS ID", p) - return "", err + return nil, err } extensions := seg[2:] - resolvable, err := path.FromSegments("/", seg[0], seg[1]) + resolvable, err := path.NewPathFromSegments(seg[0], seg[1]) if err != nil { - return "", err + return nil, err } respath, err := nsys.Resolve(ctx, resolvable.String()) if err != nil { - return "", err + return nil, err } segments := append(respath.Segments(), extensions...) - p, err = path.FromSegments("/", segments...) + p, err = path.NewPathFromSegments(segments...) if err != nil { - return "", err + return nil, err } } return p, nil diff --git a/namesys/resolve_test.go b/namesys/resolve_test.go index 3aecdccaf..158a9a26c 100644 --- a/namesys/resolve_test.go +++ b/namesys/resolve_test.go @@ -7,7 +7,7 @@ import ( "time" ipns "github.com/ipfs/boxo/ipns" - path "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" mockrouting "github.com/ipfs/boxo/routing/mock" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" @@ -25,8 +25,12 @@ func TestRoutingResolve(t *testing.T) { identity := tnet.RandIdentityOrFatal(t) - h := path.FromString("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") - err := publisher.Publish(context.Background(), identity.PrivateKey(), h) + h, err := path.NewPath("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") + if err != nil { + t.Fatal(err) + } + + err = publisher.Publish(context.Background(), identity.PrivateKey(), h) if err != nil { t.Fatal(err) } @@ -36,7 +40,7 @@ func TestRoutingResolve(t *testing.T) { t.Fatal(err) } - if res != h { + if res.String() != h.String() { t.Fatal("Got back incorrect value.") } } @@ -51,7 +55,10 @@ func TestPrexistingExpiredRecord(t *testing.T) { identity := tnet.RandIdentityOrFatal(t) // Make an expired record and put it in the datastore - h := path.FromString("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") + h, err := path.NewPath("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") + if err != nil { + t.Fatal(err) + } eol := time.Now().Add(time.Hour * -1) entry, err := ipns.NewRecord(identity.PrivateKey(), h, 0, eol, 0) @@ -85,7 +92,10 @@ func TestPrexistingRecord(t *testing.T) { identity := tnet.RandIdentityOrFatal(t) // Make a good record and put it in the datastore - h := path.FromString("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") + h, err := path.NewPath("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") + if err != nil { + t.Fatal(err) + } eol := time.Now().Add(time.Hour) entry, err := ipns.NewRecord(identity.PrivateKey(), h, 0, eol, 0) if err != nil { @@ -114,7 +124,7 @@ func verifyCanResolve(r Resolver, name string, exp path.Path) error { return err } - if res != exp { + if res.String() != exp.String() { return errors.New("got back wrong record") } diff --git a/namesys/routing.go b/namesys/routing.go index 6b706bd92..1153341ab 100644 --- a/namesys/routing.go +++ b/namesys/routing.go @@ -136,7 +136,7 @@ func (r *IpnsResolver) resolveOnceAsync(ctx context.Context, name string, option return } - emitOnceResult(ctx, out, onceResult{value: path.Path(p.String()), ttl: ttl}) + emitOnceResult(ctx, out, onceResult{value: p, ttl: ttl}) case <-ctx.Done(): return } diff --git a/path/error.go b/path/error.go index dafc446b5..0863c7cf4 100644 --- a/path/error.go +++ b/path/error.go @@ -1,25 +1,32 @@ package path import ( + "errors" "fmt" ) +var ( + ErrExpectedImmutable = errors.New("path was expected to be immutable") + ErrInsufficientComponents = errors.New("path does not have enough components") + ErrUnknownNamespace = errors.New("unknown namespace") +) + type ErrInvalidPath struct { - error error - path string + err error + path string } -func (e ErrInvalidPath) Error() string { - return fmt.Sprintf("invalid path %q: %s", e.path, e.error) +func (e *ErrInvalidPath) Error() string { + return fmt.Sprintf("invalid path %q: %s", e.path, e.err) } -func (e ErrInvalidPath) Unwrap() error { - return e.error +func (e *ErrInvalidPath) Unwrap() error { + return e.err } -func (e ErrInvalidPath) Is(err error) bool { +func (e *ErrInvalidPath) Is(err error) bool { switch err.(type) { - case ErrInvalidPath: + case *ErrInvalidPath: return true default: return false diff --git a/path/error_test.go b/path/error_test.go index 07aab6408..2b5f92945 100644 --- a/path/error_test.go +++ b/path/error_test.go @@ -6,11 +6,7 @@ import ( ) func TestErrorIs(t *testing.T) { - if !errors.Is(ErrInvalidPath{path: "foo", error: errors.New("bar")}, ErrInvalidPath{}) { - t.Fatal("error must be error") - } - - if !errors.Is(&ErrInvalidPath{path: "foo", error: errors.New("bar")}, ErrInvalidPath{}) { + if !errors.Is(&ErrInvalidPath{path: "foo", err: errors.New("bar")}, &ErrInvalidPath{}) { t.Fatal("pointer to error must be error") } } diff --git a/path/internal/tracing.go b/path/internal/tracing.go deleted file mode 100644 index f9eda2f92..000000000 --- a/path/internal/tracing.go +++ /dev/null @@ -1,13 +0,0 @@ -package internal - -import ( - "context" - "fmt" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" -) - -func StartSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { - return otel.Tracer("go-path").Start(ctx, fmt.Sprintf("Path.%s", name), opts...) -} diff --git a/path/path.go b/path/path.go index a9b36c3ce..5e39eea3a 100644 --- a/path/path.go +++ b/path/path.go @@ -3,187 +3,221 @@ package path import ( "fmt" - "path" + gopath "path" "strings" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" ) -// A Path represents an ipfs content path: -// - /path/to/file -// - /ipfs/ -// - /ipns//path/to/folder -// - etc -type Path string +const ( + IPFSNamespace = "ipfs" + IPNSNamespace = "ipns" + IPLDNamespace = "ipld" +) + +// Path is a generic, valid, and well-formed path. A valid path is shaped as follows: +// +// /{namespace}/{root}[/remaining/path] +// +// Where: +// +// 1. Namespace is "ipfs", "ipld", or "ipns". +// 2. If namespace is "ipfs" or "ipld", "root" must be a valid [cid.Cid]. +// 3. If namespace is "ipns", "root" may be a [ipns.Name] or a [DNSLink] FQDN. +// +// [DNSLink]: https://dnslink.dev/ +type Path interface { + // String returns the path as a string. + String() string + + // Namespace returns the first component of the path. For example, the namespace + // of "/ipfs/bafy" is "ipfs". + Namespace() string + + // Mutable returns false if the data under this path's namespace is guaranteed to not change. + Mutable() bool + + // Segments returns the different elements of a path delimited by a forward + // slash ("/"). The returned array must not contain any empty segments, and + // must have a length of at least two: the first element must be the namespace, + // and the second must be root. + // + // Examples: + // - "/ipld/bafkqaaa" returns ["ipld", "bafkqaaa"] + // - "/ipfs/bafkqaaa/a/b/" returns ["ipfs", "bafkqaaa", "a", "b"] + // - "/ipns/dnslink.net" returns ["ipns", "dnslink.net"] + Segments() []string +} -// ^^^ -// TODO: debate making this a private struct wrapped in a public interface -// would allow us to control creation, and cache segments. +var _ Path = path{} -// FromString safely converts a string type to a Path type. -func FromString(s string) Path { - return Path(s) +type path struct { + str string + namespace string } -// FromCid safely converts a cid.Cid type to a Path type. -func FromCid(c cid.Cid) Path { - return Path("/ipfs/" + c.String()) +func (p path) String() string { + return p.str } -// Segments returns the different elements of a path -// (elements are delimited by a /). -func (p Path) Segments() []string { - cleaned := path.Clean(string(p)) - segments := strings.Split(cleaned, "/") +func (p path) Namespace() string { + return p.namespace +} - // Ignore leading slash - if len(segments[0]) == 0 { - segments = segments[1:] - } +func (p path) Mutable() bool { + return p.Namespace() != IPFSNamespace && p.Namespace() != IPLDNamespace +} - return segments +func (p path) Segments() []string { + return StringToSegments(p.str) } -// String converts a path to string. -func (p Path) String() string { - return string(p) +// ImmutablePath is a [Path] which is guaranteed to have an immutable [Namespace]. +type ImmutablePath interface { + Path + + // RootCid returns the [cid.Cid] of the root object of the path. + RootCid() cid.Cid } -// IsJustAKey returns true if the path is of the form or /ipfs/, or -// /ipld/ -func (p Path) IsJustAKey() bool { - parts := p.Segments() - return len(parts) == 2 && (parts[0] == "ipfs" || parts[0] == "ipld") +var _ Path = immutablePath{} +var _ ImmutablePath = immutablePath{} + +type immutablePath struct { + path Path + rootCid cid.Cid } -// PopLastSegment returns a new Path without its final segment, and the final -// segment, separately. If there is no more to pop (the path is just a key), -// the original path is returned. -func (p Path) PopLastSegment() (Path, string, error) { - if p.IsJustAKey() { - return p, "", nil +func NewImmutablePath(p Path) (ImmutablePath, error) { + if p.Mutable() { + return nil, &ErrInvalidPath{err: ErrExpectedImmutable, path: p.String()} } - segs := p.Segments() - newPath, err := ParsePath("/" + strings.Join(segs[:len(segs)-1], "/")) + segments := p.Segments() + cid, err := cid.Decode(segments[1]) if err != nil { - return "", "", err + return nil, &ErrInvalidPath{err: err, path: p.String()} } - return newPath, segs[len(segs)-1], nil + return immutablePath{path: p, rootCid: cid}, nil } -// FromSegments returns a path given its different segments. -func FromSegments(prefix string, seg ...string) (Path, error) { - return ParsePath(prefix + strings.Join(seg, "/")) +func (ip immutablePath) String() string { + return ip.path.String() } -// ParsePath returns a well-formed ipfs Path. -// The returned path will always be prefixed with /ipfs/ or /ipns/. -// The prefix will be added if not present in the given string. -// This function will return an error when the given string is -// not a valid ipfs path. -func ParsePath(txt string) (Path, error) { - parts := strings.Split(txt, "/") - if len(parts) == 1 { - kp, err := ParseCidToPath(txt) - if err == nil { - return kp, nil - } - } +func (ip immutablePath) Namespace() string { + return ip.path.Namespace() +} - // if the path doesnt begin with a '/' - // we expect this to start with a hash, and be an 'ipfs' path - if parts[0] != "" { - if _, err := decodeCid(parts[0]); err != nil { - return "", &ErrInvalidPath{error: err, path: txt} - } - // The case when the path starts with hash without a protocol prefix - return Path("/ipfs/" + txt), nil - } +func (ip immutablePath) Mutable() bool { + return false +} - if len(parts) < 3 { - return "", &ErrInvalidPath{error: fmt.Errorf("invalid ipfs path"), path: txt} - } +func (ip immutablePath) Segments() []string { + return ip.path.Segments() +} - // TODO: make this smarter - switch parts[1] { - case "ipfs", "ipld": - if parts[2] == "" { - return "", &ErrInvalidPath{error: fmt.Errorf("not enough path components"), path: txt} - } - // Validate Cid. - _, err := decodeCid(parts[2]) - if err != nil { - return "", &ErrInvalidPath{error: fmt.Errorf("invalid CID: %w", err), path: txt} - } - case "ipns": - if parts[2] == "" { - return "", &ErrInvalidPath{error: fmt.Errorf("not enough path components"), path: txt} - } - default: - return "", &ErrInvalidPath{error: fmt.Errorf("unknown namespace %q", parts[1]), path: txt} - } +func (ip immutablePath) RootCid() cid.Cid { + return ip.rootCid +} - return Path(txt), nil +// FromCid returns a new "/ipfs" path with the provided CID. +func FromCid(cid cid.Cid) ImmutablePath { + return immutablePath{ + path: path{ + str: fmt.Sprintf("/%s/%s", IPFSNamespace, cid.String()), + namespace: IPFSNamespace, + }, + rootCid: cid, + } } -// ParseCidToPath takes a CID in string form and returns a valid ipfs Path. -func ParseCidToPath(txt string) (Path, error) { - if txt == "" { - return "", &ErrInvalidPath{error: fmt.Errorf("empty"), path: txt} +// NewPath takes the given string and returns a well-formed and sanitized [Path]. +// The given string is cleaned through [gopath.Clean], but preserving the final +// trailing slash. This function returns an error when the given string is not +// a valid content path. +func NewPath(str string) (Path, error) { + segments := StringToSegments(str) + + // Shortest valid path is "/{namespace}/{root}". That yields at least two + // segments: ["{namespace}" "{root}"]. Therefore, here we check if the original + // string begins with "/" (any path must), if we have at least two segments, and if + // the root is non-empty. The namespace is checked further below. + if !strings.HasPrefix(str, "/") || len(segments) < 2 || segments[1] == "" { + return nil, &ErrInvalidPath{err: ErrInsufficientComponents, path: str} } - c, err := decodeCid(txt) - if err != nil { - return "", &ErrInvalidPath{error: err, path: txt} + cleaned := SegmentsToString(segments...) + if strings.HasSuffix(str, "/") { + // Do not forget to preserve the trailing slash! + cleaned += "/" } - return FromCid(c), nil -} + switch segments[0] { + case IPFSNamespace, IPLDNamespace: + cid, err := cid.Decode(segments[1]) + if err != nil { + return nil, &ErrInvalidPath{err: err, path: str} + } -// IsValid checks if a path is a valid ipfs Path. -func (p *Path) IsValid() error { - _, err := ParsePath(p.String()) - return err + return immutablePath{ + path: path{ + str: cleaned, + namespace: segments[0], + }, + rootCid: cid, + }, nil + case "ipns": + return path{ + str: cleaned, + namespace: segments[0], + }, nil + default: + return nil, &ErrInvalidPath{err: fmt.Errorf("%w: %q", ErrUnknownNamespace, segments[0]), path: str} + } } -// Join joins strings slices using / -func Join(pths []string) string { - return strings.Join(pths, "/") +// NewPathFromSegments creates a new [Path] from the provided segments. This +// function simply calls [NewPath] internally with the segments concatenated +// using a forward slash "/" as separator. Please see [Path.Segments] for more +// information about how segments must be structured. +func NewPathFromSegments(segments ...string) (Path, error) { + return NewPath(SegmentsToString(segments...)) } -// SplitList splits strings usings / -func SplitList(pth string) []string { - return strings.Split(pth, "/") +// Join joins a [Path] with certain segments and returns a new [Path]. +func Join(p Path, segments ...string) (Path, error) { + s := p.Segments() + s = append(s, segments...) + return NewPathFromSegments(s...) } -// SplitAbsPath clean up and split fpath. It extracts the first component (which -// must be a Multihash) and return it separately. -func SplitAbsPath(fpath Path) (cid.Cid, []string, error) { - parts := fpath.Segments() - if parts[0] == "ipfs" || parts[0] == "ipld" { - parts = parts[1:] +// SegmentsToString converts an array of segments into a string. The returned string +// will always be prefixed with a "/" if there are any segments. For example, if the +// given segments array is ["foo", "bar"], the returned value will be "/foo/bar". +// Given an empty array, an empty string is returned. +func SegmentsToString(segments ...string) string { + str := strings.Join(segments, "/") + if str != "" { + str = "/" + str } - - // if nothing, bail. - if len(parts) == 0 { - return cid.Cid{}, nil, &ErrInvalidPath{error: fmt.Errorf("empty"), path: string(fpath)} - } - - c, err := decodeCid(parts[0]) - // first element in the path is a cid - if err != nil { - return cid.Cid{}, nil, &ErrInvalidPath{error: fmt.Errorf("invalid CID: %w", err), path: string(fpath)} - } - - return c, parts[1:], nil + return str } -func decodeCid(cstr string) (cid.Cid, error) { - c, err := cid.Decode(cstr) - if err != nil && len(cstr) == 46 && cstr[:2] == "qm" { // https://github.com/ipfs/go-ipfs/issues/7792 - return cid.Cid{}, fmt.Errorf("%v (possible lowercased CIDv0; consider converting to a case-agnostic CIDv1, such as base32)", err) +// StringToSegments converts a string into an array of segments. This function follows +// the rules of [Path.Segments]: the path is first cleaned through [gopath.Clean] and +// no empty segments are returned. +func StringToSegments(str string) []string { + str = gopath.Clean(str) + if str == "." { + return nil + } + // Trim slashes from beginning and end, such that we do not return empty segments. + str = strings.TrimSuffix(str, "/") + str = strings.TrimPrefix(str, "/") + if str == "" { + return nil } - return c, err + return strings.Split(str, "/") } diff --git a/path/path_test.go b/path/path_test.go index 2b26a5678..2f944e39e 100644 --- a/path/path_test.go +++ b/path/path_test.go @@ -1,128 +1,305 @@ package path import ( - "strings" + "fmt" "testing" + + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/assert" ) -func TestPathParsing(t *testing.T) { - cases := map[string]bool{ - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": true, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true, - "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, - "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": true, - "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true, - "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true, - "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, - "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f": true, - "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, - "/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false, - "/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": false, - "/ipfs/foo": false, - "/ipfs/": false, - "ipfs/": false, - "ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false, - "/ipld/foo": false, - "/ipld/": false, - "ipld/": false, - "ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false, +func newIPLDPath(cid cid.Cid) ImmutablePath { + return immutablePath{ + path: path{ + str: fmt.Sprintf("/%s/%s", IPLDNamespace, cid.String()), + namespace: IPLDNamespace, + }, + rootCid: cid, } +} + +func TestNewPath(t *testing.T) { + t.Parallel() + + t.Run("Valid Paths", func(t *testing.T) { + t.Parallel() + + testCases := []struct { + src string + canonical string + namespace string + mutable bool + }{ + // IPFS CIDv0 + {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", IPFSNamespace, false}, + {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", IPFSNamespace, false}, + {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f", "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f", IPFSNamespace, false}, + + // IPFS CIDv1 + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", "/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", IPFSNamespace, false}, + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a", "/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a", IPFSNamespace, false}, + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b/c/d/e/f", "/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b/c/d/e/f", IPFSNamespace, false}, + + // IPLD CIDv0 + {"/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", IPLDNamespace, false}, + {"/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", IPLDNamespace, false}, + {"/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f", "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f", IPLDNamespace, false}, - for p, expected := range cases { - _, err := ParsePath(p) - valid := err == nil - if valid != expected { - t.Fatalf("expected %s to have valid == %t", p, expected) + // IPLD CIDv1 + {"/ipld/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", "/ipld/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", IPLDNamespace, false}, + {"/ipld/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a", "/ipld/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a", IPLDNamespace, false}, + {"/ipld/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b/c/d/e/f", "/ipld/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b/c/d/e/f", IPLDNamespace, false}, + + // IPNS CIDv0 + {"/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", IPNSNamespace, true}, + {"/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", IPNSNamespace, true}, + {"/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f", "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/c/d/e/f", IPNSNamespace, true}, + + // IPNS CIDv1 + {"/ipns/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", "/ipns/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", IPNSNamespace, true}, + {"/ipns/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a", "/ipns/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a", IPNSNamespace, true}, + {"/ipns/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b/c/d/e/f", "/ipns/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b/c/d/e/f", IPNSNamespace, true}, + + // IPNS DNSLink + {"/ipns/domain.net", "/ipns/domain.net", IPNSNamespace, true}, + {"/ipns/domain.net/a/b/c/d", "/ipns/domain.net/a/b/c/d", IPNSNamespace, true}, + + // Cleaning checks + {"/ipfs/bafkqaaa/", "/ipfs/bafkqaaa/", IPFSNamespace, false}, + {"/ipfs/bafkqaaa//", "/ipfs/bafkqaaa/", IPFSNamespace, false}, + {"/ipfs///bafkqaaa//", "/ipfs/bafkqaaa/", IPFSNamespace, false}, + {"/ipfs///bafkqaaa/a/b/../c", "/ipfs/bafkqaaa/a/c", IPFSNamespace, false}, + {"/ipfs///bafkqaaa/a/b/../c/", "/ipfs/bafkqaaa/a/c/", IPFSNamespace, false}, } - } -} -func TestNoComponents(t *testing.T) { - for _, s := range []string{ - "/ipfs/", - "/ipns/", - "/ipld/", - } { - _, err := ParsePath(s) - if err == nil || !strings.Contains(err.Error(), "not enough path components") || !strings.Contains(err.Error(), s) { - t.Error("wrong error") + for _, testCase := range testCases { + p, err := NewPath(testCase.src) + assert.NoError(t, err) + assert.Equal(t, testCase.canonical, p.String()) + assert.Equal(t, testCase.namespace, p.Namespace()) + assert.Equal(t, testCase.mutable, p.Mutable()) } - } -} + }) + + t.Run("Invalid Paths", func(t *testing.T) { + t.Parallel() -func TestInvalidPaths(t *testing.T) { - for _, s := range []string{ - "/ipfs", - "/testfs", - "/", - } { - _, err := ParsePath(s) - if err == nil || !strings.Contains(err.Error(), "invalid ipfs path") || !strings.Contains(err.Error(), s) { - t.Error("wrong error") + testCases := []struct { + src string + err error + }{ + {"QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", ErrInsufficientComponents}, + {"QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", ErrInsufficientComponents}, + {"bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a", ErrInsufficientComponents}, + {"/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", ErrInsufficientComponents}, + {"/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", ErrUnknownNamespace}, + {"/ipfs/foo", cid.ErrInvalidCid{}}, + {"/ipfs/", ErrInsufficientComponents}, + {"ipfs/", ErrInsufficientComponents}, + {"ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", ErrInsufficientComponents}, + {"/ipld/foo", &ErrInvalidPath{}}, + {"/ipld/", ErrInsufficientComponents}, + {"ipld/", ErrInsufficientComponents}, + {"ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", ErrInsufficientComponents}, + {"/ipns", ErrInsufficientComponents}, + {"/ipfs/", ErrInsufficientComponents}, + {"/ipns/", ErrInsufficientComponents}, + {"/ipld/", ErrInsufficientComponents}, + {"/ipfs", ErrInsufficientComponents}, + {"/testfs", ErrInsufficientComponents}, + {"/", ErrInsufficientComponents}, } - } -} -func TestIsJustAKey(t *testing.T) { - cases := map[string]bool{ - "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": false, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b": false, - "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": false, - "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b": false, - "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": true, - } + for _, testCase := range testCases { + _, err := NewPath(testCase.src) + assert.ErrorIs(t, err, testCase.err) + assert.ErrorIs(t, err, &ErrInvalidPath{}) // Always an ErrInvalidPath! + } + }) + + t.Run("Returns ImmutablePath for IPFS and IPLD Paths", func(t *testing.T) { + t.Parallel() - for p, expected := range cases { - path, err := ParsePath(p) - if err != nil { - t.Fatalf("ParsePath failed to parse \"%s\", but should have succeeded", p) + testCases := []struct { + src string + }{ + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"}, + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a"}, + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b/c/d/e/f"}, + {"/ipld/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku"}, + {"/ipld/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a"}, + {"/ipld/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b/c/d/e/f"}, } - result := path.IsJustAKey() - if result != expected { - t.Fatalf("expected IsJustAKey(%s) to return %v, not %v", p, expected, result) + + for _, testCase := range testCases { + p, err := NewPath(testCase.src) + assert.NoError(t, err) + assert.IsType(t, immutablePath{}, p) } - } + }) } -func TestPopLastSegment(t *testing.T) { - cases := map[string][]string{ - "QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", ""}, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n": {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", ""}, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a": {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", "a"}, - "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b": {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a", "b"}, - "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/x/y/z": {"/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/x/y", "z"}, - "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/x/y/z": {"/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/x/y", "z"}, - } +func TestFromCid(t *testing.T) { + t.Parallel() - for p, expected := range cases { - path, err := ParsePath(p) - if err != nil { - t.Fatalf("ParsePath failed to parse \"%s\", but should have succeeded", p) - } - head, tail, err := path.PopLastSegment() - if err != nil { - t.Fatalf("PopLastSegment failed, but should have succeeded: %s", err) + t.Run("Works with CIDv0", func(t *testing.T) { + t.Parallel() + + c, err := cid.Decode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") + assert.NoError(t, err) + + p := FromCid(c) + assert.IsType(t, immutablePath{}, p) + assert.Equal(t, "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", p.String()) + assert.Equal(t, c, p.RootCid()) + }) + + t.Run("Works with CIDv1", func(t *testing.T) { + t.Parallel() + + c, err := cid.Decode("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku") + assert.NoError(t, err) + + p := FromCid(c) + assert.IsType(t, immutablePath{}, p) + assert.Equal(t, "/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", p.String()) + assert.Equal(t, c, p.RootCid()) + }) + + t.Run("newIPLDPath returns correct ImmutablePath", func(t *testing.T) { + c, err := cid.Decode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n") + assert.NoError(t, err) + + p := newIPLDPath(c) + assert.IsType(t, immutablePath{}, p) + assert.Equal(t, "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", p.String()) + assert.Equal(t, c, p.RootCid()) + + // Check if CID encoding is preserved. + c, err = cid.Decode("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku") + assert.NoError(t, err) + + p = newIPLDPath(c) + assert.IsType(t, immutablePath{}, p) + assert.Equal(t, "/ipld/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", p.String()) + assert.Equal(t, c, p.RootCid()) + }) +} + +func TestNewImmutablePath(t *testing.T) { + t.Parallel() + + t.Run("Fails on Mutable Path", func(t *testing.T) { + for _, path := range []string{ + "/ipns/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", + "/ipns/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", + "/ipns/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/with/path", + "/ipns/domain.net", + } { + p, err := NewPath(path) + assert.NoError(t, err) + + _, err = NewImmutablePath(p) + assert.ErrorIs(t, err, ErrExpectedImmutable) + assert.ErrorIs(t, err, &ErrInvalidPath{}) } - headStr := head.String() - if headStr != expected[0] { - t.Fatalf("expected head of PopLastSegment(%s) to return %v, not %v", p, expected[0], headStr) + }) + + t.Run("Succeeds on Immutable Path", func(t *testing.T) { + testCases := []struct { + path string + cid cid.Cid + }{ + {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", cid.MustParse("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n")}, + {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b", cid.MustParse("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n")}, + {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/", cid.MustParse("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n")}, + + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", cid.MustParse("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku")}, + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b", cid.MustParse("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku")}, + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b/", cid.MustParse("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku")}, + + {"/ipld/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", cid.MustParse("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku")}, + {"/ipld/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b", cid.MustParse("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku")}, + {"/ipld/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b/", cid.MustParse("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku")}, } - if tail != expected[1] { - t.Fatalf("expected tail of PopLastSegment(%s) to return %v, not %v", p, expected[1], tail) + + for _, testCase := range testCases { + p, err := NewPath(testCase.path) + assert.NoError(t, err) + + ip, err := NewImmutablePath(p) + assert.NoError(t, err) + assert.Equal(t, testCase.path, ip.String()) + assert.Equal(t, testCase.cid, ip.RootCid()) } + }) +} + +func TestJoin(t *testing.T) { + t.Parallel() + + testCases := []struct { + path string + segments []string + expected string + }{ + {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", []string{"a/b"}, "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b"}, + {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", []string{"/a/b"}, "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b"}, + {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/", []string{"/a/b"}, "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b"}, + {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", []string{"a", "b"}, "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b"}, + {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", []string{"a/b/../"}, "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/"}, + {"/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", []string{"a/b", "/"}, "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n/a/b/"}, + + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", []string{"a/b"}, "/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b"}, + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", []string{"/a/b"}, "/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b"}, + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/", []string{"/a/b"}, "/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b"}, + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", []string{"a", "b"}, "/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b"}, + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", []string{"a/b/../"}, "/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/"}, + {"/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", []string{"a/b", "/"}, "/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/a/b/"}, + } + + for _, testCase := range testCases { + p, err := NewPath(testCase.path) + assert.NoError(t, err) + jp, err := Join(p, testCase.segments...) + assert.NoError(t, err) + assert.Equal(t, testCase.expected, jp.String()) } } -func TestV0ErrorDueToLowercase(t *testing.T) { - badb58 := "/ipfs/qmbwqxbekc3p8tqskc98xmwnzrzdtrlmimpl8wbutgsmnr" - _, err := ParsePath(badb58) - if err == nil { - t.Fatal("should have failed to decode") +func TestStringToSegments(t *testing.T) { + testCases := []struct { + str string + expected []string + }{ + {"", nil}, + {"/..", nil}, + {"/a/b/c/d/./../../../../../..", nil}, + {"/a/b/c/d/./../../../", []string{"a"}}, + {"/a/b//c/d/./../../", []string{"a", "b"}}, + {"/a/b/////c/../d///f", []string{"a", "b", "d", "f"}}, } - if !strings.HasSuffix(err.Error(), "(possible lowercased CIDv0; consider converting to a case-agnostic CIDv1, such as base32)") { - t.Fatal("should have meaningful info about case-insensitive fix") + + for _, testCase := range testCases { + segments := StringToSegments(testCase.str) + assert.Equal(t, testCase.expected, segments) + } +} + +func TestSegmentsToString(t *testing.T) { + testCases := []struct { + segments []string + expected string + }{ + {[]string{"a", "b"}, "/a/b"}, + {[]string{"a", "b", "d", "f"}, "/a/b/d/f"}, + {[]string{""}, ""}, + {[]string{}, ""}, + {nil, ""}, + } + + for _, testCase := range testCases { + str := SegmentsToString(testCase.segments...) + assert.Equal(t, testCase.expected, str) } } diff --git a/path/resolver/resolver.go b/path/resolver/resolver.go index f666d4b79..e7cc25f92 100644 --- a/path/resolver/resolver.go +++ b/path/resolver/resolver.go @@ -3,19 +3,17 @@ package resolver import ( "context" - "errors" "fmt" "time" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "github.com/ipfs/boxo/fetcher" fetcherhelpers "github.com/ipfs/boxo/fetcher/helpers" - path "github.com/ipfs/boxo/path" - "github.com/ipfs/boxo/path/internal" + "github.com/ipfs/boxo/path" cid "github.com/ipfs/go-cid" - format "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" @@ -24,76 +22,73 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector/builder" ) -var log = logging.Logger("pathresolv") +var log = logging.Logger("path/resolver") -// ErrNoComponents is used when Paths after a protocol -// do not contain at least one component -var ErrNoComponents = errors.New( - "path must contain at least one component") - -// ErrNoLink is returned when a link is not found in a path +// ErrNoLink is returned when a link is not found in a path. type ErrNoLink struct { Name string Node cid.Cid } -// Error implements the Error interface for ErrNoLink with a useful -// human readable message. -func (e ErrNoLink) Error() string { +// Error implements the [errors.Error] interface. +func (e *ErrNoLink) Error() string { return fmt.Sprintf("no link named %q under %s", e.Name, e.Node.String()) } +// Is implements [errors.Is] interface. +func (e *ErrNoLink) Is(err error) bool { + switch err.(type) { + case *ErrNoLink: + return true + default: + return false + } +} + // Resolver provides path resolution to IPFS. type Resolver interface { - // ResolveToLastNode walks the given path and returns the cid of the - // last block referenced by the path, and the path segments to - // traverse from the final block boundary to the final node within the - // block. - ResolveToLastNode(ctx context.Context, fpath path.Path) (cid.Cid, []string, error) - // ResolvePath fetches the node for given path. It returns the last - // item returned by ResolvePathComponents and the last link traversed - // which can be used to recover the block. - ResolvePath(ctx context.Context, fpath path.Path) (ipld.Node, ipld.Link, error) - // ResolvePathComponents fetches the nodes for each segment of the given path. - // It uses the first path component as a hash (key) of the first node, then - // resolves all other components walking the links via a selector traversal - ResolvePathComponents(ctx context.Context, fpath path.Path) ([]ipld.Node, error) + // ResolveToLastNode walks the given path and returns the CID of the last block + // referenced by the path, as well as the remainder of the path segments to traverse + // from the final block boundary to the final node within the block. + ResolveToLastNode(context.Context, path.ImmutablePath) (cid.Cid, []string, error) + + // ResolvePath fetches the node for the given path. It returns the last item returned + // by [Resolver.ResolvePathComponents] and the last link traversed which can be used + // to recover the block. + ResolvePath(context.Context, path.ImmutablePath) (ipld.Node, ipld.Link, error) + + // ResolvePathComponents fetches the nodes for each segment of the given path. It + // uses the first path component as the CID of the first node, then resolves all + // other components walking the links via a selector traversal. + ResolvePathComponents(context.Context, path.ImmutablePath) ([]ipld.Node, error) } -// basicResolver implements the Resolver interface. -// It references a FetcherFactory, which is uses to resolve nodes. -// TODO: now that this is more modular, try to unify this code with the -// -// the resolvers in namesys. +// basicResolver implements the [Resolver] interface. It requires a [fetcher.Factory], +// which is used to resolve the nodes. type basicResolver struct { FetcherFactory fetcher.Factory } -// NewBasicResolver constructs a new basic resolver. -func NewBasicResolver(fetcherFactory fetcher.Factory) Resolver { +// NewBasicResolver constructs a new basic resolver using the given [fetcher.Factory]. +func NewBasicResolver(factory fetcher.Factory) Resolver { return &basicResolver{ - FetcherFactory: fetcherFactory, + FetcherFactory: factory, } } -// ResolveToLastNode walks the given path and returns the cid of the last -// block referenced by the path, and the path segments to traverse from the -// final block boundary to the final node within the block. -func (r *basicResolver) ResolveToLastNode(ctx context.Context, fpath path.Path) (cid.Cid, []string, error) { - ctx, span := internal.StartSpan(ctx, "basicResolver.ResolveToLastNode", trace.WithAttributes(attribute.Stringer("Path", fpath))) +// ResolveToLastNode implements [Resolver.ResolveToLastNode]. +func (r *basicResolver) ResolveToLastNode(ctx context.Context, fpath path.ImmutablePath) (cid.Cid, []string, error) { + ctx, span := startSpan(ctx, "basicResolver.ResolveToLastNode", trace.WithAttributes(attribute.Stringer("Path", fpath))) defer span.End() - c, p, err := path.SplitAbsPath(fpath) - if err != nil { - return cid.Cid{}, nil, err - } + c, remainder := fpath.RootCid(), fpath.Segments()[2:] - if len(p) == 0 { + if len(remainder) == 0 { return c, nil, nil } // create a selector to traverse and match all path segments - pathSelector := pathAllSelector(p[:len(p)-1]) + pathSelector := pathAllSelector(remainder[:len(remainder)-1]) // create a new cancellable session ctx, cancel := context.WithTimeout(ctx, time.Minute) @@ -107,19 +102,19 @@ func (r *basicResolver) ResolveToLastNode(ctx context.Context, fpath path.Path) if len(nodes) < 1 { return cid.Cid{}, nil, fmt.Errorf("path %v did not resolve to a node", fpath) - } else if len(nodes) < len(p) { - return cid.Undef, nil, ErrNoLink{Name: p[len(nodes)-1], Node: lastCid} + } else if len(nodes) < len(remainder) { + return cid.Undef, nil, &ErrNoLink{Name: remainder[len(nodes)-1], Node: lastCid} } parent := nodes[len(nodes)-1] - lastSegment := p[len(p)-1] + lastSegment := remainder[len(remainder)-1] // find final path segment within node nd, err := parent.LookupBySegment(ipld.ParsePathSegment(lastSegment)) switch err.(type) { case nil: case schema.ErrNoSuchField: - return cid.Undef, nil, ErrNoLink{Name: lastSegment, Node: lastCid} + return cid.Undef, nil, &ErrNoLink{Name: lastSegment, Node: lastCid} default: return cid.Cid{}, nil, err } @@ -127,7 +122,7 @@ func (r *basicResolver) ResolveToLastNode(ctx context.Context, fpath path.Path) // if last node is not a link, just return it's cid, add path to remainder and return if nd.Kind() != ipld.Kind_Link { // return the cid and the remainder of the path - return lastCid, p[len(p)-depth-1:], nil + return lastCid, remainder[len(remainder)-depth-1:], nil } lnk, err := nd.AsLink() @@ -143,27 +138,18 @@ func (r *basicResolver) ResolveToLastNode(ctx context.Context, fpath path.Path) return clnk.Cid, []string{}, nil } -// ResolvePath fetches the node for given path. It returns the last item -// returned by ResolvePathComponents and the last link traversed which can be used to recover the block. +// ResolvePath implements [Resolver.ResolvePath]. // -// Note: if/when the context is cancelled or expires then if a multi-block ADL node is returned then it may not be -// possible to load certain values. -func (r *basicResolver) ResolvePath(ctx context.Context, fpath path.Path) (ipld.Node, ipld.Link, error) { - ctx, span := internal.StartSpan(ctx, "basicResolver.ResolvePath", trace.WithAttributes(attribute.Stringer("Path", fpath))) +// Note: if/when the context is cancelled or expires then if a multi-block ADL +// node is returned then it may not be possible to load certain values. +func (r *basicResolver) ResolvePath(ctx context.Context, fpath path.ImmutablePath) (ipld.Node, ipld.Link, error) { + ctx, span := startSpan(ctx, "basicResolver.ResolvePath", trace.WithAttributes(attribute.Stringer("Path", fpath))) defer span.End() - // validate path - if err := fpath.IsValid(); err != nil { - return nil, nil, err - } - - c, p, err := path.SplitAbsPath(fpath) - if err != nil { - return nil, nil, err - } + c, remainder := fpath.RootCid(), fpath.Segments()[2:] // create a selector to traverse all path segments but only match the last - pathSelector := pathLeafSelector(p) + pathSelector := pathLeafSelector(remainder) nodes, c, _, err := r.resolveNodes(ctx, c, pathSelector) if err != nil { @@ -175,81 +161,29 @@ func (r *basicResolver) ResolvePath(ctx context.Context, fpath path.Path) (ipld. return nodes[len(nodes)-1], cidlink.Link{Cid: c}, nil } -// ResolveSingle simply resolves one hop of a path through a graph with no -// extra context (does not opaquely resolve through sharded nodes) -// Deprecated: fetch node as ipld-prime or convert it and then use a selector to traverse through it. -func ResolveSingle(ctx context.Context, ds format.NodeGetter, nd format.Node, names []string) (*format.Link, []string, error) { - _, span := internal.StartSpan(ctx, "ResolveSingle", trace.WithAttributes(attribute.Stringer("CID", nd.Cid()))) - defer span.End() - return nd.ResolveLink(names) -} - -// ResolvePathComponents fetches the nodes for each segment of the given path. -// It uses the first path component as a hash (key) of the first node, then -// resolves all other components walking the links via a selector traversal +// ResolvePathComponents implements [Resolver.ResolvePathComponents]. // -// Note: if/when the context is cancelled or expires then if a multi-block ADL node is returned then it may not be -// possible to load certain values. -func (r *basicResolver) ResolvePathComponents(ctx context.Context, fpath path.Path) (nodes []ipld.Node, err error) { - ctx, span := internal.StartSpan(ctx, "basicResolver.ResolvePathComponents", trace.WithAttributes(attribute.Stringer("Path", fpath))) +// Note: if/when the context is cancelled or expires then if a multi-block ADL +// node is returned then it may not be possible to load certain values. +func (r *basicResolver) ResolvePathComponents(ctx context.Context, fpath path.ImmutablePath) (nodes []ipld.Node, err error) { + ctx, span := startSpan(ctx, "basicResolver.ResolvePathComponents", trace.WithAttributes(attribute.Stringer("Path", fpath))) defer span.End() defer log.Debugw("resolvePathComponents", "fpath", fpath, "error", err) - // validate path - if err := fpath.IsValid(); err != nil { - return nil, err - } - - c, p, err := path.SplitAbsPath(fpath) - if err != nil { - return nil, err - } + c, remainder := fpath.RootCid(), fpath.Segments()[2:] // create a selector to traverse and match all path segments - pathSelector := pathAllSelector(p) + pathSelector := pathAllSelector(remainder) nodes, _, _, err = r.resolveNodes(ctx, c, pathSelector) return nodes, err } -// ResolveLinks iteratively resolves names by walking the link hierarchy. -// Every node is fetched from the Fetcher, resolving the next name. -// Returns the list of nodes forming the path, starting with ndd. This list is -// guaranteed never to be empty. -// -// ResolveLinks(nd, []string{"foo", "bar", "baz"}) -// would retrieve "baz" in ("bar" in ("foo" in nd.Links).Links).Links -// -// Note: if/when the context is cancelled or expires then if a multi-block ADL node is returned then it may not be -// possible to load certain values. -func (r *basicResolver) ResolveLinks(ctx context.Context, ndd ipld.Node, names []string) (nodes []ipld.Node, err error) { - ctx, span := internal.StartSpan(ctx, "basicResolver.ResolveLinks") - defer span.End() - - defer log.Debugw("resolvePathComponents", "names", names, "error", err) - // create a selector to traverse and match all path segments - pathSelector := pathAllSelector(names) - - session := r.FetcherFactory.NewSession(ctx) - - // traverse selector - nodes = []ipld.Node{ndd} - err = session.NodeMatching(ctx, ndd, pathSelector, func(res fetcher.FetchResult) error { - nodes = append(nodes, res.Node) - return nil - }) - if err != nil { - return nil, err - } - - return nodes, err -} - // Finds nodes matching the selector starting with a cid. Returns the matched nodes, the cid of the block containing // the last node, and the depth of the last node within its block (root is depth 0). func (r *basicResolver) resolveNodes(ctx context.Context, c cid.Cid, sel ipld.Node) ([]ipld.Node, cid.Cid, int, error) { - ctx, span := internal.StartSpan(ctx, "basicResolver.resolveNodes", trace.WithAttributes(attribute.Stringer("CID", c))) + ctx, span := startSpan(ctx, "basicResolver.resolveNodes", trace.WithAttributes(attribute.Stringer("CID", c))) defer span.End() session := r.FetcherFactory.NewSession(ctx) @@ -308,3 +242,7 @@ func pathSelector(path []string, ssb builder.SelectorSpecBuilder, reduce func(st } return spec.Node() } + +func startSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + return otel.Tracer("boxo/path/resolver").Start(ctx, fmt.Sprintf("Path.%s", name), opts...) +} diff --git a/path/resolver/resolver_test.go b/path/resolver/resolver_test.go index c20f9306d..91f05e7d6 100644 --- a/path/resolver/resolver_test.go +++ b/path/resolver/resolver_test.go @@ -3,7 +3,6 @@ package resolver_test import ( "bytes" "context" - "fmt" "math/rand" "strings" "testing" @@ -21,12 +20,11 @@ import ( merkledag "github.com/ipfs/boxo/ipld/merkledag" dagmock "github.com/ipfs/boxo/ipld/merkledag/test" - path "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/path/resolver" "github.com/ipfs/go-unixfsnode" dagcbor "github.com/ipld/go-ipld-prime/codec/dagcbor" dagjson "github.com/ipld/go-ipld-prime/codec/dagjson" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -38,7 +36,7 @@ func randNode() *merkledag.ProtoNode { return node } -func TestRecurivePathResolution(t *testing.T) { +func TestRecursivePathResolution(t *testing.T) { ctx := context.Background() bsrv := dagmock.Bserv() @@ -47,29 +45,21 @@ func TestRecurivePathResolution(t *testing.T) { c := randNode() err := b.AddNodeLink("grandchild", c) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) err = a.AddNodeLink("child", b) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) for _, n := range []*merkledag.ProtoNode{a, b, c} { err = bsrv.AddBlock(ctx, n) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) } - aKey := a.Cid() + p, err := path.Join(path.FromCid(a.Cid()), "child", "grandchild") + require.NoError(t, err) - segments := []string{aKey.String(), "child", "grandchild"} - p, err := path.FromSegments("/ipfs/", segments...) - if err != nil { - t.Fatal(err) - } + imPath, err := path.NewImmutablePath(p) + require.NoError(t, err) fetcherFactory := bsfetcher.NewFetcherConfig(bsrv) fetcherFactory.NodeReifier = unixfsnode.Reify @@ -81,56 +71,26 @@ func TestRecurivePathResolution(t *testing.T) { }) resolver := resolver.NewBasicResolver(fetcherFactory) - node, lnk, err := resolver.ResolvePath(ctx, p) - if err != nil { - t.Fatal(err) - } + node, lnk, err := resolver.ResolvePath(ctx, imPath) + require.NoError(t, err) uNode, ok := node.(unixfsnode.PathedPBNode) require.True(t, ok) fd := uNode.FieldData() byts, err := fd.Must().AsBytes() require.NoError(t, err) + require.Equal(t, cidlink.Link{Cid: c.Cid()}, lnk) + require.Equal(t, c.Data(), byts) - assert.Equal(t, cidlink.Link{Cid: c.Cid()}, lnk) - - assert.Equal(t, c.Data(), byts) - cKey := c.Cid() - - rCid, rest, err := resolver.ResolveToLastNode(ctx, p) - if err != nil { - t.Fatal(err) - } - - if len(rest) != 0 { - t.Error("expected rest to be empty") - } - - if rCid.String() != cKey.String() { - t.Fatal(fmt.Errorf( - "ResolveToLastNode failed for %s: %s != %s", - p.String(), rCid.String(), cKey.String())) - } - - p2, err := path.FromSegments("/ipfs/", aKey.String()) - if err != nil { - t.Fatal(err) - } - - rCid, rest, err = resolver.ResolveToLastNode(ctx, p2) - if err != nil { - t.Fatal(err) - } - - if len(rest) != 0 { - t.Error("expected rest to be empty") - } + rCid, remainder, err := resolver.ResolveToLastNode(ctx, imPath) + require.NoError(t, err) + require.Empty(t, remainder) + require.Equal(t, c.Cid().String(), rCid.String()) - if rCid.String() != aKey.String() { - t.Fatal(fmt.Errorf( - "ResolveToLastNode failed for %s: %s != %s", - p.String(), rCid.String(), cKey.String())) - } + rCid, remainder, err = resolver.ResolveToLastNode(ctx, path.FromCid(a.Cid())) + require.NoError(t, err) + require.Empty(t, remainder) + require.Equal(t, a.Cid().String(), rCid.String()) } func TestResolveToLastNode_ErrNoLink(t *testing.T) { @@ -142,24 +102,16 @@ func TestResolveToLastNode_ErrNoLink(t *testing.T) { c := randNode() err := b.AddNodeLink("grandchild", c) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) err = a.AddNodeLink("child", b) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) for _, n := range []*merkledag.ProtoNode{a, b, c} { err = bsrv.AddBlock(ctx, n) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) } - aKey := a.Cid() - fetcherFactory := bsfetcher.NewFetcherConfig(bsrv) fetcherFactory.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { @@ -171,21 +123,27 @@ func TestResolveToLastNode_ErrNoLink(t *testing.T) { r := resolver.NewBasicResolver(fetcherFactory) // test missing link intermediate segment - segments := []string{aKey.String(), "cheese", "time"} - p, err := path.FromSegments("/ipfs/", segments...) + p, err := path.Join(path.FromCid(a.Cid()), "cheese", "time") require.NoError(t, err) - _, _, err = r.ResolveToLastNode(ctx, p) - require.EqualError(t, err, resolver.ErrNoLink{Name: "cheese", Node: aKey}.Error()) + imPath, err := path.NewImmutablePath(p) + require.NoError(t, err) + + _, _, err = r.ResolveToLastNode(ctx, imPath) + require.ErrorIs(t, err, &resolver.ErrNoLink{}) + require.Equal(t, "cheese", err.(*resolver.ErrNoLink).Name) + require.Equal(t, a.Cid(), err.(*resolver.ErrNoLink).Node) // test missing link at end - bKey := b.Cid() - segments = []string{aKey.String(), "child", "apples"} - p, err = path.FromSegments("/ipfs/", segments...) + p, err = path.Join(path.FromCid(a.Cid()), "child", "apples") + require.NoError(t, err) + + imPath, err = path.NewImmutablePath(p) require.NoError(t, err) - _, _, err = r.ResolveToLastNode(ctx, p) - require.EqualError(t, err, resolver.ErrNoLink{Name: "apples", Node: bKey}.Error()) + _, _, err = r.ResolveToLastNode(ctx, imPath) + require.Equal(t, "apples", err.(*resolver.ErrNoLink).Name) + require.Equal(t, b.Cid(), err.(*resolver.ErrNoLink).Node) } func TestResolveToLastNode_NoUnnecessaryFetching(t *testing.T) { @@ -201,10 +159,10 @@ func TestResolveToLastNode_NoUnnecessaryFetching(t *testing.T) { err = bsrv.AddBlock(ctx, a) require.NoError(t, err) - aKey := a.Cid() + p, err := path.Join(path.FromCid(a.Cid()), "child") + require.NoError(t, err) - segments := []string{aKey.String(), "child"} - p, err := path.FromSegments("/ipfs/", segments...) + imPath, err := path.NewImmutablePath(p) require.NoError(t, err) fetcherFactory := bsfetcher.NewFetcherConfig(bsrv) @@ -217,7 +175,7 @@ func TestResolveToLastNode_NoUnnecessaryFetching(t *testing.T) { fetcherFactory.NodeReifier = unixfsnode.Reify resolver := resolver.NewBasicResolver(fetcherFactory) - resolvedCID, remainingPath, err := resolver.ResolveToLastNode(ctx, p) + resolvedCID, remainingPath, err := resolver.ResolveToLastNode(ctx, imPath) require.NoError(t, err) require.Equal(t, len(remainingPath), 0, "cannot have remaining path") @@ -232,9 +190,11 @@ func TestPathRemainder(t *testing.T) { nb := basicnode.Prototype.Any.NewBuilder() err := dagjson.Decode(nb, strings.NewReader(`{"foo": {"bar": "baz"}}`)) require.NoError(t, err) + out := new(bytes.Buffer) err = dagcbor.Encode(nb.Build(), out) require.NoError(t, err) + lnk, err := cid.Prefix{ Version: 1, Codec: cid.DagCBOR, @@ -242,37 +202,46 @@ func TestPathRemainder(t *testing.T) { MhLength: 32, }.Sum(out.Bytes()) require.NoError(t, err) + blk, err := blocks.NewBlockWithCid(out.Bytes(), lnk) require.NoError(t, err) + bsrv.AddBlock(ctx, blk) fetcherFactory := bsfetcher.NewFetcherConfig(bsrv) resolver := resolver.NewBasicResolver(fetcherFactory) - rp1, remainder, err := resolver.ResolveToLastNode(ctx, path.FromString(lnk.String()+"/foo/bar")) + p, err := path.Join(path.FromCid(lnk), "foo", "bar") require.NoError(t, err) - assert.Equal(t, lnk, rp1) - require.Equal(t, "foo/bar", path.Join(remainder)) + imPath, err := path.NewImmutablePath(p) + require.NoError(t, err) + + rp, remainder, err := resolver.ResolveToLastNode(ctx, imPath) + require.NoError(t, err) + + require.Equal(t, lnk, rp) + require.Equal(t, "foo/bar", strings.Join(remainder, "/")) } func TestResolveToLastNode_MixedSegmentTypes(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + bsrv := dagmock.Bserv() a := randNode() err := bsrv.AddBlock(ctx, a) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) nb := basicnode.Prototype.Any.NewBuilder() json := `{"foo":{"bar":[0,{"boom":["baz",1,2,{"/":"CID"},"blop"]}]}}` json = strings.ReplaceAll(json, "CID", a.Cid().String()) err = dagjson.Decode(nb, strings.NewReader(json)) require.NoError(t, err) + out := new(bytes.Buffer) err = dagcbor.Encode(nb.Build(), out) require.NoError(t, err) + lnk, err := cid.Prefix{ Version: 1, Codec: cid.DagCBOR, @@ -280,15 +249,22 @@ func TestResolveToLastNode_MixedSegmentTypes(t *testing.T) { MhLength: 32, }.Sum(out.Bytes()) require.NoError(t, err) + blk, err := blocks.NewBlockWithCid(out.Bytes(), lnk) require.NoError(t, err) + bsrv.AddBlock(ctx, blk) fetcherFactory := bsfetcher.NewFetcherConfig(bsrv) resolver := resolver.NewBasicResolver(fetcherFactory) - cid, remainder, err := resolver.ResolveToLastNode(ctx, path.FromString(lnk.String()+"/foo/bar/1/boom/3")) + newPath, err := path.Join(path.FromCid(lnk), "foo", "bar", "1", "boom", "3") require.NoError(t, err) - assert.Equal(t, 0, len(remainder)) - assert.True(t, cid.Equals(a.Cid())) + imPath, err := path.NewImmutablePath(newPath) + require.NoError(t, err) + + cid, remainder, err := resolver.ResolveToLastNode(ctx, imPath) + require.NoError(t, err) + require.Equal(t, 0, len(remainder)) + require.True(t, cid.Equals(a.Cid())) } diff --git a/routing/http/client/client_test.go b/routing/http/client/client_test.go index 3822862dd..b6aa8456b 100644 --- a/routing/http/client/client_test.go +++ b/routing/http/client/client_test.go @@ -12,9 +12,8 @@ import ( "time" "github.com/benbjohnson/clock" - "github.com/ipfs/boxo/coreiface/path" ipns "github.com/ipfs/boxo/ipns" - ipfspath "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/routing/http/server" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" @@ -627,11 +626,11 @@ func makeIPNSRecord(t *testing.T, sk crypto.PrivKey, opts ...ipns.Option) (*ipns cid, err := cid.Decode("bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") require.NoError(t, err) - path := path.IpfsPath(cid) + path := path.FromCid(cid) eol := time.Now().Add(time.Hour * 48) ttl := time.Second * 20 - record, err := ipns.NewRecord(sk, ipfspath.FromString(path.String()), 1, eol, ttl, opts...) + record, err := ipns.NewRecord(sk, path, 1, eol, ttl, opts...) require.NoError(t, err) rawRecord, err := ipns.MarshalRecord(record) diff --git a/routing/http/contentrouter/contentrouter_test.go b/routing/http/contentrouter/contentrouter_test.go index 83a086997..2147dc975 100644 --- a/routing/http/contentrouter/contentrouter_test.go +++ b/routing/http/contentrouter/contentrouter_test.go @@ -6,9 +6,8 @@ import ( "testing" "time" - "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/ipns" - ipfspath "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" "github.com/ipfs/go-cid" @@ -217,11 +216,11 @@ func makeIPNSRecord(t *testing.T, sk crypto.PrivKey, opts ...ipns.Option) (*ipns cid, err := cid.Decode("bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") require.NoError(t, err) - path := path.IpfsPath(cid) + path := path.FromCid(cid) eol := time.Now().Add(time.Hour * 48) ttl := time.Second * 20 - record, err := ipns.NewRecord(sk, ipfspath.FromString(path.String()), 1, eol, ttl, opts...) + record, err := ipns.NewRecord(sk, path, 1, eol, ttl, opts...) require.NoError(t, err) rawRecord, err := ipns.MarshalRecord(record) diff --git a/routing/http/server/server_test.go b/routing/http/server/server_test.go index f6d4a3dba..c2c752057 100644 --- a/routing/http/server/server_test.go +++ b/routing/http/server/server_test.go @@ -10,9 +10,8 @@ import ( "testing" "time" - "github.com/ipfs/boxo/coreiface/path" "github.com/ipfs/boxo/ipns" - ipfspath "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" "github.com/ipfs/go-cid" @@ -242,11 +241,11 @@ func makeName(t *testing.T) (crypto.PrivKey, ipns.Name) { } func makeIPNSRecord(t *testing.T, cid cid.Cid, sk crypto.PrivKey, opts ...ipns.Option) (*ipns.Record, []byte) { - path := path.IpfsPath(cid) + path := path.FromCid(cid) eol := time.Now().Add(time.Hour * 48) ttl := time.Second * 20 - record, err := ipns.NewRecord(sk, ipfspath.FromString(path.String()), 1, eol, ttl, opts...) + record, err := ipns.NewRecord(sk, path, 1, eol, ttl, opts...) require.NoError(t, err) rawRecord, err := ipns.MarshalRecord(record) From 45c797e0ccea0a3186795e64d9e5ed60b5d7544c Mon Sep 17 00:00:00 2001 From: Jorropo Date: Fri, 6 Oct 2023 21:17:13 +0200 Subject: [PATCH 09/63] path: replace ImmutablePath interface with struct Let's not repeat https://github.com/ipfs/go-block-format/issues/45 interface for struct with one implementation and no value added. --- coreiface/tests/unixfs.go | 2 +- gateway/blocks_backend.go | 22 +++++++++--------- gateway/gateway_test.go | 2 +- gateway/handler.go | 8 +++---- gateway/handler_unixfs__redirects.go | 16 ++++++------- gateway/utilities_test.go | 6 ++--- path/path.go | 34 +++++++++++----------------- path/path_test.go | 12 +++++----- 8 files changed, 47 insertions(+), 55 deletions(-) diff --git a/coreiface/tests/unixfs.go b/coreiface/tests/unixfs.go index e0c37fce4..31ac1b5c9 100644 --- a/coreiface/tests/unixfs.go +++ b/coreiface/tests/unixfs.go @@ -408,7 +408,7 @@ func (tp *TestSuite) TestAdd(t *testing.T) { t.Errorf("Event.Name didn't match, %s != %s", expected[0].Name, event.Name) } - if expected[0].Path != nil && event.Path != nil { + if (expected[0].Path != path.ImmutablePath{} && event.Path != path.ImmutablePath{}) { if expected[0].Path.RootCid().String() != event.Path.RootCid().String() { t.Errorf("Event.Hash didn't match, %s != %s", expected[0].Path, event.Path) } diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index 6868d3247..db5f68924 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -595,16 +595,16 @@ func (bb *BlocksBackend) getPathRoots(ctx context.Context, contentPath path.Immu sp.WriteString(root) p, err := path.NewPath(sp.String()) if err != nil { - return nil, nil, nil, err + return nil, path.ImmutablePath{}, nil, err } resolvedSubPath, remainderSubPath, err := bb.resolvePath(ctx, p) if err != nil { // TODO: should we be more explicit here and is this part of the IPFSBackend contract? // The issue here was that we returned datamodel.ErrWrongKind instead of this resolver error if isErrNotFound(err) { - return nil, nil, nil, &resolver.ErrNoLink{Name: root, Node: lastPath.RootCid()} + return nil, path.ImmutablePath{}, nil, &resolver.ErrNoLink{Name: root, Node: lastPath.RootCid()} } - return nil, nil, nil, err + return nil, path.ImmutablePath{}, nil, err } lastPath = resolvedSubPath remainder = remainderSubPath @@ -620,13 +620,13 @@ func (bb *BlocksBackend) ResolveMutable(ctx context.Context, p path.Path) (path. case path.IPNSNamespace: p, err := resolve.ResolveIPNS(ctx, bb.namesys, p) if err != nil { - return nil, err + return path.ImmutablePath{}, err } return path.NewImmutablePath(p) case path.IPFSNamespace: return path.NewImmutablePath(p) default: - return nil, NewErrorStatusCode(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) + return path.ImmutablePath{}, NewErrorStatusCode(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) } } @@ -690,32 +690,32 @@ func (bb *BlocksBackend) resolvePath(ctx context.Context, p path.Path) (path.Imm if p.Namespace() == path.IPNSNamespace { p, err = resolve.ResolveIPNS(ctx, bb.namesys, p) if err != nil { - return nil, nil, err + return path.ImmutablePath{}, nil, err } } if p.Namespace() != path.IPFSNamespace { - return nil, nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace()) + return path.ImmutablePath{}, nil, fmt.Errorf("unsupported path namespace: %s", p.Namespace()) } imPath, err := path.NewImmutablePath(p) if err != nil { - return nil, nil, err + return path.ImmutablePath{}, nil, err } node, remainder, err := bb.resolver.ResolveToLastNode(ctx, imPath) if err != nil { - return nil, nil, err + return path.ImmutablePath{}, nil, err } p, err = path.Join(path.FromCid(node), remainder...) if err != nil { - return nil, nil, err + return path.ImmutablePath{}, nil, err } imPath, err = path.NewImmutablePath(p) if err != nil { - return nil, nil, err + return path.ImmutablePath{}, nil, err } return imPath, remainder, nil diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index 775a4e24b..d3a4ac9b5 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -736,7 +736,7 @@ func (mb *errorMockBackend) GetCAR(ctx context.Context, path path.ImmutablePath, } func (mb *errorMockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { - return nil, mb.err + return path.ImmutablePath{}, mb.err } func (mb *errorMockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { diff --git a/gateway/handler.go b/gateway/handler.go index 6bcd515f9..7e3ccbf48 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -745,14 +745,14 @@ func (i *handler) handleWebRequestErrors(w http.ResponseWriter, r *http.Request, if errors.Is(err, ErrServiceUnavailable) { err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) i.webError(w, r, err, http.StatusServiceUnavailable) - return nil, false + return path.ImmutablePath{}, false } // If the error is not an IPLD traversal error then we should not be looking for _redirects or legacy 404s if !isErrNotFound(err) { err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) i.webError(w, r, err, http.StatusInternalServerError) - return nil, false + return path.ImmutablePath{}, false } // If we have origin isolation (subdomain gw, DNSLink website), @@ -774,12 +774,12 @@ func (i *handler) handleWebRequestErrors(w http.ResponseWriter, r *http.Request, // follow https://docs.ipfs.tech/how-to/websites-on-ipfs/redirects-and-custom-404s/ instead. if i.serveLegacy404IfPresent(w, r, immutableContentPath, logger) { logger.Debugw("served legacy 404") - return nil, false + return path.ImmutablePath{}, false } err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) i.webError(w, r, err, http.StatusInternalServerError) - return nil, false + return path.ImmutablePath{}, false } // Detect 'Cache-Control: only-if-cached' in request and return data if it is already in the local datastore. diff --git a/gateway/handler_unixfs__redirects.go b/gateway/handler_unixfs__redirects.go index 6b63a793a..6d1f40c25 100644 --- a/gateway/handler_unixfs__redirects.go +++ b/gateway/handler_unixfs__redirects.go @@ -42,28 +42,28 @@ func (i *handler) serveRedirectsIfPresent(w http.ResponseWriter, r *http.Request if err != nil { err = fmt.Errorf("trouble processing _redirects path %q: %w", immutableContentPath.String(), err) i.webError(w, r, err, http.StatusInternalServerError) - return nil, false, true + return path.ImmutablePath{}, false, true } redirectsPath, err := path.Join(rootPath, "_redirects") if err != nil { err = fmt.Errorf("trouble processing _redirects path %q: %w", rootPath.String(), err) i.webError(w, r, err, http.StatusInternalServerError) - return nil, false, true + return path.ImmutablePath{}, false, true } imRedirectsPath, err := path.NewImmutablePath(redirectsPath) if err != nil { err = fmt.Errorf("trouble processing _redirects path %q: %w", redirectsPath, err) i.webError(w, r, err, http.StatusInternalServerError) - return nil, false, true + return path.ImmutablePath{}, false, true } foundRedirect, redirectRules, err := i.getRedirectRules(r, imRedirectsPath) if err != nil { err = fmt.Errorf("trouble processing _redirects file at %q: %w", redirectsPath, err) i.webError(w, r, err, http.StatusInternalServerError) - return nil, false, true + return path.ImmutablePath{}, false, true } if foundRedirect { @@ -71,11 +71,11 @@ func (i *handler) serveRedirectsIfPresent(w http.ResponseWriter, r *http.Request if err != nil { err = fmt.Errorf("trouble processing _redirects file at %q: %w", redirectsPath, err) i.webError(w, r, err, http.StatusInternalServerError) - return nil, false, true + return path.ImmutablePath{}, false, true } if redirected { - return nil, false, true + return path.ImmutablePath{}, false, true } // 200 is treated as a rewrite, so update the path and continue @@ -85,13 +85,13 @@ func (i *handler) serveRedirectsIfPresent(w http.ResponseWriter, r *http.Request if err != nil { err = fmt.Errorf("could not use _redirects file to %q: %w", p, err) i.webError(w, r, err, http.StatusInternalServerError) - return nil, false, true + return path.ImmutablePath{}, false, true } imPath, err := path.NewImmutablePath(p) if err != nil { err = fmt.Errorf("could not use _redirects file to %q: %w", p, err) i.webError(w, r, err, http.StatusInternalServerError) - return nil, false, true + return path.ImmutablePath{}, false, true } return imPath, true, true } diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index 89f955cbc..bba66d94d 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -186,18 +186,18 @@ func (mb *mockBackend) resolvePathNoRootsReturned(ctx context.Context, ip path.P if ip.Mutable() { imPath, err = mb.ResolveMutable(ctx, ip) if err != nil { - return nil, err + return path.ImmutablePath{}, err } } else { imPath, err = path.NewImmutablePath(ip) if err != nil { - return nil, err + return path.ImmutablePath{}, err } } md, err := mb.ResolvePath(ctx, imPath) if err != nil { - return nil, err + return path.ImmutablePath{}, err } return md.LastSegment, nil } diff --git a/path/path.go b/path/path.go index 5e39eea3a..8b67af4d0 100644 --- a/path/path.go +++ b/path/path.go @@ -73,58 +73,50 @@ func (p path) Segments() []string { } // ImmutablePath is a [Path] which is guaranteed to have an immutable [Namespace]. -type ImmutablePath interface { - Path - - // RootCid returns the [cid.Cid] of the root object of the path. - RootCid() cid.Cid -} - -var _ Path = immutablePath{} -var _ ImmutablePath = immutablePath{} - -type immutablePath struct { +type ImmutablePath struct { path Path rootCid cid.Cid } +var _ Path = ImmutablePath{} + func NewImmutablePath(p Path) (ImmutablePath, error) { if p.Mutable() { - return nil, &ErrInvalidPath{err: ErrExpectedImmutable, path: p.String()} + return ImmutablePath{}, &ErrInvalidPath{err: ErrExpectedImmutable, path: p.String()} } segments := p.Segments() cid, err := cid.Decode(segments[1]) if err != nil { - return nil, &ErrInvalidPath{err: err, path: p.String()} + return ImmutablePath{}, &ErrInvalidPath{err: err, path: p.String()} } - return immutablePath{path: p, rootCid: cid}, nil + return ImmutablePath{path: p, rootCid: cid}, nil } -func (ip immutablePath) String() string { +func (ip ImmutablePath) String() string { return ip.path.String() } -func (ip immutablePath) Namespace() string { +func (ip ImmutablePath) Namespace() string { return ip.path.Namespace() } -func (ip immutablePath) Mutable() bool { +func (ip ImmutablePath) Mutable() bool { return false } -func (ip immutablePath) Segments() []string { +func (ip ImmutablePath) Segments() []string { return ip.path.Segments() } -func (ip immutablePath) RootCid() cid.Cid { +func (ip ImmutablePath) RootCid() cid.Cid { return ip.rootCid } // FromCid returns a new "/ipfs" path with the provided CID. func FromCid(cid cid.Cid) ImmutablePath { - return immutablePath{ + return ImmutablePath{ path: path{ str: fmt.Sprintf("/%s/%s", IPFSNamespace, cid.String()), namespace: IPFSNamespace, @@ -161,7 +153,7 @@ func NewPath(str string) (Path, error) { return nil, &ErrInvalidPath{err: err, path: str} } - return immutablePath{ + return ImmutablePath{ path: path{ str: cleaned, namespace: segments[0], diff --git a/path/path_test.go b/path/path_test.go index 2f944e39e..b432f11a9 100644 --- a/path/path_test.go +++ b/path/path_test.go @@ -9,7 +9,7 @@ import ( ) func newIPLDPath(cid cid.Cid) ImmutablePath { - return immutablePath{ + return ImmutablePath{ path: path{ str: fmt.Sprintf("/%s/%s", IPLDNamespace, cid.String()), namespace: IPLDNamespace, @@ -134,7 +134,7 @@ func TestNewPath(t *testing.T) { for _, testCase := range testCases { p, err := NewPath(testCase.src) assert.NoError(t, err) - assert.IsType(t, immutablePath{}, p) + assert.IsType(t, ImmutablePath{}, p) } }) } @@ -149,7 +149,7 @@ func TestFromCid(t *testing.T) { assert.NoError(t, err) p := FromCid(c) - assert.IsType(t, immutablePath{}, p) + assert.IsType(t, ImmutablePath{}, p) assert.Equal(t, "/ipfs/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", p.String()) assert.Equal(t, c, p.RootCid()) }) @@ -161,7 +161,7 @@ func TestFromCid(t *testing.T) { assert.NoError(t, err) p := FromCid(c) - assert.IsType(t, immutablePath{}, p) + assert.IsType(t, ImmutablePath{}, p) assert.Equal(t, "/ipfs/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", p.String()) assert.Equal(t, c, p.RootCid()) }) @@ -171,7 +171,7 @@ func TestFromCid(t *testing.T) { assert.NoError(t, err) p := newIPLDPath(c) - assert.IsType(t, immutablePath{}, p) + assert.IsType(t, ImmutablePath{}, p) assert.Equal(t, "/ipld/QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n", p.String()) assert.Equal(t, c, p.RootCid()) @@ -180,7 +180,7 @@ func TestFromCid(t *testing.T) { assert.NoError(t, err) p = newIPLDPath(c) - assert.IsType(t, immutablePath{}, p) + assert.IsType(t, ImmutablePath{}, p) assert.Equal(t, "/ipld/bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", p.String()) assert.Equal(t, c, p.RootCid()) }) From 25ae0bd151b43ff3ec56aac44fc8a03337a72f7b Mon Sep 17 00:00:00 2001 From: xiaolou86 Date: Wed, 11 Oct 2023 16:59:37 +0800 Subject: [PATCH 10/63] chore: fix typos [skip changelog] (#483) --- .../blockpresencemanager/blockpresencemanager_test.go | 2 +- bitswap/client/internal/getter/getter.go | 2 +- .../providerquerymanager/providerquerymanager_test.go | 2 +- bitswap/metrics/metrics.go | 4 ++-- bitswap/server/internal/decision/engine_test.go | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bitswap/client/internal/blockpresencemanager/blockpresencemanager_test.go b/bitswap/client/internal/blockpresencemanager/blockpresencemanager_test.go index b0616e4e3..b977c28ff 100644 --- a/bitswap/client/internal/blockpresencemanager/blockpresencemanager_test.go +++ b/bitswap/client/internal/blockpresencemanager/blockpresencemanager_test.go @@ -218,7 +218,7 @@ func TestAllPeersDoNotHaveBlock(t *testing.T) { {[]peer.ID{p1}, []cid.Cid{c2}, []cid.Cid{}}, {[]peer.ID{p2}, []cid.Cid{c2}, []cid.Cid{c2}}, - // p0 recieved DONT_HAVE for c1 & c2 (but not for c0) + // p0 received DONT_HAVE for c1 & c2 (but not for c0) {[]peer.ID{p0}, []cid.Cid{c0, c1, c2}, []cid.Cid{c1, c2}}, {[]peer.ID{p0, p1}, []cid.Cid{c0, c1, c2}, []cid.Cid{}}, // Both p0 and p2 received DONT_HAVE for c2 diff --git a/bitswap/client/internal/getter/getter.go b/bitswap/client/internal/getter/getter.go index 713394b08..822d319b7 100644 --- a/bitswap/client/internal/getter/getter.go +++ b/bitswap/client/internal/getter/getter.go @@ -21,7 +21,7 @@ type GetBlocksFunc func(context.Context, []cid.Cid) (<-chan blocks.Block, error) // SyncGetBlock takes a block cid and an async function for getting several // blocks that returns a channel, and uses that function to return the -// block syncronously. +// block synchronously. func SyncGetBlock(p context.Context, k cid.Cid, gb GetBlocksFunc) (blocks.Block, error) { p, span := internal.StartSpan(p, "Getter.SyncGetBlock") defer span.End() diff --git a/bitswap/client/internal/providerquerymanager/providerquerymanager_test.go b/bitswap/client/internal/providerquerymanager/providerquerymanager_test.go index afdd19595..52447e2c1 100644 --- a/bitswap/client/internal/providerquerymanager/providerquerymanager_test.go +++ b/bitswap/client/internal/providerquerymanager/providerquerymanager_test.go @@ -281,7 +281,7 @@ func TestRateLimitingRequests(t *testing.T) { defer fpn.queriesMadeMutex.Unlock() if fpn.queriesMade != maxInProcessRequests+1 { t.Logf("Queries made: %d\n", fpn.queriesMade) - t.Fatal("Did not make all seperate requests") + t.Fatal("Did not make all separate requests") } } diff --git a/bitswap/metrics/metrics.go b/bitswap/metrics/metrics.go index b71923727..e40276842 100644 --- a/bitswap/metrics/metrics.go +++ b/bitswap/metrics/metrics.go @@ -14,11 +14,11 @@ var ( ) func DupHist(ctx context.Context) metrics.Histogram { - return metrics.NewCtx(ctx, "recv_dup_blocks_bytes", "Summary of duplicate data blocks recived").Histogram(metricsBuckets) + return metrics.NewCtx(ctx, "recv_dup_blocks_bytes", "Summary of duplicate data blocks received").Histogram(metricsBuckets) } func AllHist(ctx context.Context) metrics.Histogram { - return metrics.NewCtx(ctx, "recv_all_blocks_bytes", "Summary of all data blocks recived").Histogram(metricsBuckets) + return metrics.NewCtx(ctx, "recv_all_blocks_bytes", "Summary of all data blocks received").Histogram(metricsBuckets) } func SentHist(ctx context.Context) metrics.Histogram { diff --git a/bitswap/server/internal/decision/engine_test.go b/bitswap/server/internal/decision/engine_test.go index efc4408ff..b7382f817 100644 --- a/bitswap/server/internal/decision/engine_test.go +++ b/bitswap/server/internal/decision/engine_test.go @@ -138,7 +138,7 @@ func TestConsistentAccounting(t *testing.T) { t.Fatal("Inconsistent book-keeping. Strategies don't agree") } - // Ensure sender didn't record receving anything. And that the receiver + // Ensure sender didn't record receiving anything. And that the receiver // didn't record sending anything if receiver.Engine.numBytesSentTo(sender.Peer) != 0 || sender.Engine.numBytesReceivedFrom(receiver.Peer) != 0 { t.Fatal("Bert didn't send bytes to Ernie") From 6602207a8fa3624bfd52d8859c544abdb1ef0324 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 12 Oct 2023 13:25:07 +0000 Subject: [PATCH 11/63] gateway/BlocksBackend: add option for custom Resolver (#484) --- CHANGELOG.md | 3 +++ gateway/blocks_backend.go | 18 ++++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 111044407..1f59cae02 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ The following emojis are used to highlight certain changes: ### Added +* `boxo/gateway`: + * A new `WithResolver(...)` option can be used with `NewBlocksBackend(...)` allowing the user to pass their custom `Resolver` implementation. + ### Changed * `boxo/gateway` diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index db5f68924..d4edb8959 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -67,6 +67,7 @@ var _ IPFSBackend = (*BlocksBackend)(nil) type blocksBackendOptions struct { ns namesys.NameSystem vs routing.ValueStore + r resolver.Resolver } // WithNameSystem sets the name system to use with the [BlocksBackend]. If not set @@ -87,6 +88,14 @@ func WithValueStore(vs routing.ValueStore) BlocksBackendOption { } } +// WithResolver sets the [resolver.Resolver] to use with the [BlocksBackend]. +func WithResolver(r resolver.Resolver) BlocksBackendOption { + return func(opts *blocksBackendOptions) error { + opts.r = r + return nil + } +} + type BlocksBackendOption func(options *blocksBackendOptions) error func NewBlocksBackend(blockService blockservice.BlockService, opts ...BlocksBackendOption) (*BlocksBackend, error) { @@ -108,13 +117,12 @@ func NewBlocksBackend(blockService blockservice.BlockService, opts ...BlocksBack } return basicnode.Prototype.Any, nil }) - fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) - r := resolver.NewBasicResolver(fetcher) // Setup a name system so that we are able to resolve /ipns links. var ( ns namesys.NameSystem vs routing.ValueStore + r resolver.Resolver ) vs = compiledOptions.vs @@ -135,6 +143,12 @@ func NewBlocksBackend(blockService blockservice.BlockService, opts ...BlocksBack } } + r = compiledOptions.r + if r == nil { + fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) + r = resolver.NewBasicResolver(fetcher) + } + return &BlocksBackend{ blockStore: blockService.Blockstore(), blockService: blockService, From 435daa2a6b8b613b79d120c7e3783b965f1af659 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Mon, 6 Mar 2023 13:51:50 +0100 Subject: [PATCH 12/63] bitswap/client: add option to disable duplicated block stats Keeping a counter should not require query-ing the blockstore repeteadly for every single block received, so I have added an option to disable it. Currently there is the assumption that doing Has() calls is cheap. That can be correct for most cases but also badly incorrect for others. --- bitswap/client/client.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/bitswap/client/client.go b/bitswap/client/client.go index 854d030d1..52ac96c5e 100644 --- a/bitswap/client/client.go +++ b/bitswap/client/client.go @@ -79,6 +79,19 @@ func WithBlockReceivedNotifier(brn BlockReceivedNotifier) Option { } } +// WithoutDuplicatedBlockStats disable collecting counts of duplicated blocks +// received. This counter requires triggering a blockstore.Has() call for +// every block received by launching goroutines in parallel. In the worst case +// (no caching/blooms etc), this is an expensive call for the datastore to +// answer. In a normal case (caching), this has the power of evicting a +// different block from intermediary caches. In the best case, it doesn't +// affect performance. Use if this stat is not relevant. +func WithoutDuplicatedBlockStats() Option { + return func(bs *Client) { + bs.skipDuplicatedBlocksStats = true + } +} + type BlockReceivedNotifier interface { // ReceivedBlocks notifies the decision engine that a peer is well-behaving // and gave us useful data, potentially increasing its score and making us @@ -226,6 +239,9 @@ type Client struct { // whether we should actually simulate dont haves on request timeout simulateDontHavesOnTimeout bool + + // dupMetric will stay at 0 + skipDuplicatedBlocksStats bool } type counters struct { @@ -373,7 +389,12 @@ func (bs *Client) updateReceiveCounters(blocks []blocks.Block) { // Check which blocks are in the datastore // (Note: any errors from the blockstore are simply logged out in // blockstoreHas()) - blocksHas := bs.blockstoreHas(blocks) + var blocksHas []bool + if !bs.skipDuplicatedBlocksStats { + blocksHas = bs.blockstoreHas(blocks) + } else { + blocksHas = make([]bool, len(blocks)) + } bs.counterLk.Lock() defer bs.counterLk.Unlock() From 6bf5de7712734107407dd917f22f7d0ccd93f0f3 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 12 Oct 2023 18:50:52 +0200 Subject: [PATCH 13/63] bitswap/client: do not allocate an empty []bool... ... when duplicated block stats are disabled. --- bitswap/client/client.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bitswap/client/client.go b/bitswap/client/client.go index 52ac96c5e..2b4d76a54 100644 --- a/bitswap/client/client.go +++ b/bitswap/client/client.go @@ -392,8 +392,6 @@ func (bs *Client) updateReceiveCounters(blocks []blocks.Block) { var blocksHas []bool if !bs.skipDuplicatedBlocksStats { blocksHas = bs.blockstoreHas(blocks) - } else { - blocksHas = make([]bool, len(blocks)) } bs.counterLk.Lock() @@ -401,7 +399,7 @@ func (bs *Client) updateReceiveCounters(blocks []blocks.Block) { // Do some accounting for each block for i, b := range blocks { - has := blocksHas[i] + has := (blocksHas != nil) && blocksHas[i] blkLen := len(b.RawData()) bs.allMetric.Observe(float64(blkLen)) From 4e7a9a314c14d5e7eccd5ac253e93889e93941d5 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 12 Oct 2023 18:56:18 +0200 Subject: [PATCH 14/63] bitswap/client: update changelog for WithoutDuplicatedBlockStats() --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f59cae02..d8a66f2a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ The following emojis are used to highlight certain changes: * `boxo/gateway`: * A new `WithResolver(...)` option can be used with `NewBlocksBackend(...)` allowing the user to pass their custom `Resolver` implementation. +* `boxo/bitswap/client`: + * A new `WithoutDuplicatedBlockStats()` option can be used with `bitswap.New` and `bsclient.New`. This disable accounting for duplicated blocks, which requires a `blockstore.Has()` lookup for every received block and thus, can impact performance. ### Changed From 38b183c956a7b6dfdeee956ed68d7b4e9564713d Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 13 Oct 2023 16:20:12 +0200 Subject: [PATCH 15/63] gateway: simplify fetcher setup --- gateway/blocks_backend.go | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index d4edb8959..05b0d1e0b 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -109,15 +109,6 @@ func NewBlocksBackend(blockService blockservice.BlockService, opts ...BlocksBack // Setup the DAG services, which use the CAR block store. dagService := merkledag.NewDAGService(blockService) - // Setup the UnixFS resolver. - fetcherConfig := bsfetcher.NewFetcherConfig(blockService) - fetcherConfig.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) { - if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok { - return tlnkNd.LinkTargetNodePrototype(), nil - } - return basicnode.Prototype.Any, nil - }) - // Setup a name system so that we are able to resolve /ipns links. var ( ns namesys.NameSystem @@ -145,7 +136,10 @@ func NewBlocksBackend(blockService blockservice.BlockService, opts ...BlocksBack r = compiledOptions.r if r == nil { - fetcher := fetcherConfig.WithReifier(unixfsnode.Reify) + // Setup the UnixFS resolver. + fetcherCfg := bsfetcher.NewFetcherConfig(blockService) + fetcherCfg.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser) + fetcher := fetcherCfg.WithReifier(unixfsnode.Reify) r = resolver.NewBasicResolver(fetcher) } From 611b790d2bcaee755ac04da5551a0dc112289f3f Mon Sep 17 00:00:00 2001 From: tkzktk Date: Sun, 15 Oct 2023 01:13:23 -0400 Subject: [PATCH 16/63] docs: fix typos --- CHANGELOG.md | 2 +- RELEASE.md | 2 +- pinning/remote/client/openapi/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8a66f2a2..79ff0f4b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -263,7 +263,7 @@ None. - `InternalKeys` - 🛠 `provider/batched.New` has been moved to `provider.New` and arguments has been changed. (https://github.com/ipfs/boxo/pulls/273) - A routing system is now passed with the `provider.Online` option, by default the system run in offline mode (push stuff onto the queue). - - When using `provider.Online` calling the `.Run` method is not required anymore, the background worker is implicitely started in the background by `provider.New`. + - When using `provider.Online` calling the `.Run` method is not required anymore, the background worker is implicitly started in the background by `provider.New`. - You do not have to pass a queue anymore, you pass a `datastore.Datastore` exclusively. - 🛠 `provider.NewOfflineProvider` has been renamed to `provider.NewNoopProvider` to show more clearly that is does nothing. (https://github.com/ipfs/boxo/pulls/273) - 🛠 `provider.Provider` and `provider.Reprovider` has been merged under one `provider.System`. (https://github.com/ipfs/boxo/pulls/273) diff --git a/RELEASE.md b/RELEASE.md index deadd7231..3d6ac8400 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -27,7 +27,7 @@ The amount of backporting of a fix depends on the severity of the issue and the As a result, Boxo maintainers recommend that consumers stay up-to-date with Boxo releases. ### Go Compatibility -At any given point, the Go team supports only the latest two versions of Go released (see https://go.dev/doc/devel/release). Boxo maintainers will strive to maintain compatibilty with the older of the two supported versions, so that Boxo is also compatible with the latest two versions of Go. +At any given point, the Go team supports only the latest two versions of Go released (see https://go.dev/doc/devel/release). Boxo maintainers will strive to maintain compatibility with the older of the two supported versions, so that Boxo is also compatible with the latest two versions of Go. ### Release Criteria Boxo releases occur _at least_ on every Kubo release. Releases can also be initiated on-demand, regardless of Kubo's release cadence, whenever there are significant changes (new features, refactorings, deprecations, etc.). diff --git a/pinning/remote/client/openapi/README.md b/pinning/remote/client/openapi/README.md index fe21b9a00..eddae08b7 100644 --- a/pinning/remote/client/openapi/README.md +++ b/pinning/remote/client/openapi/README.md @@ -151,7 +151,7 @@ Note, enum values are always validated and all unused variables are silently ign ### URLs Configuration per Operation Each operation can use different server URL defined using `OperationServers` map in the `Configuration`. -An operation is uniquely identifield by `"{classname}Service.{nickname}"` string. +An operation is uniquely identified by `"{classname}Service.{nickname}"` string. Similar rules for overriding default operation server index and variables applies by using `sw.ContextOperationServerIndices` and `sw.ContextOperationServerVariables` context maps. ``` From 8660c625624d226bce2ca1ffccbb2b21c7e4132d Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Mon, 9 Oct 2023 10:32:56 +0200 Subject: [PATCH 17/63] docs: how to use and create examples --- examples/README.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index c0795436d..fa5408732 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,9 +1,29 @@ # Boxo Examples and Tutorials -In this folder, you can find some examples to help you get started using Boxo and its associated libraries in your applications. +In this directory, you can find some examples to help you get started using Boxo and its associated libraries in your applications. Let us know if you find any issue or if you want to contribute and add a new tutorial, feel welcome to submit a PR, thank you! +## How To Use the Examples + +The examples are designed to give users a starting point to create certain things using Boxo. However, this directory is not meant to be directly copied out of Boxo, since it uses a replacement directive in the [`go.mod`](go.mod) file. You can also not `go install`. + +If you want to copy of the examples out of Boxo in order to use it in your own product, you need to remove the replacement directive and ensure you're using the latest Boxo version: + +```bash +> go mod edit -dropreplace=github.com/ipfs/boxo +> go get github.com/ipfs/boxo@latest +> go mod tidy +``` + +## How To Create an Example + +All examples are self-contained inside the same module ([`examples`](go.mod)). To create an example, clone Boxo, navigate to this directory and create a sub-directory with a descriptive name for the example. If the example pertains a topic that has multiple examples, such as `gateway`, create a sub-directory there. + +The new example must contain a descriptive `README.md` file, which explains what the example is, how to build and use it. See the existing examples to have an idea of how extensive it should be. In addition, your code must be properly documented. + +Once you have your example finished, do not forget to run `go mod tidy` and adding a link to the example in the section [Examples and Tutorials](#examples-and-tutorials) below. + ## Examples and Tutorials - [Fetching a UnixFS file by CID](./unixfs-file-cid) From aa6fa14f1e1f53a7150df6658b71bede59e0da9d Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 17 Oct 2023 16:51:50 +0200 Subject: [PATCH 18/63] feat!: remove util.MultiErr --- CHANGELOG.md | 2 ++ util/util.go | 18 ------------------ 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79ff0f4b8..150d73d10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,8 @@ The following emojis are used to highlight certain changes: ### Removed +* 🛠 `util.MultiErr` has been removed. Please use Go's native support for wrapping errors, or `errors.Join` instead. + ### Fixed ### Security diff --git a/util/util.go b/util/util.go index 74cf1aaff..7a96ae393 100644 --- a/util/util.go +++ b/util/util.go @@ -89,24 +89,6 @@ func GetenvBool(name string) bool { return v == "true" || v == "t" || v == "1" } -// MultiErr is a util to return multiple errors -type MultiErr []error - -func (m MultiErr) Error() string { - if len(m) == 0 { - return "no errors" - } - - s := "Multiple errors: " - for i, e := range m { - if i != 0 { - s += ", " - } - s += e.Error() - } - return s -} - // Partition splits a subject 3 parts: prefix, separator, suffix. // The first occurrence of the separator will be matched. // ie. Partition("Ready, steady, go!", ", ") -> ["Ready", ", ", "steady, go!"] From 0a566c9ee75f123c91a5fb9df93e8c20afa97b10 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 17 Oct 2023 18:40:40 +0200 Subject: [PATCH 19/63] routing/http: return PeerRecord for FindPeers (#490) --- CHANGELOG.md | 1 + .../routing/delegated-routing-client/main.go | 54 +++++++++++++------ routing/http/client/client.go | 19 ++++--- routing/http/client/client_test.go | 12 ++--- routing/http/contentrouter/contentrouter.go | 31 ++++------- .../http/contentrouter/contentrouter_test.go | 13 ++--- routing/http/server/server.go | 10 ++-- routing/http/server/server_test.go | 8 +-- routing/http/types/json/responses.go | 2 +- routing/http/types/ndjson/records.go | 27 ++++++++++ 10 files changed, 109 insertions(+), 68 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 150d73d10..c1c02b94d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ The following emojis are used to highlight certain changes: - Eliminate `..` elements that begin a rooted path: that is, replace "`/..`" by "`/`" at the beginning of a path. * 🛠 The signature of `CoreAPI.ResolvePath` in `coreiface` has changed to now return the remainder segments as a second return value, matching the signature of `resolver.ResolveToLastNode`. +* 🛠 `routing/http/client.FindPeers` now returns `iter.ResultIter[types.PeerRecord]` instead of `iter.ResultIter[types.Record]`. The specification indicates that records for this method will always be Peer Records. ### Removed diff --git a/examples/routing/delegated-routing-client/main.go b/examples/routing/delegated-routing-client/main.go index 0b586d4bf..8fac342ac 100644 --- a/examples/routing/delegated-routing-client/main.go +++ b/examples/routing/delegated-routing-client/main.go @@ -69,22 +69,6 @@ func findProviders(w io.Writer, ctx context.Context, client *client.Client, cidS return printIter(w, recordsIter) } -func findPeers(w io.Writer, ctx context.Context, client *client.Client, pidStr string) error { - // Parses the given Peer ID to lookup the information for. - pid, err := peer.Decode(pidStr) - if err != nil { - return err - } - - // Ask for information about the peer with the given peer ID. - recordsIter, err := client.FindPeers(ctx, pid) - if err != nil { - return err - } - defer recordsIter.Close() - return printIter(w, recordsIter) -} - func printIter(w io.Writer, iter iter.ResultIter[types.Record]) error { // The response is streamed. Alternatively, you could use [iter.ReadAll] // to fetch all the results all at once, instead of iterating as they are @@ -118,6 +102,44 @@ func printIter(w io.Writer, iter iter.ResultIter[types.Record]) error { return nil } +func findPeers(w io.Writer, ctx context.Context, client *client.Client, pidStr string) error { + // Parses the given Peer ID to lookup the information for. + pid, err := peer.Decode(pidStr) + if err != nil { + return err + } + + // Ask for information about the peer with the given peer ID. + recordsIter, err := client.FindPeers(ctx, pid) + if err != nil { + return err + } + defer recordsIter.Close() + + // The response is streamed. Alternatively, you could use [iter.ReadAll] + // to fetch all the results all at once, instead of iterating as they are + // streamed. + for recordsIter.Next() { + res := recordsIter.Val() + + // Check for error, but do not complain if we exceeded the timeout. We are + // expecting that to happen: we explicitly defined a timeout. + if res.Err != nil { + if !errors.Is(res.Err, context.DeadlineExceeded) { + return res.Err + } + + return nil + } + + fmt.Fprintln(w, res.Val.ID) + fmt.Fprintln(w, "\tProtocols:", res.Val.Protocols) + fmt.Fprintln(w, "\tAddresses:", res.Val.Addrs) + } + + return nil +} + func findIPNS(w io.Writer, ctx context.Context, client *client.Client, nameStr string) error { // Parses the given name string to get a record for. name, err := ipns.NameFromString(nameStr) diff --git a/routing/http/client/client.go b/routing/http/client/client.go index 39dd698cb..cda9097fd 100644 --- a/routing/http/client/client.go +++ b/routing/http/client/client.go @@ -160,6 +160,8 @@ func (c *measuringIter[T]) Close() error { return c.Iter.Close() } +// FindProviders searches for providers that are able to provide the given [cid.Cid]. +// In a more generic way, it is also used as a mapping between CIDs and relevant metadata. func (c *Client) FindProviders(ctx context.Context, key cid.Cid) (providers iter.ResultIter[types.Record], err error) { // TODO test measurements m := newMeasurement("FindProviders") @@ -332,7 +334,8 @@ func (c *Client) provideSignedBitswapRecord(ctx context.Context, bswp *types.Wri return 0, nil } -func (c *Client) FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) { +// FindPeers searches for information for the given [peer.ID]. +func (c *Client) FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[*types.PeerRecord], err error) { m := newMeasurement("FindPeers") url := c.baseURL + "/routing/v1/peers/" + peer.ToCid(pid).String() @@ -359,7 +362,7 @@ func (c *Client) FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultI if resp.StatusCode == http.StatusNotFound { resp.Body.Close() m.record(ctx) - return iter.FromSlice[iter.Result[types.Record]](nil), nil + return iter.FromSlice[iter.Result[*types.PeerRecord]](nil), nil } if resp.StatusCode != http.StatusOK { @@ -387,24 +390,27 @@ func (c *Client) FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultI } }() - var it iter.ResultIter[types.Record] + var it iter.ResultIter[*types.PeerRecord] switch mediaType { case mediaTypeJSON: parsedResp := &jsontypes.PeersResponse{} err = json.NewDecoder(resp.Body).Decode(parsedResp) - var sliceIt iter.Iter[types.Record] = iter.FromSlice(parsedResp.Peers) + var sliceIt iter.Iter[*types.PeerRecord] = iter.FromSlice(parsedResp.Peers) it = iter.ToResultIter(sliceIt) case mediaTypeNDJSON: skipBodyClose = true - it = ndjson.NewRecordsIter(resp.Body) + it = ndjson.NewPeerRecordsIter(resp.Body) default: logger.Errorw("unknown media type", "MediaType", mediaType, "ContentType", respContentType) return nil, errors.New("unknown content type") } - return &measuringIter[iter.Result[types.Record]]{Iter: it, ctx: ctx, m: m}, nil + return &measuringIter[iter.Result[*types.PeerRecord]]{Iter: it, ctx: ctx, m: m}, nil } +// GetIPNS tries to retrieve the [ipns.Record] for the given [ipns.Name]. The record is +// validated against the given name. If validation fails, an error is returned, but no +// record. func (c *Client) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { url := c.baseURL + "/routing/v1/ipns/" + name.String() @@ -443,6 +449,7 @@ func (c *Client) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, err return record, nil } +// PutIPNS attempts at putting the given [ipns.Record] for the given [ipns.Name]. func (c *Client) PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error { url := c.baseURL + "/routing/v1/ipns/" + name.String() diff --git a/routing/http/client/client_test.go b/routing/http/client/client_test.go index b6aa8456b..7edd77c10 100644 --- a/routing/http/client/client_test.go +++ b/routing/http/client/client_test.go @@ -42,9 +42,9 @@ func (m *mockContentRouter) ProvideBitswap(ctx context.Context, req *server.Bits return args.Get(0).(time.Duration), args.Error(1) } -func (m *mockContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) { +func (m *mockContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[*types.PeerRecord], error) { args := m.Called(ctx, pid, limit) - return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) + return args.Get(0).(iter.ResultIter[*types.PeerRecord]), args.Error(1) } func (m *mockContentRouter) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { @@ -486,7 +486,7 @@ func TestClient_Provide(t *testing.T) { func TestClient_FindPeers(t *testing.T) { peerRecord := makePeerRecord() - peerRecords := []iter.Result[types.Record]{ + peerRecords := []iter.Result[*types.PeerRecord]{ {Val: &peerRecord}, } pid := *peerRecord.ID @@ -495,13 +495,13 @@ func TestClient_FindPeers(t *testing.T) { name string httpStatusCode int stopServer bool - routerResult []iter.Result[types.Record] + routerResult []iter.Result[*types.PeerRecord] routerErr error clientRequiresStreaming bool serverStreamingDisabled bool expErrContains osErrContains - expResult []iter.Result[types.Record] + expResult []iter.Result[*types.PeerRecord] expStreamingResponse bool expJSONResponse bool }{ @@ -606,7 +606,7 @@ func TestClient_FindPeers(t *testing.T) { resultIter, err := client.FindPeers(ctx, pid) c.expErrContains.errContains(t, err) - results := iter.ReadAll[iter.Result[types.Record]](resultIter) + results := iter.ReadAll[iter.Result[*types.PeerRecord]](resultIter) assert.Equal(t, c.expResult, results) }) } diff --git a/routing/http/contentrouter/contentrouter.go b/routing/http/contentrouter/contentrouter.go index 2438d4fea..9115ef154 100644 --- a/routing/http/contentrouter/contentrouter.go +++ b/routing/http/contentrouter/contentrouter.go @@ -26,7 +26,7 @@ const ttl = 24 * time.Hour type Client interface { FindProviders(ctx context.Context, key cid.Cid) (iter.ResultIter[types.Record], error) ProvideBitswap(ctx context.Context, keys []cid.Cid, ttl time.Duration) (time.Duration, error) - FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[types.Record], err error) + FindPeers(ctx context.Context, pid peer.ID) (peers iter.ResultIter[*types.PeerRecord], err error) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) PutIPNS(ctx context.Context, name ipns.Name, record *ipns.Record) error } @@ -196,28 +196,15 @@ func (c *contentRouter) FindPeer(ctx context.Context, pid peer.ID) (peer.AddrInf logger.Warnw("error iterating provider responses: %s", res.Err) continue } - v := res.Val - if v.GetSchema() == types.SchemaPeer { - result, ok := v.(*types.PeerRecord) - if !ok { - logger.Errorw( - "problem casting find providers result", - "Schema", v.GetSchema(), - "Type", reflect.TypeOf(v).String(), - ) - continue - } - - var addrs []multiaddr.Multiaddr - for _, a := range result.Addrs { - addrs = append(addrs, a.Multiaddr) - } - - return peer.AddrInfo{ - ID: *result.ID, - Addrs: addrs, - }, nil + var addrs []multiaddr.Multiaddr + for _, a := range res.Val.Addrs { + addrs = append(addrs, a.Multiaddr) } + + return peer.AddrInfo{ + ID: *res.Val.ID, + Addrs: addrs, + }, nil } return peer.AddrInfo{}, err diff --git a/routing/http/contentrouter/contentrouter_test.go b/routing/http/contentrouter/contentrouter_test.go index 2147dc975..1c47850b9 100644 --- a/routing/http/contentrouter/contentrouter_test.go +++ b/routing/http/contentrouter/contentrouter_test.go @@ -32,9 +32,9 @@ func (m *mockClient) FindProviders(ctx context.Context, key cid.Cid) (iter.Resul return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) } -func (m *mockClient) FindPeers(ctx context.Context, pid peer.ID) (iter.ResultIter[types.Record], error) { +func (m *mockClient) FindPeers(ctx context.Context, pid peer.ID) (iter.ResultIter[*types.PeerRecord], error) { args := m.Called(ctx, pid) - return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) + return args.Get(0).(iter.ResultIter[*types.PeerRecord]), args.Error(1) } func (m *mockClient) Ready(ctx context.Context) (bool, error) { @@ -183,17 +183,14 @@ func TestFindPeer(t *testing.T) { crc := NewContentRoutingClient(client) p1 := peer.ID("peer1") - ais := []types.Record{ - &types.UnknownRecord{ - Schema: "unknown", - }, - &types.PeerRecord{ + ais := []*types.PeerRecord{ + { Schema: types.SchemaPeer, ID: &p1, Protocols: []string{"transport-bitswap"}, }, } - aisIter := iter.ToResultIter[types.Record](iter.FromSlice(ais)) + aisIter := iter.ToResultIter[*types.PeerRecord](iter.FromSlice(ais)) client.On("FindPeers", ctx, p1).Return(aisIter, nil) diff --git a/routing/http/server/server.go b/routing/http/server/server.go index 9e7d81a04..d9be47eb2 100644 --- a/routing/http/server/server.go +++ b/routing/http/server/server.go @@ -63,7 +63,7 @@ type ContentRouter interface { // FindPeers searches for peers who have the provided [peer.ID]. // Limit indicates the maximum amount of results to return; 0 means unbounded. - FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) + FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[*types.PeerRecord], error) // GetIPNS searches for an [ipns.Record] for the given [ipns.Name]. GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) @@ -267,7 +267,7 @@ func (s *server) findPeers(w http.ResponseWriter, r *http.Request) { } var ( - handlerFunc func(w http.ResponseWriter, provIter iter.ResultIter[types.Record]) + handlerFunc func(w http.ResponseWriter, provIter iter.ResultIter[*types.PeerRecord]) recordsLimit int ) @@ -347,7 +347,7 @@ func (s *server) provide(w http.ResponseWriter, httpReq *http.Request) { writeJSONResult(w, "Provide", resp) } -func (s *server) findPeersJSON(w http.ResponseWriter, peersIter iter.ResultIter[types.Record]) { +func (s *server) findPeersJSON(w http.ResponseWriter, peersIter iter.ResultIter[*types.PeerRecord]) { defer peersIter.Close() peers, err := iter.ReadAllResults(peersIter) @@ -361,7 +361,7 @@ func (s *server) findPeersJSON(w http.ResponseWriter, peersIter iter.ResultIter[ }) } -func (s *server) findPeersNDJSON(w http.ResponseWriter, peersIter iter.ResultIter[types.Record]) { +func (s *server) findPeersNDJSON(w http.ResponseWriter, peersIter iter.ResultIter[*types.PeerRecord]) { writeResultsIterNDJSON(w, peersIter) } @@ -491,7 +491,7 @@ func logErr(method, msg string, err error) { logger.Infow(msg, "Method", method, "Error", err) } -func writeResultsIterNDJSON(w http.ResponseWriter, resultIter iter.ResultIter[types.Record]) { +func writeResultsIterNDJSON[T any](w http.ResponseWriter, resultIter iter.ResultIter[T]) { defer resultIter.Close() w.Header().Set("Content-Type", mediaTypeNDJSON) diff --git a/routing/http/server/server_test.go b/routing/http/server/server_test.go index c2c752057..f823ac25a 100644 --- a/routing/http/server/server_test.go +++ b/routing/http/server/server_test.go @@ -168,7 +168,7 @@ func TestPeers(t *testing.T) { t.Parallel() _, pid := makePeerID(t) - results := iter.FromSlice([]iter.Result[types.Record]{ + results := iter.FromSlice([]iter.Result[*types.PeerRecord]{ {Val: &types.PeerRecord{ Schema: types.SchemaPeer, ID: &pid, @@ -203,7 +203,7 @@ func TestPeers(t *testing.T) { t.Parallel() _, pid := makePeerID(t) - results := iter.FromSlice([]iter.Result[types.Record]{ + results := iter.FromSlice([]iter.Result[*types.PeerRecord]{ {Val: &types.PeerRecord{ Schema: types.SchemaPeer, ID: &pid, @@ -374,9 +374,9 @@ func (m *mockContentRouter) ProvideBitswap(ctx context.Context, req *BitswapWrit return args.Get(0).(time.Duration), args.Error(1) } -func (m *mockContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[types.Record], error) { +func (m *mockContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[*types.PeerRecord], error) { args := m.Called(ctx, pid, limit) - return args.Get(0).(iter.ResultIter[types.Record]), args.Error(1) + return args.Get(0).(iter.ResultIter[*types.PeerRecord]), args.Error(1) } func (m *mockContentRouter) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) { diff --git a/routing/http/types/json/responses.go b/routing/http/types/json/responses.go index dfcfad830..cc687df48 100644 --- a/routing/http/types/json/responses.go +++ b/routing/http/types/json/responses.go @@ -13,7 +13,7 @@ type ProvidersResponse struct { // PeersResponse is the result of a GET Peers request. type PeersResponse struct { - Peers RecordsArray + Peers []*types.PeerRecord } // RecordsArray is an array of [types.Record] diff --git a/routing/http/types/ndjson/records.go b/routing/http/types/ndjson/records.go index d1a36b411..819cd521d 100644 --- a/routing/http/types/ndjson/records.go +++ b/routing/http/types/ndjson/records.go @@ -44,3 +44,30 @@ func NewRecordsIter(r io.Reader) iter.Iter[iter.Result[types.Record]] { return iter.Map[iter.Result[types.UnknownRecord]](jsonIter, mapFn) } + +// NewPeerRecordsIter returns an iterator that reads [types.PeerRecord] from the given [io.Reader]. +// Records with a different schema are safely ignored. If you want to read all records, use +// [NewRecordsIter] instead. +func NewPeerRecordsIter(r io.Reader) iter.Iter[iter.Result[*types.PeerRecord]] { + jsonIter := iter.FromReaderJSON[types.UnknownRecord](r) + mapFn := func(upr iter.Result[types.UnknownRecord]) iter.Result[*types.PeerRecord] { + var result iter.Result[*types.PeerRecord] + if upr.Err != nil { + result.Err = upr.Err + return result + } + switch upr.Val.Schema { + case types.SchemaPeer: + var prov types.PeerRecord + err := json.Unmarshal(upr.Val.Bytes, &prov) + if err != nil { + result.Err = err + return result + } + result.Val = &prov + } + return result + } + + return iter.Map[iter.Result[types.UnknownRecord]](jsonIter, mapFn) +} From a50f784985ddfc25ec8d25aced2ca058469a3390 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 18 Oct 2023 10:12:37 +0200 Subject: [PATCH 20/63] feat!: namesys refactor, ipns TTL bubbled up to gateway (#459) --- CHANGELOG.md | 16 ++ coreiface/options/name.go | 6 +- coreiface/options/namesys/opts.go | 131 ------------ gateway/blocks_backend.go | 46 +++-- gateway/gateway.go | 5 +- gateway/gateway_test.go | 64 ++++-- gateway/handler.go | 35 ++-- gateway/handler_block.go | 2 +- gateway/handler_car.go | 2 +- gateway/handler_codec.go | 2 +- gateway/handler_defaults.go | 12 +- gateway/handler_tar.go | 2 +- gateway/handler_unixfs__redirects.go | 3 +- gateway/handler_unixfs_dir.go | 36 ++-- gateway/handler_unixfs_dir_test.go | 2 +- gateway/handler_unixfs_file.go | 8 +- gateway/hostname_test.go | 4 +- gateway/metrics.go | 6 +- gateway/utilities_test.go | 58 ++++-- ipns/defaults.go | 17 ++ ipns/name.go | 14 +- namesys/base.go | 126 ------------ namesys/cache.go | 62 ------ namesys/{dns.go => dns_resolver.go} | 103 ++++------ namesys/dns_resolver_test.go | 197 ++++++++++++++++++ namesys/dns_test.go | 184 ----------------- namesys/interface.go | 234 +++++++++++++++++----- namesys/ipns_publisher.go | 283 ++++++++++++++++++++++++++ namesys/ipns_publisher_test.go | 121 +++++++++++ namesys/ipns_resolver.go | 158 +++++++++++++++ namesys/ipns_resolver_test.go | 131 ++++++++++++ namesys/namesys.go | 231 ++++++++++----------- namesys/namesys_cache.go | 78 ++++++++ namesys/namesys_test.go | 184 ++++++++--------- namesys/publisher.go | 287 --------------------------- namesys/publisher_test.go | 159 --------------- namesys/republisher/repub.go | 56 +++--- namesys/republisher/repub_test.go | 103 ++++------ namesys/resolve/resolve.go | 56 ------ namesys/resolve_test.go | 132 ------------ namesys/routing.go | 147 -------------- namesys/tracing.go | 13 -- namesys/utilities.go | 146 ++++++++++++++ 43 files changed, 1819 insertions(+), 1843 deletions(-) delete mode 100644 coreiface/options/namesys/opts.go create mode 100644 ipns/defaults.go delete mode 100644 namesys/base.go delete mode 100644 namesys/cache.go rename namesys/{dns.go => dns_resolver.go} (58%) create mode 100644 namesys/dns_resolver_test.go delete mode 100644 namesys/dns_test.go create mode 100644 namesys/ipns_publisher.go create mode 100644 namesys/ipns_publisher_test.go create mode 100644 namesys/ipns_resolver.go create mode 100644 namesys/ipns_resolver_test.go create mode 100644 namesys/namesys_cache.go delete mode 100644 namesys/publisher.go delete mode 100644 namesys/publisher_test.go delete mode 100644 namesys/resolve/resolve.go delete mode 100644 namesys/resolve_test.go delete mode 100644 namesys/routing.go delete mode 100644 namesys/tracing.go create mode 100644 namesys/utilities.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c1c02b94d..71971240a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ The following emojis are used to highlight certain changes: * `boxo/gateway`: * A new `WithResolver(...)` option can be used with `NewBlocksBackend(...)` allowing the user to pass their custom `Resolver` implementation. + * The gateway now sets a `Cache-Control` header for requests under the `/ipns/` namespace if the TTL for the corresponding IPNS Records or DNSLink entities is known. * `boxo/bitswap/client`: * A new `WithoutDuplicatedBlockStats()` option can be used with `bitswap.New` and `bsclient.New`. This disable accounting for duplicated blocks, which requires a `blockstore.Has()` lookup for every received block and thus, can impact performance. @@ -41,6 +42,21 @@ The following emojis are used to highlight certain changes: * 🛠 The signature of `CoreAPI.ResolvePath` in `coreiface` has changed to now return the remainder segments as a second return value, matching the signature of `resolver.ResolveToLastNode`. * 🛠 `routing/http/client.FindPeers` now returns `iter.ResultIter[types.PeerRecord]` instead of `iter.ResultIter[types.Record]`. The specification indicates that records for this method will always be Peer Records. +* 🛠 The `namesys` package has been refactored. The following are the largest modifications: + * The options in `coreiface/options/namesys` have been moved to `namesys` and their names + have been made more consistent. + * Many of the exported structs and functions have been renamed in order to be consistent with + the remaining packages. + * `namesys.Resolver.Resolve` now returns a TTL, in addition to the resolved path. If the + TTL is unknown, 0 is returned. `IPNSResolver` is able to resolve a TTL, while `DNSResolver` + is not. + * `namesys/resolver.ResolveIPNS` has been moved to `namesys.ResolveIPNS` and now returns a TTL + in addition to the resolved path. +* ✹ `boxo/ipns` record defaults follow recommendations from [IPNS Record Specification](https://specs.ipfs.tech/ipns/ipns-record/#ipns-record): + * `DefaultRecordTTL` is now set to `1h` + * `DefaultRecordLifetime` follows the increased expiration window of Amino DHT ([go-libp2p-kad-dht#793](https://github.com/libp2p/go-libp2p-kad-dht/pull/793)) and is set to `48h` +* 🛠 The `gateway`'s `IPFSBackend.ResolveMutable` is now expected to return a TTL in addition to + the resolved path. If the TTL is unknown, 0 should be returned. ### Removed diff --git a/coreiface/options/name.go b/coreiface/options/name.go index 35e78c394..7b4b6a8fd 100644 --- a/coreiface/options/name.go +++ b/coreiface/options/name.go @@ -3,7 +3,7 @@ package options import ( "time" - ropts "github.com/ipfs/boxo/coreiface/options/namesys" + "github.com/ipfs/boxo/namesys" ) const ( @@ -21,7 +21,7 @@ type NamePublishSettings struct { type NameResolveSettings struct { Cache bool - ResolveOpts []ropts.ResolveOpt + ResolveOpts []namesys.ResolveOption } type ( @@ -123,7 +123,7 @@ func (nameOpts) Cache(cache bool) NameResolveOption { } } -func (nameOpts) ResolveOption(opt ropts.ResolveOpt) NameResolveOption { +func (nameOpts) ResolveOption(opt namesys.ResolveOption) NameResolveOption { return func(settings *NameResolveSettings) error { settings.ResolveOpts = append(settings.ResolveOpts, opt) return nil diff --git a/coreiface/options/namesys/opts.go b/coreiface/options/namesys/opts.go deleted file mode 100644 index ed568200b..000000000 --- a/coreiface/options/namesys/opts.go +++ /dev/null @@ -1,131 +0,0 @@ -package nsopts - -import ( - "time" -) - -const ( - // DefaultDepthLimit is the default depth limit used by Resolve. - DefaultDepthLimit = 32 - - // UnlimitedDepth allows infinite recursion in Resolve. You - // probably don't want to use this, but it's here if you absolutely - // trust resolution to eventually complete and can't put an upper - // limit on how many steps it will take. - UnlimitedDepth = 0 - - // DefaultIPNSRecordTTL specifies the time that the record can be cached - // before checking if its validity again. - DefaultIPNSRecordTTL = time.Minute - - // DefaultIPNSRecordEOL specifies the time that the network will cache IPNS - // records after being published. Records should be re-published before this - // interval expires. We use the same default expiration as the DHT. - DefaultIPNSRecordEOL = 48 * time.Hour -) - -// ResolveOpts specifies options for resolving an IPNS path -type ResolveOpts struct { - // Recursion depth limit - Depth uint - // The number of IPNS records to retrieve from the DHT - // (the best record is selected from this set) - DhtRecordCount uint - // The amount of time to wait for DHT records to be fetched - // and verified. A zero value indicates that there is no explicit - // timeout (although there is an implicit timeout due to dial - // timeouts within the DHT) - DhtTimeout time.Duration -} - -// DefaultResolveOpts returns the default options for resolving -// an IPNS path -func DefaultResolveOpts() ResolveOpts { - return ResolveOpts{ - Depth: DefaultDepthLimit, - DhtRecordCount: 16, - DhtTimeout: time.Minute, - } -} - -// ResolveOpt is used to set an option -type ResolveOpt func(*ResolveOpts) - -// Depth is the recursion depth limit -func Depth(depth uint) ResolveOpt { - return func(o *ResolveOpts) { - o.Depth = depth - } -} - -// DhtRecordCount is the number of IPNS records to retrieve from the DHT -func DhtRecordCount(count uint) ResolveOpt { - return func(o *ResolveOpts) { - o.DhtRecordCount = count - } -} - -// DhtTimeout is the amount of time to wait for DHT records to be fetched -// and verified. A zero value indicates that there is no explicit timeout -func DhtTimeout(timeout time.Duration) ResolveOpt { - return func(o *ResolveOpts) { - o.DhtTimeout = timeout - } -} - -// ProcessOpts converts an array of ResolveOpt into a ResolveOpts object -func ProcessOpts(opts []ResolveOpt) ResolveOpts { - rsopts := DefaultResolveOpts() - for _, option := range opts { - option(&rsopts) - } - return rsopts -} - -// PublishOptions specifies options for publishing an IPNS record. -type PublishOptions struct { - EOL time.Time - TTL time.Duration - CompatibleWithV1 bool -} - -// DefaultPublishOptions returns the default options for publishing an IPNS record. -func DefaultPublishOptions() PublishOptions { - return PublishOptions{ - EOL: time.Now().Add(DefaultIPNSRecordEOL), - TTL: DefaultIPNSRecordTTL, - } -} - -// PublishOption is used to set an option for PublishOpts. -type PublishOption func(*PublishOptions) - -// PublishWithEOL sets an EOL. -func PublishWithEOL(eol time.Time) PublishOption { - return func(o *PublishOptions) { - o.EOL = eol - } -} - -// PublishWithEOL sets a TTL. -func PublishWithTTL(ttl time.Duration) PublishOption { - return func(o *PublishOptions) { - o.TTL = ttl - } -} - -// PublishCompatibleWithV1 sets compatibility with IPNS Records V1. -func PublishCompatibleWithV1(compatible bool) PublishOption { - return func(o *PublishOptions) { - o.CompatibleWithV1 = compatible - } -} - -// ProcessPublishOptions converts an array of PublishOpt into a PublishOpts object. -func ProcessPublishOptions(opts []PublishOption) PublishOptions { - rsopts := DefaultPublishOptions() - for _, option := range opts { - option(&rsopts) - } - return rsopts -} diff --git a/gateway/blocks_backend.go b/gateway/blocks_backend.go index 05b0d1e0b..b4dd705d2 100644 --- a/gateway/blocks_backend.go +++ b/gateway/blocks_backend.go @@ -8,18 +8,18 @@ import ( "io" "net/http" "strings" + "time" "github.com/ipfs/boxo/blockservice" blockstore "github.com/ipfs/boxo/blockstore" - nsopts "github.com/ipfs/boxo/coreiface/options/namesys" "github.com/ipfs/boxo/fetcher" bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/ipld/merkledag" ufile "github.com/ipfs/boxo/ipld/unixfs/file" uio "github.com/ipfs/boxo/ipld/unixfs/io" + "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/namesys" - "github.com/ipfs/boxo/namesys/resolve" "github.com/ipfs/boxo/path" "github.com/ipfs/boxo/path/resolver" blocks "github.com/ipfs/go-block-format" @@ -39,7 +39,6 @@ import ( "github.com/ipld/go-ipld-prime/traversal/selector" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" - "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/routing" mc "github.com/multiformats/go-multicodec" @@ -623,18 +622,23 @@ func (bb *BlocksBackend) getPathRoots(ctx context.Context, contentPath path.Immu return pathRoots, lastPath, remainder, nil } -func (bb *BlocksBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { +func (bb *BlocksBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, time.Duration, time.Time, error) { switch p.Namespace() { case path.IPNSNamespace: - p, err := resolve.ResolveIPNS(ctx, bb.namesys, p) + res, err := namesys.Resolve(ctx, bb.namesys, p) if err != nil { - return path.ImmutablePath{}, err + return path.ImmutablePath{}, 0, time.Time{}, err } - return path.NewImmutablePath(p) + ip, err := path.NewImmutablePath(res.Path) + if err != nil { + return path.ImmutablePath{}, 0, time.Time{}, err + } + return ip, res.TTL, res.LastMod, nil case path.IPFSNamespace: - return path.NewImmutablePath(p) + ip, err := path.NewImmutablePath(p) + return ip, 0, time.Time{}, err default: - return path.ImmutablePath{}, NewErrorStatusCode(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) + return path.ImmutablePath{}, 0, time.Time{}, NewErrorStatusCode(fmt.Errorf("unsupported path namespace: %s", p.Namespace()), http.StatusNotImplemented) } } @@ -643,28 +647,25 @@ func (bb *BlocksBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, return nil, NewErrorStatusCode(errors.New("IPNS Record responses are not supported by this gateway"), http.StatusNotImplemented) } - // Fails fast if the CID is not an encoded Libp2p Key, avoids wasteful - // round trips to the remote routing provider. - if mc.Code(c.Type()) != mc.Libp2pKey { - return nil, NewErrorStatusCode(errors.New("cid codec must be libp2p-key"), http.StatusBadRequest) - } - - // The value store expects the key itself to be encoded as a multihash. - id, err := peer.FromCid(c) + name, err := ipns.NameFromCid(c) if err != nil { - return nil, err + return nil, NewErrorStatusCode(err, http.StatusBadRequest) } - return bb.routing.GetValue(ctx, "/ipns/"+string(id)) + return bb.routing.GetValue(ctx, string(name.RoutingKey())) } func (bb *BlocksBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) { if bb.namesys != nil { - p, err := bb.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) + p, err := path.NewPath("/ipns/" + hostname) + if err != nil { + return nil, err + } + res, err := bb.namesys.Resolve(ctx, p, namesys.ResolveWithDepth(1)) if err == namesys.ErrResolveRecursion { err = nil } - return p, err + return res.Path, err } return nil, NewErrorStatusCode(errors.New("not implemented"), http.StatusNotImplemented) @@ -696,10 +697,11 @@ func (bb *BlocksBackend) ResolvePath(ctx context.Context, path path.ImmutablePat func (bb *BlocksBackend) resolvePath(ctx context.Context, p path.Path) (path.ImmutablePath, []string, error) { var err error if p.Namespace() == path.IPNSNamespace { - p, err = resolve.ResolveIPNS(ctx, bb.namesys, p) + res, err := namesys.Resolve(ctx, bb.namesys, p) if err != nil { return path.ImmutablePath{}, nil, err } + p = res.Path } if p.Namespace() != path.IPFSNamespace { diff --git a/gateway/gateway.go b/gateway/gateway.go index bfafa48e1..06d236d1f 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -8,6 +8,7 @@ import ( "sort" "strconv" "strings" + "time" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/gateway/assets" @@ -369,11 +370,11 @@ type IPFSBackend interface { GetIPNSRecord(context.Context, cid.Cid) ([]byte, error) // ResolveMutable takes a mutable path and resolves it into an immutable one. This means recursively resolving any - // DNSLink or IPNS records. + // DNSLink or IPNS records. It should also return a TTL. If the TTL is unknown, 0 should be returned. // // For example, given a mapping from `/ipns/dnslink.tld -> /ipns/ipns-id/mydirectory` and `/ipns/ipns-id` to // `/ipfs/some-cid`, the result of passing `/ipns/dnslink.tld/myfile` would be `/ipfs/some-cid/mydirectory/myfile`. - ResolveMutable(context.Context, path.Path) (path.ImmutablePath, error) + ResolveMutable(context.Context, path.Path) (path.ImmutablePath, time.Duration, time.Time, error) // GetDNSLinkRecord returns the DNSLink TXT record for the provided FQDN. // Unlike ResolvePath, it does not perform recursive resolution. It only diff --git a/gateway/gateway_test.go b/gateway/gateway_test.go index d3a4ac9b5..e9cb1c150 100644 --- a/gateway/gateway_test.go +++ b/gateway/gateway_test.go @@ -37,11 +37,11 @@ func TestGatewayGet(t *testing.T) { return p } - backend.namesys["/ipns/example.com"] = path.FromCid(k.RootCid()) - backend.namesys["/ipns/working.example.com"] = k - backend.namesys["/ipns/double.example.com"] = mustMakeDNSLinkPath("working.example.com") - backend.namesys["/ipns/triple.example.com"] = mustMakeDNSLinkPath("double.example.com") - backend.namesys["/ipns/broken.example.com"] = mustMakeDNSLinkPath(k.RootCid().String()) + backend.namesys["/ipns/example.com"] = newMockNamesysItem(path.FromCid(k.RootCid()), 0) + backend.namesys["/ipns/working.example.com"] = newMockNamesysItem(k, 0) + backend.namesys["/ipns/double.example.com"] = newMockNamesysItem(mustMakeDNSLinkPath("working.example.com"), 0) + backend.namesys["/ipns/triple.example.com"] = newMockNamesysItem(mustMakeDNSLinkPath("double.example.com"), 0) + backend.namesys["/ipns/broken.example.com"] = newMockNamesysItem(mustMakeDNSLinkPath(k.RootCid().String()), 0) // We picked .man because: // 1. It's a valid TLD. // 2. Go treats it as the file extension for "man" files (even though @@ -49,7 +49,7 @@ func TestGatewayGet(t *testing.T) { // // Unfortunately, this may not work on all platforms as file type // detection is platform dependent. - backend.namesys["/ipns/example.man"] = k + backend.namesys["/ipns/example.man"] = newMockNamesysItem(k, 0) for _, test := range []struct { host string @@ -98,7 +98,7 @@ func TestPretty404(t *testing.T) { t.Logf("test server url: %s", ts.URL) host := "example.net" - backend.namesys["/ipns/"+host] = path.FromCid(root) + backend.namesys["/ipns/"+host] = newMockNamesysItem(path.FromCid(root), 0) for _, test := range []struct { path string @@ -158,7 +158,41 @@ func TestHeaders(t *testing.T) { dagCborRoots = dirRoots + "," + dagCborCID ) - t.Run("Cache-Control is not immutable on generated /ipfs/ HTML dir listings", func(t *testing.T) { + t.Run("Cache-Control uses TTL for /ipns/ when it is known", func(t *testing.T) { + t.Parallel() + + ts, backend, root := newTestServerAndNode(t, nil, "ipns-hostname-redirects.car") + backend.namesys["/ipns/example.net"] = newMockNamesysItem(path.FromCid(root), time.Second*30) + backend.namesys["/ipns/example.com"] = newMockNamesysItem(path.FromCid(root), time.Second*55) + backend.namesys["/ipns/unknown.com"] = newMockNamesysItem(path.FromCid(root), 0) + + testCases := []struct { + path string + cacheControl string + }{ + {"/ipns/example.net/", "public, max-age=30"}, // As generated directory listing + {"/ipns/example.com/", "public, max-age=55"}, // As generated directory listing (different) + {"/ipns/unknown.com/", ""}, // As generated directory listing (unknown) + {"/ipns/example.net/foo/", "public, max-age=30"}, // As index.html directory listing + {"/ipns/example.net/foo/index.html", "public, max-age=30"}, // As deserialized UnixFS file + {"/ipns/example.net/?format=raw", "public, max-age=30"}, // As Raw block + {"/ipns/example.net/?format=dag-json", "public, max-age=30"}, // As DAG-JSON block + {"/ipns/example.net/?format=dag-cbor", "public, max-age=30"}, // As DAG-CBOR block + {"/ipns/example.net/?format=car", "public, max-age=30"}, // As CAR block + } + + for _, testCase := range testCases { + req := mustNewRequest(t, http.MethodGet, ts.URL+testCase.path, nil) + res := mustDoWithoutRedirect(t, req) + if testCase.cacheControl == "" { + assert.Empty(t, res.Header["Cache-Control"]) + } else { + assert.Equal(t, testCase.cacheControl, res.Header.Get("Cache-Control")) + } + } + }) + + t.Run("Cache-Control is not immutable on generated /ipfs/ HTML dir listings", func(t *testing.T) { req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+rootCID+"/", nil) res := mustDoWithoutRedirect(t, req) @@ -500,7 +534,7 @@ func TestRedirects(t *testing.T) { t.Parallel() ts, backend, root := newTestServerAndNode(t, nil, "ipns-hostname-redirects.car") - backend.namesys["/ipns/example.net"] = path.FromCid(root) + backend.namesys["/ipns/example.net"] = newMockNamesysItem(path.FromCid(root), 0) // make request to directory containing index.html req := mustNewRequest(t, http.MethodGet, ts.URL+"/foo", nil) @@ -535,7 +569,7 @@ func TestRedirects(t *testing.T) { t.Parallel() backend, root := newMockBackend(t, "redirects-spa.car") - backend.namesys["/ipns/example.com"] = path.FromCid(root) + backend.namesys["/ipns/example.com"] = newMockNamesysItem(path.FromCid(root), 0) ts := newTestServerWithConfig(t, backend, Config{ Headers: map[string][]string{}, @@ -672,8 +706,8 @@ func TestDeserializedResponses(t *testing.T) { t.Parallel() backend, root := newMockBackend(t, "fixtures.car") - backend.namesys["/ipns/trustless.com"] = path.FromCid(root) - backend.namesys["/ipns/trusted.com"] = path.FromCid(root) + backend.namesys["/ipns/trustless.com"] = newMockNamesysItem(path.FromCid(root), 0) + backend.namesys["/ipns/trusted.com"] = newMockNamesysItem(path.FromCid(root), 0) ts := newTestServerWithConfig(t, backend, Config{ Headers: map[string][]string{}, @@ -735,8 +769,8 @@ func (mb *errorMockBackend) GetCAR(ctx context.Context, path path.ImmutablePath, return ContentPathMetadata{}, nil, mb.err } -func (mb *errorMockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { - return path.ImmutablePath{}, mb.err +func (mb *errorMockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, time.Duration, time.Time, error) { + return path.ImmutablePath{}, 0, time.Time{}, mb.err } func (mb *errorMockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) { @@ -819,7 +853,7 @@ func (mb *panicMockBackend) GetCAR(ctx context.Context, immutablePath path.Immut panic("i am panicking") } -func (mb *panicMockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { +func (mb *panicMockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, time.Duration, time.Time, error) { panic("i am panicking") } diff --git a/gateway/handler.go b/gateway/handler.go index 7e3ccbf48..44218975f 100644 --- a/gateway/handler.go +++ b/gateway/handler.go @@ -34,7 +34,7 @@ var log = logging.Logger("boxo/gateway") const ( ipfsPathPrefix = "/ipfs/" - ipnsPathPrefix = "/ipns/" + ipnsPathPrefix = ipns.NamespacePrefix immutableCacheControl = "public, max-age=29030400, immutable" ) @@ -194,6 +194,8 @@ type requestData struct { // Defined for non IPNS Record requests. immutablePath path.ImmutablePath + ttl time.Duration + lastMod time.Time // Defined if resolution has already happened. pathMetadata *ContentPathMetadata @@ -285,7 +287,7 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) { } if contentPath.Mutable() { - rq.immutablePath, err = i.backend.ResolveMutable(r.Context(), contentPath) + rq.immutablePath, rq.ttl, rq.lastMod, err = i.backend.ResolveMutable(r.Context(), contentPath) if err != nil { err = fmt.Errorf("failed to resolve %s: %w", debugStr(contentPath.String()), err) i.webError(w, r, err, http.StatusInternalServerError) @@ -415,7 +417,7 @@ func panicHandler(w http.ResponseWriter) { } } -func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath path.Path, cid cid.Cid, responseFormat string) (modtime time.Time) { +func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath path.Path, ttl time.Duration, lastMod time.Time, cid cid.Cid, responseFormat string) (modtime time.Time) { // Best effort attempt to set an Etag based on the CID and response format. // Setting an ETag is handled separately for CARs and IPNS records. if etag := getEtag(r, cid, responseFormat); etag != "" { @@ -424,23 +426,24 @@ func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath // Set Cache-Control and Last-Modified based on contentPath properties if contentPath.Mutable() { - // mutable namespaces such as /ipns/ can't be cached forever - - // For now we set Last-Modified to Now() to leverage caching heuristics built into modern browsers: - // https://github.com/ipfs/kubo/pull/8074#pullrequestreview-645196768 - // but we should not set it to fake values and use Cache-Control based on TTL instead - modtime = time.Now() + if ttl > 0 { + // When we know the TTL, set the Cache-Control header and disable Last-Modified. + w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", int(ttl.Seconds()))) + } - // TODO: set Cache-Control based on TTL of IPNS/DNSLink: https://github.com/ipfs/kubo/issues/1818#issuecomment-1015849462 - // TODO: set Last-Modified based on /ipns/ publishing timestamp? + if lastMod.IsZero() { + // Otherwise, we set Last-Modified to the current time to leverage caching heuristics + // built into modern browsers: https://github.com/ipfs/kubo/pull/8074#pullrequestreview-645196768 + modtime = time.Now() + } else { + modtime = lastMod + } } else { - // immutable! CACHE ALL THE THINGS, FOREVER! wolololol w.Header().Set("Cache-Control", immutableCacheControl) + modtime = noModtime // disable Last-Modified - // Set modtime to 'zero time' to disable Last-Modified header (superseded by Cache-Control) - modtime = noModtime - - // TODO: set Last-Modified? - TBD - /ipfs/ modification metadata is present in unixfs 1.5 https://github.com/ipfs/kubo/issues/6920? + // TODO: consider setting Last-Modified if UnixFS V1.5 ever gets released + // with metadata: https://github.com/ipfs/kubo/issues/6920 } return modtime diff --git a/gateway/handler_block.go b/gateway/handler_block.go index 461e306d8..e6bf2267f 100644 --- a/gateway/handler_block.go +++ b/gateway/handler_block.go @@ -34,7 +34,7 @@ func (i *handler) serveRawBlock(ctx context.Context, w http.ResponseWriter, r *h setContentDispositionHeader(w, name, "attachment") // Set remaining headers - modtime := addCacheControlHeaders(w, r, rq.contentPath, blockCid, rawResponseFormat) + modtime := addCacheControlHeaders(w, r, rq.contentPath, rq.ttl, rq.lastMod, blockCid, rawResponseFormat) w.Header().Set("Content-Type", rawResponseFormat) w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^) diff --git a/gateway/handler_car.go b/gateway/handler_car.go index 0b2162c4f..1c670b045 100644 --- a/gateway/handler_car.go +++ b/gateway/handler_car.go @@ -57,7 +57,7 @@ func (i *handler) serveCAR(ctx context.Context, w http.ResponseWriter, r *http.R setContentDispositionHeader(w, name, "attachment") // Set Cache-Control (same logic as for a regular files) - addCacheControlHeaders(w, r, rq.contentPath, rootCid, carResponseFormat) + addCacheControlHeaders(w, r, rq.contentPath, rq.ttl, rq.lastMod, rootCid, carResponseFormat) // Generate the CAR Etag. etag := getCarEtag(rq.immutablePath, params, rootCid) diff --git a/gateway/handler_codec.go b/gateway/handler_codec.go index c1abb1d27..617eb9396 100644 --- a/gateway/handler_codec.go +++ b/gateway/handler_codec.go @@ -111,7 +111,7 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt } // Set HTTP headers (for caching, etc). Etag will be replaced if handled by serveCodecHTML. - modtime := addCacheControlHeaders(w, r, rq.contentPath, resolvedPath.RootCid(), responseContentType) + modtime := addCacheControlHeaders(w, r, rq.contentPath, rq.ttl, rq.lastMod, resolvedPath.RootCid(), responseContentType) _ = setCodecContentDisposition(w, r, resolvedPath, responseContentType) w.Header().Set("Content-Type", responseContentType) w.Header().Set("X-Content-Type-Options", "nosniff") diff --git a/gateway/handler_defaults.go b/gateway/handler_defaults.go index e56e6c5d9..648d51328 100644 --- a/gateway/handler_defaults.go +++ b/gateway/handler_defaults.go @@ -122,13 +122,13 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h if headResp != nil { if headResp.isFile { rq.logger.Debugw("serving unixfs file", "path", rq.contentPath) - return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, headResp.bytesSize, headResp.startingBytes, false, true, pathMetadata.ContentType, rq.begin) + return i.serveFile(ctx, w, r, resolvedPath, rq, headResp.bytesSize, headResp.startingBytes, false, true, pathMetadata.ContentType) } else if headResp.isSymLink { rq.logger.Debugw("serving unixfs file", "path", rq.contentPath) - return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, headResp.bytesSize, nil, true, true, pathMetadata.ContentType, rq.begin) + return i.serveFile(ctx, w, r, resolvedPath, rq, headResp.bytesSize, nil, true, true, pathMetadata.ContentType) } else if headResp.isDir { rq.logger.Debugw("serving unixfs directory", "path", rq.contentPath) - return i.serveDirectory(ctx, w, r, resolvedPath, rq.contentPath, true, nil, ranges, rq.begin, rq.logger) + return i.serveDirectory(ctx, w, r, resolvedPath, rq, true, nil, ranges) } } else { if getResp.bytes != nil { @@ -140,14 +140,14 @@ func (i *handler) serveDefaults(ctx context.Context, w http.ResponseWriter, r *h rangeRequestStartsAtZero = false } } - return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, getResp.bytesSize, getResp.bytes, false, rangeRequestStartsAtZero, pathMetadata.ContentType, rq.begin) + return i.serveFile(ctx, w, r, resolvedPath, rq, getResp.bytesSize, getResp.bytes, false, rangeRequestStartsAtZero, pathMetadata.ContentType) } else if getResp.symlink != nil { rq.logger.Debugw("serving unixfs file", "path", rq.contentPath) // Note: this ignores range requests against symlinks - return i.serveFile(ctx, w, r, resolvedPath, rq.contentPath, getResp.bytesSize, getResp.symlink, true, true, pathMetadata.ContentType, rq.begin) + return i.serveFile(ctx, w, r, resolvedPath, rq, getResp.bytesSize, getResp.symlink, true, true, pathMetadata.ContentType) } else if getResp.directoryMetadata != nil { rq.logger.Debugw("serving unixfs directory", "path", rq.contentPath) - return i.serveDirectory(ctx, w, r, resolvedPath, rq.contentPath, false, getResp.directoryMetadata, ranges, rq.begin, rq.logger) + return i.serveDirectory(ctx, w, r, resolvedPath, rq, false, getResp.directoryMetadata, ranges) } } diff --git a/gateway/handler_tar.go b/gateway/handler_tar.go index 6af1d0c4d..0b2634804 100644 --- a/gateway/handler_tar.go +++ b/gateway/handler_tar.go @@ -31,7 +31,7 @@ func (i *handler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.R rootCid := pathMetadata.LastSegment.RootCid() // Set Cache-Control and read optional Last-Modified time - modtime := addCacheControlHeaders(w, r, rq.contentPath, rootCid, tarResponseFormat) + modtime := addCacheControlHeaders(w, r, rq.contentPath, rq.ttl, rq.lastMod, rootCid, tarResponseFormat) // Set Content-Disposition var name string diff --git a/gateway/handler_unixfs__redirects.go b/gateway/handler_unixfs__redirects.go index 6d1f40c25..0cd4d3c71 100644 --- a/gateway/handler_unixfs__redirects.go +++ b/gateway/handler_unixfs__redirects.go @@ -7,6 +7,7 @@ import ( gopath "path" "strconv" "strings" + "time" "github.com/ipfs/boxo/path" redirects "github.com/ipfs/go-ipfs-redirects-file" @@ -223,7 +224,7 @@ func (i *handler) serve4xx(w http.ResponseWriter, r *http.Request, content4xxPat logger.Debugf("using _redirects: custom %d file at %q", status, content4xxPath) w.Header().Set("Content-Type", "text/html") w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) - addCacheControlHeaders(w, r, content4xxPath, content4xxCid, "") + addCacheControlHeaders(w, r, content4xxPath, 0, time.Time{}, content4xxCid, "") w.WriteHeader(status) _, err = io.CopyN(w, content4xxFile, size) return err diff --git a/gateway/handler_unixfs_dir.go b/gateway/handler_unixfs_dir.go index 479f4e3c9..678c51aba 100644 --- a/gateway/handler_unixfs_dir.go +++ b/gateway/handler_unixfs_dir.go @@ -17,13 +17,12 @@ import ( cid "github.com/ipfs/go-cid" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "go.uber.org/zap" ) // serveDirectory returns the best representation of UnixFS directory // // It will return index.html if present, or generate directory listing otherwise. -func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, contentPath path.Path, isHeadRequest bool, directoryMetadata *directoryMetadata, ranges []ByteRange, begin time.Time, logger *zap.SugaredLogger) bool { +func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, rq *requestData, isHeadRequest bool, directoryMetadata *directoryMetadata, ranges []ByteRange) bool { ctx, span := spanTrace(ctx, "Handler.ServeDirectory", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() @@ -52,14 +51,14 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } // /ipfs/cid/foo?bar must be redirected to /ipfs/cid/foo/?bar redirectURL := originalURLPath + suffix - logger.Debugw("directory location moved permanently", "status", http.StatusMovedPermanently) + rq.logger.Debugw("directory location moved permanently", "status", http.StatusMovedPermanently) http.Redirect(w, r, redirectURL, http.StatusMovedPermanently) return true } } // Check if directory has index.html, if so, serveFile - idxPath, err := path.Join(contentPath, "index.html") + idxPath, err := path.Join(rq.contentPath, "index.html") if err != nil { i.webError(w, r, err, http.StatusInternalServerError) return false @@ -113,17 +112,19 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } if err == nil { - logger.Debugw("serving index.html file", "path", idxPath) + rq.logger.Debugw("serving index.html file", "path", idxPath) + originalContentPath := rq.contentPath + rq.contentPath = idxPath // write to request - success := i.serveFile(ctx, w, r, resolvedPath, idxPath, idxFileSize, idxFileBytes, false, returnRangeStartsAtZero, "text/html", begin) + success := i.serveFile(ctx, w, r, resolvedPath, rq, idxFileSize, idxFileBytes, false, returnRangeStartsAtZero, "text/html") if success { - i.unixfsDirIndexGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.unixfsDirIndexGetMetric.WithLabelValues(originalContentPath.Namespace()).Observe(time.Since(rq.begin).Seconds()) } return success } if isErrNotFound(err) { - logger.Debugw("no index.html; noop", "path", idxPath) + rq.logger.Debugw("no index.html; noop", "path", idxPath) } else if err != nil { i.webError(w, r, err, http.StatusInternalServerError) return false @@ -137,8 +138,13 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * dirEtag := getDirListingEtag(resolvedPath.RootCid()) w.Header().Set("Etag", dirEtag) + // Add TTL if known. + if rq.ttl > 0 { + w.Header().Set("Cache-Control", fmt.Sprintf("public, max-age=%d", int(rq.ttl.Seconds()))) + } + if r.Method == http.MethodHead { - logger.Debug("return as request's HTTP method is HEAD") + rq.logger.Debug("return as request's HTTP method is HEAD") return true } @@ -169,7 +175,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * backLink := originalURLPath // don't go further up than /ipfs/$hash/ - pathSplit := strings.Split(contentPath.String(), "/") + pathSplit := strings.Split(rq.contentPath.String(), "/") switch { // skip backlink when listing a content root case len(pathSplit) == 3: // url: /ipfs/$hash @@ -190,20 +196,20 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * size := humanize.Bytes(directoryMetadata.dagSize) hash := resolvedPath.RootCid().String() - globalData := i.getTemplateGlobalData(r, contentPath) + globalData := i.getTemplateGlobalData(r, rq.contentPath) // See comment above where originalUrlPath is declared. tplData := assets.DirectoryTemplateData{ GlobalData: globalData, Listing: dirListing, Size: size, - Path: contentPath.String(), - Breadcrumbs: assets.Breadcrumbs(contentPath.String(), globalData.DNSLink), + Path: rq.contentPath.String(), + Breadcrumbs: assets.Breadcrumbs(rq.contentPath.String(), globalData.DNSLink), BackLink: backLink, Hash: hash, } - logger.Debugw("request processed", "tplDataDNSLink", globalData.DNSLink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash) + rq.logger.Debugw("request processed", "tplDataDNSLink", globalData.DNSLink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash) if err := assets.DirectoryTemplate.Execute(w, tplData); err != nil { i.webError(w, r, err, http.StatusInternalServerError) @@ -211,7 +217,7 @@ func (i *handler) serveDirectory(ctx context.Context, w http.ResponseWriter, r * } // Update metrics - i.unixfsGenDirListingGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.unixfsGenDirListingGetMetric.WithLabelValues(rq.contentPath.Namespace()).Observe(time.Since(rq.begin).Seconds()) return true } diff --git a/gateway/handler_unixfs_dir_test.go b/gateway/handler_unixfs_dir_test.go index 48f2625d6..e44708687 100644 --- a/gateway/handler_unixfs_dir_test.go +++ b/gateway/handler_unixfs_dir_test.go @@ -28,7 +28,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { k3, err := backend.resolvePathNoRootsReturned(ctx, p3) require.NoError(t, err) - backend.namesys["/ipns/example.net"] = path.FromCid(root) + backend.namesys["/ipns/example.net"] = newMockNamesysItem(path.FromCid(root), 0) // make request to directory listing req := mustNewRequest(t, http.MethodGet, ts.URL+"/foo%3F%20%23%3C%27/", nil) diff --git a/gateway/handler_unixfs_file.go b/gateway/handler_unixfs_file.go index fb1f9940c..734e11025 100644 --- a/gateway/handler_unixfs_file.go +++ b/gateway/handler_unixfs_file.go @@ -19,15 +19,15 @@ import ( // serveFile returns data behind a file along with HTTP headers based on // the file itself, its CID and the contentPath used for accessing it. -func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, contentPath path.Path, fileSize int64, fileBytes io.ReadCloser, isSymlink bool, returnRangeStartsAtZero bool, fileContentType string, begin time.Time) bool { +func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath path.ImmutablePath, rq *requestData, fileSize int64, fileBytes io.ReadCloser, isSymlink bool, returnRangeStartsAtZero bool, fileContentType string) bool { _, span := spanTrace(ctx, "Handler.ServeFile", trace.WithAttributes(attribute.String("path", resolvedPath.String()))) defer span.End() // Set Cache-Control and read optional Last-Modified time - modtime := addCacheControlHeaders(w, r, contentPath, resolvedPath.RootCid(), "") + modtime := addCacheControlHeaders(w, r, rq.contentPath, rq.ttl, rq.lastMod, resolvedPath.RootCid(), "") // Set Content-Disposition - name := addContentDispositionHeader(w, r, contentPath) + name := addContentDispositionHeader(w, r, rq.contentPath) if fileSize == 0 { // We override null files to 200 to avoid issues with fragment caching reverse proxies. @@ -89,7 +89,7 @@ func (i *handler) serveFile(ctx context.Context, w http.ResponseWriter, r *http. // Was response successful? if dataSent { // Update metrics - i.unixfsFileGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds()) + i.unixfsFileGetMetric.WithLabelValues(rq.contentPath.Namespace()).Observe(time.Since(rq.begin).Seconds()) } return dataSent diff --git a/gateway/hostname_test.go b/gateway/hostname_test.go index 3a6809555..7a8d7a170 100644 --- a/gateway/hostname_test.go +++ b/gateway/hostname_test.go @@ -20,8 +20,8 @@ func TestToSubdomainURL(t *testing.T) { testCID, err := cid.Decode("bafkqaglimvwgy3zakrsxg5cun5jxkyten5wwc2lokvjeycq") require.NoError(t, err) - backend.namesys["/ipns/dnslink.long-name.example.com"] = path.FromCid(testCID) - backend.namesys["/ipns/dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com"] = path.FromCid(testCID) + backend.namesys["/ipns/dnslink.long-name.example.com"] = newMockNamesysItem(path.FromCid(testCID), 0) + backend.namesys["/ipns/dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com"] = newMockNamesysItem(path.FromCid(testCID), 0) httpRequest := httptest.NewRequest("GET", "http://127.0.0.1:8080", nil) httpsRequest := httptest.NewRequest("GET", "https://https-request-stub.example.com", nil) httpsProxiedRequest := httptest.NewRequest("GET", "http://proxied-https-request-stub.example.com", nil) diff --git a/gateway/metrics.go b/gateway/metrics.go index bccaefef2..df8639956 100644 --- a/gateway/metrics.go +++ b/gateway/metrics.go @@ -155,16 +155,16 @@ func (b *ipfsBackendWithMetrics) GetIPNSRecord(ctx context.Context, cid cid.Cid) return r, err } -func (b *ipfsBackendWithMetrics) ResolveMutable(ctx context.Context, path path.Path) (path.ImmutablePath, error) { +func (b *ipfsBackendWithMetrics) ResolveMutable(ctx context.Context, path path.Path) (path.ImmutablePath, time.Duration, time.Time, error) { begin := time.Now() name := "IPFSBackend.ResolveMutable" ctx, span := spanTrace(ctx, name, trace.WithAttributes(attribute.String("path", path.String()))) defer span.End() - p, err := b.backend.ResolveMutable(ctx, path) + p, ttl, lastMod, err := b.backend.ResolveMutable(ctx, path) b.updateBackendCallMetric(name, err, begin) - return p, err + return p, ttl, lastMod, err } func (b *ipfsBackendWithMetrics) GetDNSLinkRecord(ctx context.Context, fqdn string) (path.Path, error) { diff --git a/gateway/utilities_test.go b/gateway/utilities_test.go index bba66d94d..85153f808 100644 --- a/gateway/utilities_test.go +++ b/gateway/utilities_test.go @@ -11,9 +11,9 @@ import ( "regexp" "strings" "testing" + "time" "github.com/ipfs/boxo/blockservice" - nsopts "github.com/ipfs/boxo/coreiface/options/namesys" offline "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/boxo/files" "github.com/ipfs/boxo/namesys" @@ -51,43 +51,60 @@ func mustDo(t *testing.T, req *http.Request) *http.Response { return res } -type mockNamesys map[string]path.Path +type mockNamesysItem struct { + path path.Path + ttl time.Duration +} + +func newMockNamesysItem(p path.Path, ttl time.Duration) *mockNamesysItem { + return &mockNamesysItem{path: p, ttl: ttl} +} + +type mockNamesys map[string]*mockNamesysItem -func (m mockNamesys) Resolve(ctx context.Context, name string, opts ...nsopts.ResolveOpt) (value path.Path, err error) { - cfg := nsopts.DefaultResolveOpts() +func (m mockNamesys) Resolve(ctx context.Context, p path.Path, opts ...namesys.ResolveOption) (result namesys.Result, err error) { + cfg := namesys.DefaultResolveOptions() for _, o := range opts { o(&cfg) } depth := cfg.Depth - if depth == nsopts.UnlimitedDepth { + if depth == namesys.UnlimitedDepth { // max uint depth = ^uint(0) } + var ( + value path.Path + ttl time.Duration + ) + name := path.SegmentsToString(p.Segments()[:2]...) for strings.HasPrefix(name, "/ipns/") { if depth == 0 { - return value, namesys.ErrResolveRecursion + return namesys.Result{Path: value, TTL: ttl}, namesys.ErrResolveRecursion } depth-- - var ok bool - value, ok = m[name] + v, ok := m[name] if !ok { - return nil, namesys.ErrResolveFailed + return namesys.Result{}, namesys.ErrResolveFailed } + value = v.path + ttl = v.ttl name = value.String() } - return value, nil + + value, err = path.Join(value, p.Segments()[2:]...) + return namesys.Result{Path: value, TTL: ttl}, err } -func (m mockNamesys) ResolveAsync(ctx context.Context, name string, opts ...nsopts.ResolveOpt) <-chan namesys.Result { - out := make(chan namesys.Result, 1) - v, err := m.Resolve(ctx, name, opts...) - out <- namesys.Result{Path: v, Err: err} +func (m mockNamesys) ResolveAsync(ctx context.Context, p path.Path, opts ...namesys.ResolveOption) <-chan namesys.AsyncResult { + out := make(chan namesys.AsyncResult, 1) + res, err := m.Resolve(ctx, p, opts...) + out <- namesys.AsyncResult{Path: res.Path, TTL: res.TTL, LastMod: res.LastMod, Err: err} close(out) return out } -func (m mockNamesys) Publish(ctx context.Context, name crypto.PrivKey, value path.Path, opts ...nsopts.PublishOption) error { +func (m mockNamesys) Publish(ctx context.Context, name crypto.PrivKey, value path.Path, opts ...namesys.PublishOption) error { return errors.New("not implemented for mockNamesys") } @@ -152,7 +169,7 @@ func (mb *mockBackend) GetCAR(ctx context.Context, immutablePath path.ImmutableP return mb.gw.GetCAR(ctx, immutablePath, params) } -func (mb *mockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, error) { +func (mb *mockBackend) ResolveMutable(ctx context.Context, p path.Path) (path.ImmutablePath, time.Duration, time.Time, error) { return mb.gw.ResolveMutable(ctx, p) } @@ -162,10 +179,15 @@ func (mb *mockBackend) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, er func (mb *mockBackend) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) { if mb.namesys != nil { - p, err := mb.namesys.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1)) + p, err := path.NewPath("/ipns/" + hostname) + if err != nil { + return nil, err + } + res, err := mb.namesys.Resolve(ctx, p, namesys.ResolveWithDepth(1)) if err == namesys.ErrResolveRecursion { err = nil } + p = res.Path return p, err } @@ -184,7 +206,7 @@ func (mb *mockBackend) resolvePathNoRootsReturned(ctx context.Context, ip path.P var imPath path.ImmutablePath var err error if ip.Mutable() { - imPath, err = mb.ResolveMutable(ctx, ip) + imPath, _, _, err = mb.ResolveMutable(ctx, ip) if err != nil { return path.ImmutablePath{}, err } diff --git a/ipns/defaults.go b/ipns/defaults.go new file mode 100644 index 000000000..c2179e61e --- /dev/null +++ b/ipns/defaults.go @@ -0,0 +1,17 @@ +package ipns + +import "time" + +const ( + + // DefaultRecordLifetime defines for how long IPNS record should be valid + // when ValidityType is 0. The default here aims to match the record + // expiration window of Amino DHT. + DefaultRecordLifetime = 48 * time.Hour + + // DefaultRecordTTL specifies how long the record can be returned from + // cache before checking for update again. The function of this TTL is + // similar to TTL of DNS record, and the default here is a trade-off + // between faster updates and benefiting from various types of caching. + DefaultRecordTTL = 1 * time.Hour +) diff --git a/ipns/name.go b/ipns/name.go index b35e8f9d8..e76a3ae33 100644 --- a/ipns/name.go +++ b/ipns/name.go @@ -25,7 +25,7 @@ const ( // [Multihash]: https://multiformats.io/multihash/ // [IPNS Name]: https://specs.ipfs.tech/ipns/ipns-record/#ipns-name type Name struct { - src []byte + multihash string // binary Multihash without multibase envelope } // NameFromString creates a [Name] from the given IPNS Name in its [string representation]. @@ -57,7 +57,7 @@ func NameFromRoutingKey(data []byte) (Name, error) { // NameFromPeer creates a [Name] from the given [peer.ID]. func NameFromPeer(pid peer.ID) Name { - return Name{src: []byte(pid)} + return Name{multihash: string(pid)} } // NameFromCid creates a [Name] from the given [cid.Cid]. @@ -66,7 +66,7 @@ func NameFromCid(c cid.Cid) (Name, error) { if code != mc.Libp2pKey { return Name{}, fmt.Errorf("CID codec %q is not allowed for IPNS Names, use %q instead", code, mc.Libp2pKey) } - return Name{src: c.Hash()}, nil + return Name{multihash: string(c.Hash())}, nil } // RoutingKey returns the binary IPNS Routing Key for the given [Name]. Note that @@ -77,14 +77,14 @@ func NameFromCid(c cid.Cid) (Name, error) { func (n Name) RoutingKey() []byte { var buffer bytes.Buffer buffer.WriteString(NamespacePrefix) - buffer.Write(n.src) // Note: we append raw multihash bytes (no multibase) + buffer.WriteString(n.multihash) // Note: we append raw multihash bytes (no multibase) return buffer.Bytes() } // Cid returns [Name] encoded as a [cid.Cid] of the public key. If the IPNS Name // is invalid (e.g., empty), this will return the empty Cid. func (n Name) Cid() cid.Cid { - m, err := mh.Cast([]byte(n.src)) + m, err := mh.Cast([]byte(n.multihash)) if err != nil { return cid.Undef } @@ -93,7 +93,7 @@ func (n Name) Cid() cid.Cid { // Peer returns [Name] as a [peer.ID]. func (n Name) Peer() peer.ID { - return peer.ID(n.src) + return peer.ID(n.multihash) } // String returns the human-readable IPNS Name, encoded as a CIDv1 with libp2p-key @@ -132,7 +132,7 @@ func (n Name) MarshalJSON() ([]byte, error) { // Equal returns whether the records are equal. func (n Name) Equal(other Name) bool { - return bytes.Equal(n.src, other.src) + return n.multihash == other.multihash } // AsPath returns the IPNS Name as a [path.Path] prefixed by [path.IPNSNamespace]. diff --git a/namesys/base.go b/namesys/base.go deleted file mode 100644 index 6b8e41ab5..000000000 --- a/namesys/base.go +++ /dev/null @@ -1,126 +0,0 @@ -package namesys - -import ( - "context" - "strings" - "time" - - opts "github.com/ipfs/boxo/coreiface/options/namesys" - "github.com/ipfs/boxo/path" -) - -type onceResult struct { - value path.Path - ttl time.Duration - err error -} - -type resolver interface { - resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult -} - -// resolve is a helper for implementing Resolver.ResolveN using resolveOnce. -func resolve(ctx context.Context, r resolver, name string, options opts.ResolveOpts) (path.Path, error) { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - err := ErrResolveFailed - var p path.Path - - resCh := resolveAsync(ctx, r, name, options) - - for res := range resCh { - p, err = res.Path, res.Err - if err != nil { - break - } - } - - return p, err -} - -func resolveAsync(ctx context.Context, r resolver, name string, options opts.ResolveOpts) <-chan Result { - ctx, span := StartSpan(ctx, "ResolveAsync") - defer span.End() - - resCh := r.resolveOnceAsync(ctx, name, options) - depth := options.Depth - outCh := make(chan Result, 1) - - go func() { - defer close(outCh) - ctx, span := StartSpan(ctx, "ResolveAsync.Worker") - defer span.End() - - var subCh <-chan Result - var cancelSub context.CancelFunc - defer func() { - if cancelSub != nil { - cancelSub() - } - }() - - for { - select { - case res, ok := <-resCh: - if !ok { - resCh = nil - break - } - - if res.err != nil { - emitResult(ctx, outCh, Result{Err: res.err}) - return - } - log.Debugf("resolved %s to %s", name, res.value.String()) - if !strings.HasPrefix(res.value.String(), ipnsPrefix) { - emitResult(ctx, outCh, Result{Path: res.value}) - break - } - - if depth == 1 { - emitResult(ctx, outCh, Result{Path: res.value, Err: ErrResolveRecursion}) - break - } - - subopts := options - if subopts.Depth > 1 { - subopts.Depth-- - } - - var subCtx context.Context - if cancelSub != nil { - // Cancel previous recursive resolve since it won't be used anyways - cancelSub() - } - subCtx, cancelSub = context.WithCancel(ctx) - _ = cancelSub - - p := strings.TrimPrefix(res.value.String(), ipnsPrefix) - subCh = resolveAsync(subCtx, r, p, subopts) - case res, ok := <-subCh: - if !ok { - subCh = nil - break - } - - // We don't bother returning here in case of context timeout as there is - // no good reason to do that, and we may still be able to emit a result - emitResult(ctx, outCh, res) - case <-ctx.Done(): - return - } - if resCh == nil && subCh == nil { - return - } - } - }() - return outCh -} - -func emitResult(ctx context.Context, outCh chan<- Result, r Result) { - select { - case outCh <- r: - case <-ctx.Done(): - } -} diff --git a/namesys/cache.go b/namesys/cache.go deleted file mode 100644 index 51fe3149b..000000000 --- a/namesys/cache.go +++ /dev/null @@ -1,62 +0,0 @@ -package namesys - -import ( - "time" - - "github.com/ipfs/boxo/path" -) - -func (ns *mpns) cacheGet(name string) (path.Path, bool) { - // existence of optional mapping defined via IPFS_NS_MAP is checked first - if ns.staticMap != nil { - val, ok := ns.staticMap[name] - if ok { - return val, true - } - } - - if ns.cache == nil { - return nil, false - } - - ientry, ok := ns.cache.Get(name) - if !ok { - return nil, false - } - - entry, ok := ientry.(cacheEntry) - if !ok { - // should never happen, purely for sanity - log.Panicf("unexpected type %T in cache for %q.", ientry, name) - } - - if time.Now().Before(entry.eol) { - return entry.val, true - } - - ns.cache.Remove(name) - - return nil, false -} - -func (ns *mpns) cacheSet(name string, val path.Path, ttl time.Duration) { - if ns.cache == nil || ttl <= 0 { - return - } - ns.cache.Add(name, cacheEntry{ - val: val, - eol: time.Now().Add(ttl), - }) -} - -func (ns *mpns) cacheInvalidate(name string) { - if ns.cache == nil { - return - } - ns.cache.Remove(name) -} - -type cacheEntry struct { - val path.Path - eol time.Time -} diff --git a/namesys/dns.go b/namesys/dns_resolver.go similarity index 58% rename from namesys/dns.go rename to namesys/dns_resolver.go index 48becfc1c..ef46ce399 100644 --- a/namesys/dns.go +++ b/namesys/dns_resolver.go @@ -7,9 +7,9 @@ import ( "net" gopath "path" "strings" + "time" - opts "github.com/ipfs/boxo/coreiface/options/namesys" - "github.com/ipfs/boxo/path" + path "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" dns "github.com/miekg/dns" "go.opentelemetry.io/otel/attribute" @@ -19,80 +19,65 @@ import ( // LookupTXTFunc is a function that lookups TXT record values. type LookupTXTFunc func(ctx context.Context, name string) (txt []string, err error) -// DNSResolver implements a Resolver on DNS domains +// DNSResolver implements [Resolver] on DNS domains. type DNSResolver struct { lookupTXT LookupTXTFunc - // TODO: maybe some sort of caching? - // cache would need a timeout } +var _ Resolver = &DNSResolver{} + // NewDNSResolver constructs a name resolver using DNS TXT records. func NewDNSResolver(lookup LookupTXTFunc) *DNSResolver { return &DNSResolver{lookupTXT: lookup} } -// Resolve implements Resolver. -func (r *DNSResolver) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) { - ctx, span := StartSpan(ctx, "DNSResolver.Resolve") +func (r *DNSResolver) Resolve(ctx context.Context, p path.Path, options ...ResolveOption) (Result, error) { + ctx, span := startSpan(ctx, "DNSResolver.Resolve", trace.WithAttributes(attribute.Stringer("Path", p))) defer span.End() - return resolve(ctx, r, name, opts.ProcessOpts(options)) + return resolve(ctx, r, p, ProcessResolveOptions(options)) } -// ResolveAsync implements Resolver. -func (r *DNSResolver) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result { - ctx, span := StartSpan(ctx, "DNSResolver.ResolveAsync") +func (r *DNSResolver) ResolveAsync(ctx context.Context, p path.Path, options ...ResolveOption) <-chan AsyncResult { + ctx, span := startSpan(ctx, "DNSResolver.ResolveAsync", trace.WithAttributes(attribute.Stringer("Path", p))) defer span.End() - return resolveAsync(ctx, r, name, opts.ProcessOpts(options)) -} - -type lookupRes struct { - path path.Path - error error + return resolveAsync(ctx, r, p, ProcessResolveOptions(options)) } -// resolveOnce implements resolver. -// TXT records for a given domain name should contain a b58 -// encoded multihash. -func (r *DNSResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult { - ctx, span := StartSpan(ctx, "DNSResolver.ResolveOnceAsync") +func (r *DNSResolver) resolveOnceAsync(ctx context.Context, p path.Path, options ResolveOptions) <-chan AsyncResult { + ctx, span := startSpan(ctx, "DNSResolver.ResolveOnceAsync", trace.WithAttributes(attribute.Stringer("Path", p))) defer span.End() - var fqdn string - out := make(chan onceResult, 1) - segments := strings.SplitN(name, "/", 2) - domain := segments[0] + out := make(chan AsyncResult, 1) + if p.Namespace() != path.IPNSNamespace { + out <- AsyncResult{Err: fmt.Errorf("unsupported namespace: %q", p.Namespace())} + close(out) + return out + } - if _, ok := dns.IsDomainName(domain); !ok { - out <- onceResult{err: fmt.Errorf("not a valid domain name: %s", domain)} + fqdn := p.Segments()[1] + if _, ok := dns.IsDomainName(fqdn); !ok { + out <- AsyncResult{Err: fmt.Errorf("not a valid domain name: %q", fqdn)} close(out) return out } - log.Debugf("DNSResolver resolving %s", domain) - if strings.HasSuffix(domain, ".") { - fqdn = domain - } else { - fqdn = domain + "." + log.Debugf("DNSResolver resolving %q", fqdn) + + if !strings.HasSuffix(fqdn, ".") { + fqdn += "." } - rootChan := make(chan lookupRes, 1) + rootChan := make(chan AsyncResult, 1) go workDomain(ctx, r, fqdn, rootChan) - subChan := make(chan lookupRes, 1) + subChan := make(chan AsyncResult, 1) go workDomain(ctx, r, "_dnslink."+fqdn, subChan) - appendPath := func(p path.Path) (path.Path, error) { - if len(segments) > 1 { - return path.Join(p, segments[1]) - } - return p, nil - } - go func() { defer close(out) - ctx, span := StartSpan(ctx, "DNSResolver.ResolveOnceAsync.Worker") + ctx, span := startSpan(ctx, "DNSResolver.ResolveOnceAsync.Worker") defer span.End() var rootResErr, subResErr error @@ -103,26 +88,26 @@ func (r *DNSResolver) resolveOnceAsync(ctx context.Context, name string, options subChan = nil break } - if subRes.error == nil { - p, err := appendPath(subRes.path) - emitOnceResult(ctx, out, onceResult{value: p, err: err}) + if subRes.Err == nil { + p, err := joinPaths(subRes.Path, p) + emitOnceResult(ctx, out, AsyncResult{Path: p, LastMod: time.Now(), Err: err}) // Return without waiting for rootRes, since this result // (for "_dnslink."+fqdn) takes precedence return } - subResErr = subRes.error + subResErr = subRes.Err case rootRes, ok := <-rootChan: if !ok { rootChan = nil break } - if rootRes.error == nil { - p, err := appendPath(rootRes.path) - emitOnceResult(ctx, out, onceResult{value: p, err: err}) + if rootRes.Err == nil { + p, err := joinPaths(rootRes.Path, p) + emitOnceResult(ctx, out, AsyncResult{Path: p, LastMod: time.Now(), Err: err}) // Do not return here. Wait for subRes so that it is // output last if good, thereby giving subRes precedence. } else { - rootResErr = rootRes.error + rootResErr = rootRes.Err } case <-ctx.Done(): return @@ -134,8 +119,8 @@ func (r *DNSResolver) resolveOnceAsync(ctx context.Context, name string, options // dnslink, then output a more specific error message if rootResErr == ErrResolveFailed && subResErr == ErrResolveFailed { // Wrap error so that it can be tested if it is a ErrResolveFailed - err := fmt.Errorf("%w: _dnslink subdomain at %q is missing a TXT record (https://docs.ipfs.tech/concepts/dnslink/)", ErrResolveFailed, gopath.Base(name)) - emitOnceResult(ctx, out, onceResult{err: err}) + err := fmt.Errorf("%w: _dnslink subdomain at %q is missing a TXT record (https://docs.ipfs.tech/concepts/dnslink/)", ErrResolveFailed, gopath.Base(fqdn)) + emitOnceResult(ctx, out, AsyncResult{Err: err}) } return } @@ -145,8 +130,8 @@ func (r *DNSResolver) resolveOnceAsync(ctx context.Context, name string, options return out } -func workDomain(ctx context.Context, r *DNSResolver, name string, res chan lookupRes) { - ctx, span := StartSpan(ctx, "DNSResolver.WorkDomain", trace.WithAttributes(attribute.String("Name", name))) +func workDomain(ctx context.Context, r *DNSResolver, name string, res chan AsyncResult) { + ctx, span := startSpan(ctx, "DNSResolver.WorkDomain", trace.WithAttributes(attribute.String("Name", name))) defer span.End() defer close(res) @@ -161,20 +146,20 @@ func workDomain(ctx context.Context, r *DNSResolver, name string, res chan looku } } // Could not look up any text records for name - res <- lookupRes{nil, err} + res <- AsyncResult{Err: err} return } for _, t := range txt { p, err := parseEntry(t) if err == nil { - res <- lookupRes{p, nil} + res <- AsyncResult{Path: p} return } } // There were no TXT records with a dnslink - res <- lookupRes{nil, ErrResolveFailed} + res <- AsyncResult{Err: ErrResolveFailed} } func parseEntry(txt string) (path.Path, error) { diff --git a/namesys/dns_resolver_test.go b/namesys/dns_resolver_test.go new file mode 100644 index 000000000..f45020a5c --- /dev/null +++ b/namesys/dns_resolver_test.go @@ -0,0 +1,197 @@ +package namesys + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDNSParseEntry(t *testing.T) { + t.Parallel() + + t.Run("Valid entries", func(t *testing.T) { + t.Parallel() + + for _, entry := range []string{ + "QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo", + "dnslink=/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz/", + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + } { + _, err := parseEntry(entry) + assert.NoError(t, err) + } + }) + + t.Run("Invalid entries", func(t *testing.T) { + t.Parallel() + + for _, entry := range []string{ + "QmYhE8xgFCjGcz6PHgnvJz5NOTCORRECT", + "quux=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + "dnslink=", + "dnslink=/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo", + "dnslink=ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar", + } { + _, err := parseEntry(entry) + assert.Error(t, err) + } + }) +} + +type mockDNS struct { + entries map[string][]string +} + +func (m *mockDNS) lookupTXT(ctx context.Context, name string) (txt []string, err error) { + txt, ok := m.entries[name] + if !ok { + return nil, fmt.Errorf("no TXT entry for %s", name) + } + return txt, nil +} + +func newMockDNS() *mockDNS { + return &mockDNS{ + entries: map[string][]string{ + "multihash.example.com.": { + "dnslink=QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "ipfs.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "_dnslink.dipfs.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "dns1.example.com.": { + "dnslink=/ipns/ipfs.example.com", + }, + "dns2.example.com.": { + "dnslink=/ipns/dns1.example.com", + }, + "multi.example.com.": { + "some stuff", + "dnslink=/ipns/dns1.example.com", + "masked dnslink=/ipns/example.invalid", + }, + "equals.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/=equals", + }, + "loop1.example.com.": { + "dnslink=/ipns/loop2.example.com", + }, + "loop2.example.com.": { + "dnslink=/ipns/loop1.example.com", + }, + "_dnslink.dloop1.example.com.": { + "dnslink=/ipns/loop2.example.com", + }, + "_dnslink.dloop2.example.com.": { + "dnslink=/ipns/loop1.example.com", + }, + "bad.example.com.": { + "dnslink=", + }, + "withsegment.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment", + }, + "withrecsegment.example.com.": { + "dnslink=/ipns/withsegment.example.com/subsub", + }, + "withtrailing.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/", + }, + "withtrailingrec.example.com.": { + "dnslink=/ipns/withtrailing.example.com/segment/", + }, + "double.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "_dnslink.double.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "double.conflict.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", + }, + "_dnslink.conflict.example.com.": { + "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjE", + }, + "fqdn.example.com.": { + "dnslink=/ipfs/QmYvMB9yrsSf7RKBghkfwmHJkzJhW2ZgVwq3LxBXXPasFr", + }, + "en.wikipedia-on-ipfs.org.": { + "dnslink=/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze", + }, + "custom.non-icann.tldextravaganza.": { + "dnslink=/ipfs/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm", + }, + "singlednslabelshouldbeok.": { + "dnslink=/ipfs/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi", + }, + "www.wealdtech.eth.": { + "dnslink=/ipns/ipfs.example.com", + }, + }, + } +} + +func TestDNSResolution(t *testing.T) { + t.Parallel() + r := &DNSResolver{lookupTXT: newMockDNS().lookupTXT} + + for _, testCase := range []struct { + name string + depth uint + expectedPath string + expectedError error + }{ + {"/ipns/multihash.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/ipfs.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/dipfs.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/dns1.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/dns1.example.com", 1, "/ipns/ipfs.example.com", ErrResolveRecursion}, + {"/ipns/dns2.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/dns2.example.com", 1, "/ipns/dns1.example.com", ErrResolveRecursion}, + {"/ipns/dns2.example.com", 2, "/ipns/ipfs.example.com", ErrResolveRecursion}, + {"/ipns/multi.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/multi.example.com", 1, "/ipns/dns1.example.com", ErrResolveRecursion}, + {"/ipns/multi.example.com", 2, "/ipns/ipfs.example.com", ErrResolveRecursion}, + {"/ipns/equals.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/=equals", nil}, + {"/ipns/loop1.example.com", 1, "/ipns/loop2.example.com", ErrResolveRecursion}, + {"/ipns/loop1.example.com", 2, "/ipns/loop1.example.com", ErrResolveRecursion}, + {"/ipns/loop1.example.com", 3, "/ipns/loop2.example.com", ErrResolveRecursion}, + {"/ipns/loop1.example.com", DefaultDepthLimit, "/ipns/loop1.example.com", ErrResolveRecursion}, + {"/ipns/dloop1.example.com", 1, "/ipns/loop2.example.com", ErrResolveRecursion}, + {"/ipns/dloop1.example.com", 2, "/ipns/loop1.example.com", ErrResolveRecursion}, + {"/ipns/dloop1.example.com", 3, "/ipns/loop2.example.com", ErrResolveRecursion}, + {"/ipns/dloop1.example.com", DefaultDepthLimit, "/ipns/loop1.example.com", ErrResolveRecursion}, + {"/ipns/bad.example.com", DefaultDepthLimit, "", ErrResolveFailed}, + {"/ipns/bad.example.com", DefaultDepthLimit, "", ErrResolveFailed}, + {"/ipns/withsegment.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment", nil}, + {"/ipns/withrecsegment.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub", nil}, + {"/ipns/withsegment.example.com/test1", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/test1", nil}, + {"/ipns/withrecsegment.example.com/test2", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test2", nil}, + {"/ipns/withrecsegment.example.com/test3/", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test3/", nil}, + {"/ipns/withtrailingrec.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/", nil}, + {"/ipns/double.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/conflict.example.com", DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjE", nil}, + {"/ipns/fqdn.example.com.", DefaultDepthLimit, "/ipfs/QmYvMB9yrsSf7RKBghkfwmHJkzJhW2ZgVwq3LxBXXPasFr", nil}, + {"/ipns/en.wikipedia-on-ipfs.org", 2, "/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze", nil}, + {"/ipns/custom.non-icann.tldextravaganza.", 2, "/ipfs/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm", nil}, + {"/ipns/singlednslabelshouldbeok", 2, "/ipfs/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi", nil}, + {"/ipns/www.wealdtech.eth", 1, "/ipns/ipfs.example.com", ErrResolveRecursion}, + {"/ipns/www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + {"/ipns/www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil}, + } { + t.Run(testCase.name, func(t *testing.T) { + testResolution(t, r, testCase.name, (testCase.depth), testCase.expectedPath, 0, testCase.expectedError) + }) + } +} diff --git a/namesys/dns_test.go b/namesys/dns_test.go deleted file mode 100644 index a31a53582..000000000 --- a/namesys/dns_test.go +++ /dev/null @@ -1,184 +0,0 @@ -package namesys - -import ( - "context" - "fmt" - "testing" - - opts "github.com/ipfs/boxo/coreiface/options/namesys" -) - -type mockDNS struct { - entries map[string][]string -} - -func (m *mockDNS) lookupTXT(ctx context.Context, name string) (txt []string, err error) { - txt, ok := m.entries[name] - if !ok { - return nil, fmt.Errorf("no TXT entry for %s", name) - } - return txt, nil -} - -func TestDnsEntryParsing(t *testing.T) { - goodEntries := []string{ - "QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - "dnslink=/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo", - "dnslink=/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo/bar/baz/", - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - } - - badEntries := []string{ - "QmYhE8xgFCjGcz6PHgnvJz5NOTCORRECT", - "quux=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - "dnslink=", - "dnslink=/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/foo", - "dnslink=ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/bar", - } - - for _, e := range goodEntries { - _, err := parseEntry(e) - if err != nil { - t.Log("expected entry to parse correctly!") - t.Log(e) - t.Fatal(err) - } - } - - for _, e := range badEntries { - _, err := parseEntry(e) - if err == nil { - t.Log("expected entry parse to fail!") - t.Fatal(err) - } - } -} - -func newMockDNS() *mockDNS { - return &mockDNS{ - entries: map[string][]string{ - "multihash.example.com.": { - "dnslink=QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - }, - "ipfs.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - }, - "_dnslink.dipfs.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - }, - "dns1.example.com.": { - "dnslink=/ipns/ipfs.example.com", - }, - "dns2.example.com.": { - "dnslink=/ipns/dns1.example.com", - }, - "multi.example.com.": { - "some stuff", - "dnslink=/ipns/dns1.example.com", - "masked dnslink=/ipns/example.invalid", - }, - "equals.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/=equals", - }, - "loop1.example.com.": { - "dnslink=/ipns/loop2.example.com", - }, - "loop2.example.com.": { - "dnslink=/ipns/loop1.example.com", - }, - "_dnslink.dloop1.example.com.": { - "dnslink=/ipns/loop2.example.com", - }, - "_dnslink.dloop2.example.com.": { - "dnslink=/ipns/loop1.example.com", - }, - "bad.example.com.": { - "dnslink=", - }, - "withsegment.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment", - }, - "withrecsegment.example.com.": { - "dnslink=/ipns/withsegment.example.com/subsub", - }, - "withtrailing.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/", - }, - "withtrailingrec.example.com.": { - "dnslink=/ipns/withtrailing.example.com/segment/", - }, - "double.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - }, - "_dnslink.double.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - }, - "double.conflict.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", - }, - "_dnslink.conflict.example.com.": { - "dnslink=/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjE", - }, - "fqdn.example.com.": { - "dnslink=/ipfs/QmYvMB9yrsSf7RKBghkfwmHJkzJhW2ZgVwq3LxBXXPasFr", - }, - "en.wikipedia-on-ipfs.org.": { - "dnslink=/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze", - }, - "custom.non-icann.tldextravaganza.": { - "dnslink=/ipfs/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm", - }, - "singlednslabelshouldbeok.": { - "dnslink=/ipfs/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi", - }, - "www.wealdtech.eth.": { - "dnslink=/ipns/ipfs.example.com", - }, - }, - } -} - -func TestDNSResolution(t *testing.T) { - mock := newMockDNS() - r := &DNSResolver{lookupTXT: mock.lookupTXT} - testResolution(t, r, "multihash.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "ipfs.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "dipfs.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "dns1.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "dns1.example.com", 1, "/ipns/ipfs.example.com", ErrResolveRecursion) - testResolution(t, r, "dns2.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "dns2.example.com", 1, "/ipns/dns1.example.com", ErrResolveRecursion) - testResolution(t, r, "dns2.example.com", 2, "/ipns/ipfs.example.com", ErrResolveRecursion) - testResolution(t, r, "multi.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "multi.example.com", 1, "/ipns/dns1.example.com", ErrResolveRecursion) - testResolution(t, r, "multi.example.com", 2, "/ipns/ipfs.example.com", ErrResolveRecursion) - testResolution(t, r, "equals.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/=equals", nil) - testResolution(t, r, "loop1.example.com", 1, "/ipns/loop2.example.com", ErrResolveRecursion) - testResolution(t, r, "loop1.example.com", 2, "/ipns/loop1.example.com", ErrResolveRecursion) - testResolution(t, r, "loop1.example.com", 3, "/ipns/loop2.example.com", ErrResolveRecursion) - testResolution(t, r, "loop1.example.com", opts.DefaultDepthLimit, "/ipns/loop1.example.com", ErrResolveRecursion) - testResolution(t, r, "dloop1.example.com", 1, "/ipns/loop2.example.com", ErrResolveRecursion) - testResolution(t, r, "dloop1.example.com", 2, "/ipns/loop1.example.com", ErrResolveRecursion) - testResolution(t, r, "dloop1.example.com", 3, "/ipns/loop2.example.com", ErrResolveRecursion) - testResolution(t, r, "dloop1.example.com", opts.DefaultDepthLimit, "/ipns/loop1.example.com", ErrResolveRecursion) - testResolution(t, r, "bad.example.com", opts.DefaultDepthLimit, "", ErrResolveFailed) - testResolution(t, r, "withsegment.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment", nil) - testResolution(t, r, "withrecsegment.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub", nil) - testResolution(t, r, "withsegment.example.com/test1", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/test1", nil) - testResolution(t, r, "withrecsegment.example.com/test2", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test2", nil) - testResolution(t, r, "withrecsegment.example.com/test3/", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/subsub/test3/", nil) - testResolution(t, r, "withtrailingrec.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD/sub/segment/", nil) - testResolution(t, r, "double.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "conflict.example.com", opts.DefaultDepthLimit, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjE", nil) - testResolution(t, r, "fqdn.example.com.", opts.DefaultDepthLimit, "/ipfs/QmYvMB9yrsSf7RKBghkfwmHJkzJhW2ZgVwq3LxBXXPasFr", nil) - testResolution(t, r, "en.wikipedia-on-ipfs.org", 2, "/ipfs/bafybeiaysi4s6lnjev27ln5icwm6tueaw2vdykrtjkwiphwekaywqhcjze", nil) - testResolution(t, r, "custom.non-icann.tldextravaganza.", 2, "/ipfs/bafybeieto6mcuvqlechv4iadoqvnffondeiwxc2bcfcewhvpsd2odvbmvm", nil) - testResolution(t, r, "singlednslabelshouldbeok", 2, "/ipfs/bafybeih4a6ylafdki6ailjrdvmr7o4fbbeceeeuty4v3qyyouiz5koqlpi", nil) - testResolution(t, r, "www.wealdtech.eth", 1, "/ipns/ipfs.example.com", ErrResolveRecursion) - testResolution(t, r, "www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) - testResolution(t, r, "www.wealdtech.eth", 2, "/ipfs/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", nil) -} diff --git a/namesys/interface.go b/namesys/interface.go index 5d50936ee..21c93126c 100644 --- a/namesys/interface.go +++ b/namesys/interface.go @@ -1,52 +1,50 @@ -/* -Package namesys implements resolvers and publishers for the IPFS -naming system (IPNS). - -The core of IPFS is an immutable, content-addressable Merkle graph. -That works well for many use cases, but doesn't allow you to answer -questions like "what is Alice's current homepage?". The mutable name -system allows Alice to publish information like: - - The current homepage for alice.example.com is - /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj - -or: - - The current homepage for node - QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy - is - /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj - -The mutable name system also allows users to resolve those references -to find the immutable IPFS object currently referenced by a given -mutable name. - -For command-line bindings to this functionality, see: - - ipfs name - ipfs dns - ipfs resolve -*/ package namesys import ( "context" "errors" + "time" - opts "github.com/ipfs/boxo/coreiface/options/namesys" + "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" + logging "github.com/ipfs/go-log/v2" ci "github.com/libp2p/go-libp2p/core/crypto" ) -// ErrResolveFailed signals an error when attempting to resolve. -var ErrResolveFailed = errors.New("could not resolve name") +var log = logging.Logger("namesys") -// ErrResolveRecursion signals a recursion-depth limit. -var ErrResolveRecursion = errors.New( - "could not resolve name (recursion limit exceeded)") +var ( + // ErrResolveFailed signals an error when attempting to resolve. + ErrResolveFailed = errors.New("could not resolve name") -// ErrPublishFailed signals an error when attempting to publish. -var ErrPublishFailed = errors.New("could not publish name") + // ErrResolveRecursion signals a recursion-depth limit. + ErrResolveRecursion = errors.New("could not resolve name (recursion limit exceeded)") + + // ErrNoNamesys is an explicit error for when no [NameSystem] is provided. + ErrNoNamesys = errors.New("no namesys has been provided") +) + +const ( + // DefaultDepthLimit is the default depth limit used by [Resolver]. + DefaultDepthLimit = 32 + + // UnlimitedDepth allows infinite recursion in [Resolver]. You probably don't want + // to use this, but it's here if you absolutely trust resolution to eventually + // complete and can't put an upper limit on how many steps it will take. + UnlimitedDepth = 0 + + // DefaultResolverRecordCount is the number of IPNS Record copies to + // retrieve from the routing system like Amino DHT (the best record is + // selected from this set). + DefaultResolverDhtRecordCount = 16 + + // DefaultResolverDhtTimeout is the amount of time to wait for records to be fetched + // and verified. + DefaultResolverDhtTimeout = time.Minute + + // DefaultResolverCacheTTL defines default TTL of a record placed in [NameSystem] cache. + DefaultResolverCacheTTL = time.Minute +) // NameSystem represents a cohesive name publishing and resolving system. // @@ -60,39 +58,169 @@ type NameSystem interface { Publisher } -// Result is the return type for Resolver.ResolveAsync. +// Result is the return type for [Resolver.Resolve]. type Result struct { - Path path.Path - Err error + Path path.Path + TTL time.Duration + LastMod time.Time +} + +// AsyncResult is the return type for [Resolver.ResolveAsync]. +type AsyncResult struct { + Path path.Path + TTL time.Duration + LastMod time.Time + Err error } // Resolver is an object capable of resolving names. type Resolver interface { - // Resolve performs a recursive lookup, returning the dereferenced - // path. For example, if ipfs.io has a DNS TXT record pointing to + // Resolve performs a recursive lookup, returning the dereferenced path and the TTL. + // If the TTL is unknown, then a TTL of 0 is returned. For example, if example.com + // has a DNS TXT record pointing to: + // // /ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy + // // and there is a DHT IPNS entry for + // // QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy // -> /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj + // // then + // // Resolve(ctx, "/ipns/ipfs.io") + // // will resolve both names, returning + // // /ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj // - // There is a default depth-limit to avoid infinite recursion. Most - // users will be fine with this default limit, but if you need to - // adjust the limit you can specify it as an option. - Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (value path.Path, err error) + // There is a default depth-limit to avoid infinite recursion. Most users will be fine with + // this default limit, but if you need to adjust the limit you can specify it as an option. + Resolve(context.Context, path.Path, ...ResolveOption) (Result, error) + + // ResolveAsync performs recursive name lookup, like Resolve, but it returns entries as + // they are discovered in the DHT. Each returned result is guaranteed to be "better" + // (which usually means newer) than the previous one. + ResolveAsync(context.Context, path.Path, ...ResolveOption) <-chan AsyncResult +} + +// ResolveOptions specifies options for resolving an IPNS Path. +type ResolveOptions struct { + // Depth is the recursion depth limit. + Depth uint + + // DhtRecordCount is the number of IPNS Records to retrieve from the routing system + // (the best record is selected from this set). + DhtRecordCount uint - // ResolveAsync performs recursive name lookup, like Resolve, but it returns - // entries as they are discovered in the DHT. Each returned result is guaranteed - // to be "better" (which usually means newer) than the previous one. - ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result + // DhtTimeout is the amount of time to wait for records to be fetched and + // verified. A zero value indicates that there is no explicit timeout + // (although there may be an implicit timeout due to dial timeouts within + // the specific routing system like DHT). + DhtTimeout time.Duration +} + +// DefaultResolveOptions returns the default options for resolving an IPNS Path. +func DefaultResolveOptions() ResolveOptions { + return ResolveOptions{ + Depth: DefaultDepthLimit, + DhtRecordCount: DefaultResolverDhtRecordCount, + DhtTimeout: DefaultResolverDhtTimeout, + } +} + +// ResolveOption is used to set a resolve option. +type ResolveOption func(*ResolveOptions) + +// ResolveWithDepth sets [ResolveOptions.Depth]. +func ResolveWithDepth(depth uint) ResolveOption { + return func(o *ResolveOptions) { + o.Depth = depth + } +} + +// ResolveWithDhtRecordCount sets [ResolveOptions.DhtRecordCount]. +func ResolveWithDhtRecordCount(count uint) ResolveOption { + return func(o *ResolveOptions) { + o.DhtRecordCount = count + } +} + +// ResolveWithDhtTimeout sets [ResolveOptions.ResolveWithDhtTimeout]. +func ResolveWithDhtTimeout(timeout time.Duration) ResolveOption { + return func(o *ResolveOptions) { + o.DhtTimeout = timeout + } +} + +// ProcessResolveOptions converts an array of [ResolveOption] into a [ResolveOptions] object. +func ProcessResolveOptions(opts []ResolveOption) ResolveOptions { + resolveOptions := DefaultResolveOptions() + for _, option := range opts { + option(&resolveOptions) + } + return resolveOptions } // Publisher is an object capable of publishing particular names. type Publisher interface { - // Publish establishes a name-value mapping. - // TODO make this not PrivKey specific. - Publish(ctx context.Context, name ci.PrivKey, value path.Path, options ...opts.PublishOption) error + // Publish publishes the given value under the name represented by the given private key. + Publish(ctx context.Context, sk ci.PrivKey, value path.Path, options ...PublishOption) error +} + +// PublishOptions specifies options for publishing an IPNS Record. +type PublishOptions struct { + // EOL defines for how long the published value is valid. + EOL time.Time + + // TTL defines for how long the published value is cached locally before checking for updates. + TTL time.Duration + + // IPNSOptions are options passed by [IPNSPublisher] to [ipns.NewRecord] when + // creating a new record to publish. With this options, you can further customize + // the way IPNS Records are created. + IPNSOptions []ipns.Option +} + +// DefaultPublishOptions returns the default options for publishing an IPNS Record. +func DefaultPublishOptions() PublishOptions { + return PublishOptions{ + EOL: time.Now().Add(ipns.DefaultRecordLifetime), + TTL: ipns.DefaultRecordTTL, + } +} + +// PublishOption is used to set an option for [PublishOptions]. +type PublishOption func(*PublishOptions) + +// PublishWithEOL sets [PublishOptions.EOL]. +func PublishWithEOL(eol time.Time) PublishOption { + return func(o *PublishOptions) { + o.EOL = eol + } +} + +// PublishWithEOL sets [PublishOptions.TTL]. +func PublishWithTTL(ttl time.Duration) PublishOption { + return func(o *PublishOptions) { + o.TTL = ttl + } +} + +// PublishWithIPNSOption adds an [ipns.Option] to [PublishOptions.IPNSOptions]. +// These options are used by [IPNSPublisher], which passes them onto the IPNS +// record creation at [ipns.NewRecord] +func PublishWithIPNSOption(option ipns.Option) PublishOption { + return func(o *PublishOptions) { + o.IPNSOptions = append(o.IPNSOptions, option) + } +} + +// ProcessPublishOptions converts an array of [PublishOption] into a [PublishOptions] object. +func ProcessPublishOptions(opts []PublishOption) PublishOptions { + publishOptions := DefaultPublishOptions() + for _, option := range opts { + option(&publishOptions) + } + return publishOptions } diff --git a/namesys/ipns_publisher.go b/namesys/ipns_publisher.go new file mode 100644 index 000000000..74ef35853 --- /dev/null +++ b/namesys/ipns_publisher.go @@ -0,0 +1,283 @@ +package namesys + +import ( + "context" + "errors" + "strings" + "sync" + "time" + + "github.com/ipfs/boxo/ipns" + "github.com/ipfs/boxo/path" + ds "github.com/ipfs/go-datastore" + dsquery "github.com/ipfs/go-datastore/query" + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/routing" + "github.com/whyrusleeping/base32" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// IPNSPublisher implements [Publisher] for IPNS Records. +type IPNSPublisher struct { + routing routing.ValueStore + ds ds.Datastore + + // Used to ensure we assign IPNS records sequential sequence numbers. + mu sync.Mutex +} + +var _ Publisher = &IPNSPublisher{} + +// NewIPNSResolver constructs a new [IPNSResolver] from a [routing.ValueStore] and +// a [ds.Datastore]. +func NewIPNSPublisher(route routing.ValueStore, ds ds.Datastore) *IPNSPublisher { + if ds == nil { + panic("nil datastore") + } + + return &IPNSPublisher{routing: route, ds: ds} +} + +func (p *IPNSPublisher) Publish(ctx context.Context, priv crypto.PrivKey, value path.Path, options ...PublishOption) error { + log.Debugf("Publish %s", value) + + ctx, span := startSpan(ctx, "IPNSPublisher.Publish", trace.WithAttributes(attribute.String("Value", value.String()))) + defer span.End() + + record, err := p.updateRecord(ctx, priv, value, options...) + if err != nil { + return err + } + + return PublishIPNSRecord(ctx, p.routing, priv.GetPublic(), record) +} + +// IpnsDsKey returns a datastore key given an IPNS identifier (peer +// ID). Defines the storage key for IPNS records in the local datastore. +func IpnsDsKey(name ipns.Name) ds.Key { + return ds.NewKey("/ipns/" + base32.RawStdEncoding.EncodeToString([]byte(name.Peer()))) +} + +// ListPublished returns the latest IPNS records published by this node and +// their expiration times. +// +// This method will not search the routing system for records published by other +// nodes. +func (p *IPNSPublisher) ListPublished(ctx context.Context) (map[ipns.Name]*ipns.Record, error) { + query, err := p.ds.Query(ctx, dsquery.Query{ + Prefix: ipns.NamespacePrefix, + }) + if err != nil { + return nil, err + } + defer query.Close() + + records := make(map[ipns.Name]*ipns.Record) + for { + select { + case result, ok := <-query.Next(): + if !ok { + return records, nil + } + if result.Error != nil { + return nil, result.Error + } + rec, err := ipns.UnmarshalRecord(result.Value) + if err != nil { + // Might as well return what we can. + log.Error("found an invalid IPNS entry:", err) + continue + } + if !strings.HasPrefix(result.Key, ipns.NamespacePrefix) { + log.Errorf("datastore query for keys with prefix %s returned a key: %s", ipns.NamespacePrefix, result.Key) + continue + } + k := result.Key[len(ipns.NamespacePrefix):] + pid, err := base32.RawStdEncoding.DecodeString(k) + if err != nil { + log.Errorf("ipns ds key invalid: %s", result.Key) + continue + } + records[ipns.NameFromPeer(peer.ID(pid))] = rec + case <-ctx.Done(): + return nil, ctx.Err() + } + } +} + +// GetPublished returns the record this node has published corresponding to the +// given peer ID. +// +// If `checkRouting` is true and we have no existing record, this method will +// check the routing system for any existing records. +func (p *IPNSPublisher) GetPublished(ctx context.Context, name ipns.Name, checkRouting bool) (*ipns.Record, error) { + ctx, cancel := context.WithTimeout(ctx, time.Second*30) + defer cancel() + + value, err := p.ds.Get(ctx, IpnsDsKey(name)) + switch err { + case nil: + case ds.ErrNotFound: + if !checkRouting { + return nil, nil + } + routingKey := name.RoutingKey() + value, err = p.routing.GetValue(ctx, string(routingKey)) + if err != nil { + // Not found or other network issue. Can't really do + // anything about this case. + if err != routing.ErrNotFound { + log.Debugf("error when determining the last published IPNS record for %s: %s", name, err) + } + + return nil, nil + } + default: + return nil, err + } + + return ipns.UnmarshalRecord(value) +} + +func (p *IPNSPublisher) updateRecord(ctx context.Context, k crypto.PrivKey, value path.Path, options ...PublishOption) (*ipns.Record, error) { + id, err := peer.IDFromPrivateKey(k) + if err != nil { + return nil, err + } + name := ipns.NameFromPeer(id) + + p.mu.Lock() + defer p.mu.Unlock() + + // get previous records sequence number + rec, err := p.GetPublished(ctx, name, true) + if err != nil { + return nil, err + } + + seq := uint64(0) + if rec != nil { + seq, err = rec.Sequence() + if err != nil { + return nil, err + } + + p, err := rec.Value() + if err != nil { + return nil, err + } + if value.String() != p.String() { + // Don't bother incrementing the sequence number unless the + // value changes. + seq++ + } + } + + opts := ProcessPublishOptions(options) + + // Create record + r, err := ipns.NewRecord(k, value, seq, opts.EOL, opts.TTL, opts.IPNSOptions...) + if err != nil { + return nil, err + } + + data, err := ipns.MarshalRecord(r) + if err != nil { + return nil, err + } + + // Put the new record. + dsKey := IpnsDsKey(name) + if err := p.ds.Put(ctx, dsKey, data); err != nil { + return nil, err + } + if err := p.ds.Sync(ctx, dsKey); err != nil { + return nil, err + } + + return r, nil +} + +// PublishIPNSRecord publishes the given [ipns.Record] for the provided [crypto.PubKey] in +// the provided [routing.ValueStore]. The public key is also made available to the routing +// system if it cannot be derived from the corresponding [peer.ID]. +func PublishIPNSRecord(ctx context.Context, r routing.ValueStore, pubKey crypto.PubKey, rec *ipns.Record) error { + ctx, span := startSpan(ctx, "PublishIPNSRecord") + defer span.End() + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + errs := make(chan error, 2) // At most two errors (IPNS, and public key) + + pid, err := peer.IDFromPublicKey(pubKey) + if err != nil { + return err + } + + go func() { + errs <- PutIPNSRecord(ctx, r, ipns.NameFromPeer(pid), rec) + }() + + // Publish the public key if the public key cannot be extracted from the peer ID. + // This is most likely not necessary since IPNS Records include, by default, the public + // key in those cases. However, this ensures it's still possible to easily retrieve + // the public key if, for some reason, it is not embedded. + if _, err := pid.ExtractPublicKey(); errors.Is(err, peer.ErrNoPublicKey) { + go func() { + errs <- PutPublicKey(ctx, r, pid, pubKey) + }() + + if err := waitOnErrChan(ctx, errs); err != nil { + return err + } + } + + return waitOnErrChan(ctx, errs) +} + +func waitOnErrChan(ctx context.Context, errs chan error) error { + select { + case err := <-errs: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +// PutPublicKey puts the given [crypto.PubKey] for the given [peer.ID] in the [routing.ValueStore]. +func PutPublicKey(ctx context.Context, r routing.ValueStore, pid peer.ID, pubKey crypto.PubKey) error { + routingKey := PkRoutingKey(pid) + ctx, span := startSpan(ctx, "PutPublicKey", trace.WithAttributes(attribute.String("Key", routingKey))) + defer span.End() + + bytes, err := crypto.MarshalPublicKey(pubKey) + if err != nil { + return err + } + + log.Debugf("Storing public key at: %x", routingKey) + return r.PutValue(ctx, routingKey, bytes) +} + +// PkRoutingKey returns the public key routing key for the given [peer.ID]. +func PkRoutingKey(id peer.ID) string { + return "/pk/" + string(id) +} + +// PutIPNSRecord puts the given [ipns.Record] for the given [ipns.Name] in the [routing.ValueStore]. +func PutIPNSRecord(ctx context.Context, r routing.ValueStore, name ipns.Name, rec *ipns.Record) error { + routingKey := string(name.RoutingKey()) + ctx, span := startSpan(ctx, "PutIPNSRecord", trace.WithAttributes(attribute.String("IPNSKey", routingKey))) + defer span.End() + + bytes, err := ipns.MarshalRecord(rec) + if err != nil { + return err + } + + log.Debugf("Storing ipns record at: %x", routingKey) + return r.PutValue(ctx, routingKey, bytes) +} diff --git a/namesys/ipns_publisher_test.go b/namesys/ipns_publisher_test.go new file mode 100644 index 000000000..a783ddd61 --- /dev/null +++ b/namesys/ipns_publisher_test.go @@ -0,0 +1,121 @@ +package namesys + +import ( + "context" + "crypto/rand" + "testing" + "time" + + "github.com/ipfs/boxo/path" + "github.com/stretchr/testify/require" + + dshelp "github.com/ipfs/boxo/datastore/dshelp" + "github.com/ipfs/boxo/ipns" + mockrouting "github.com/ipfs/boxo/routing/mock" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + testutil "github.com/libp2p/go-libp2p-testing/net" + ci "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" +) + +func TestIPNSPublisher(t *testing.T) { + t.Parallel() + + test := func(t *testing.T, keyType int, expectedErr error, expectedExistence bool) { + ctx := context.Background() + + // Create test identity + privKey, pubKey, err := ci.GenerateKeyPairWithReader(keyType, 2048, rand.Reader) + require.NoError(t, err) + + pid, err := peer.IDFromPublicKey(pubKey) + require.NoError(t, err) + + // Create IPNS Record + value, err := path.NewPath("/ipfs/bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") + require.NoError(t, err) + rec, err := ipns.NewRecord(privKey, value, 0, time.Now().Add(24*time.Hour), 0) + require.NoError(t, err) + + // Routing value store + dstore := dssync.MutexWrap(ds.NewMapDatastore()) + serv := mockrouting.NewServer() + r := serv.ClientWithDatastore(context.Background(), testutil.NewIdentity(pid, testutil.ZeroLocalTCPAddress, privKey, pubKey), dstore) + + // Publish IPNS Record + err = PublishIPNSRecord(ctx, r, pubKey, rec) + require.NoError(t, err) + + // Check if IPNS Record is stored in value store + _, err = r.GetValue(ctx, string(ipns.NameFromPeer(pid).RoutingKey())) + require.NoError(t, err) + + key := dshelp.NewKeyFromBinary(ipns.NameFromPeer(pid).RoutingKey()) + exists, err := dstore.Has(ctx, key) + require.NoError(t, err) + require.True(t, exists) + + // Check for Public Key is stored in value store + pkRoutingKey := PkRoutingKey(pid) + _, err = r.GetValue(ctx, pkRoutingKey) + require.ErrorIs(t, err, expectedErr) + + // Check if Public Key is in data store for completeness + key = dshelp.NewKeyFromBinary([]byte(pkRoutingKey)) + exists, err = dstore.Has(ctx, key) + require.NoError(t, err) + require.Equal(t, expectedExistence, exists) + } + + t.Run("RSA", func(t *testing.T) { + t.Parallel() + test(t, ci.RSA, nil, true) + }) + + t.Run("Ed22519", func(t *testing.T) { + t.Parallel() + test(t, ci.Ed25519, ds.ErrNotFound, false) + }) +} + +func TestAsyncDS(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + rt := mockrouting.NewServer().Client(testutil.RandIdentityOrFatal(t)) + ds := &checkSyncDS{ + Datastore: ds.NewMapDatastore(), + syncKeys: make(map[ds.Key]struct{}), + } + publisher := NewIPNSPublisher(rt, ds) + + ipnsFakeID := testutil.RandIdentityOrFatal(t) + ipnsVal, err := path.NewPath("/ipns/foo.bar") + require.NoError(t, err) + + err = publisher.Publish(ctx, ipnsFakeID.PrivateKey(), ipnsVal) + require.NoError(t, err) + + ipnsKey := IpnsDsKey(ipns.NameFromPeer(ipnsFakeID.ID())) + + for k := range ds.syncKeys { + if k.IsAncestorOf(ipnsKey) || k.Equal(ipnsKey) { + return + } + } + + t.Fatal("ipns key not synced") +} + +type checkSyncDS struct { + ds.Datastore + syncKeys map[ds.Key]struct{} +} + +func (d *checkSyncDS) Sync(ctx context.Context, prefix ds.Key) error { + d.syncKeys[prefix] = struct{}{} + return d.Datastore.Sync(ctx, prefix) +} diff --git a/namesys/ipns_resolver.go b/namesys/ipns_resolver.go new file mode 100644 index 000000000..5efcf8785 --- /dev/null +++ b/namesys/ipns_resolver.go @@ -0,0 +1,158 @@ +package namesys + +import ( + "context" + "fmt" + "time" + + "github.com/ipfs/boxo/ipns" + "github.com/ipfs/boxo/path" + + dht "github.com/libp2p/go-libp2p-kad-dht" + "github.com/libp2p/go-libp2p/core/routing" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +// IPNSResolver implements [Resolver] for IPNS Records. This resolver always returns +// a TTL if the record is still valid. It happens as follows: +// +// 1. Provisory TTL is chosen: record TTL if it exists, otherwise `ipns.DefaultRecordTTL`. +// 2. If provisory TTL expires before EOL, then returned TTL is duration between EOL and now. +// 3. If record is expired, 0 is returned as TTL. +type IPNSResolver struct { + routing routing.ValueStore +} + +var _ Resolver = &IPNSResolver{} + +// NewIPNSResolver constructs a new [IPNSResolver] from a [routing.ValueStore]. +func NewIPNSResolver(route routing.ValueStore) *IPNSResolver { + if route == nil { + panic("attempt to create resolver with nil routing system") + } + + return &IPNSResolver{ + routing: route, + } +} + +func (r *IPNSResolver) Resolve(ctx context.Context, p path.Path, options ...ResolveOption) (Result, error) { + ctx, span := startSpan(ctx, "IPNSResolver.Resolve", trace.WithAttributes(attribute.Stringer("Path", p))) + defer span.End() + + return resolve(ctx, r, p, ProcessResolveOptions(options)) +} + +func (r *IPNSResolver) ResolveAsync(ctx context.Context, p path.Path, options ...ResolveOption) <-chan AsyncResult { + ctx, span := startSpan(ctx, "IPNSResolver.ResolveAsync", trace.WithAttributes(attribute.Stringer("Path", p))) + defer span.End() + + return resolveAsync(ctx, r, p, ProcessResolveOptions(options)) +} + +func (r *IPNSResolver) resolveOnceAsync(ctx context.Context, p path.Path, options ResolveOptions) <-chan AsyncResult { + ctx, span := startSpan(ctx, "IPNSResolver.ResolveOnceAsync", trace.WithAttributes(attribute.Stringer("Path", p))) + defer span.End() + + out := make(chan AsyncResult, 1) + if p.Namespace() != path.IPNSNamespace { + out <- AsyncResult{Err: fmt.Errorf("unsupported namespace: %s", p.Namespace())} + close(out) + return out + } + + cancel := func() {} + if options.DhtTimeout != 0 { + // Resolution must complete within the timeout + ctx, cancel = context.WithTimeout(ctx, options.DhtTimeout) + } + + name, err := ipns.NameFromString(p.Segments()[1]) + if err != nil { + out <- AsyncResult{Err: err} + close(out) + cancel() + return out + } + + vals, err := r.routing.SearchValue(ctx, string(name.RoutingKey()), dht.Quorum(int(options.DhtRecordCount))) + if err != nil { + out <- AsyncResult{Err: err} + close(out) + cancel() + return out + } + + go func() { + defer cancel() + defer close(out) + ctx, span := startSpan(ctx, "IPNSResolver.ResolveOnceAsync.Worker") + defer span.End() + + for { + select { + case val, ok := <-vals: + if !ok { + return + } + + rec, err := ipns.UnmarshalRecord(val) + if err != nil { + emitOnceResult(ctx, out, AsyncResult{Err: err}) + return + } + + resolvedBase, err := rec.Value() + if err != nil { + emitOnceResult(ctx, out, AsyncResult{Err: err}) + return + } + + resolvedBase, err = joinPaths(resolvedBase, p) + if err != nil { + emitOnceResult(ctx, out, AsyncResult{Err: err}) + return + } + + ttl, err := calculateBestTTL(rec) + if err != nil { + emitOnceResult(ctx, out, AsyncResult{Err: err}) + return + } + + // TODO: in the future it would be interesting to set the last modified date + // as the date in which the record has been signed. + emitOnceResult(ctx, out, AsyncResult{Path: resolvedBase, TTL: ttl, LastMod: time.Now()}) + case <-ctx.Done(): + return + } + } + }() + + return out +} + +func calculateBestTTL(rec *ipns.Record) (time.Duration, error) { + ttl := DefaultResolverCacheTTL + if recordTTL, err := rec.TTL(); err == nil { + ttl = recordTTL + } + + switch eol, err := rec.Validity(); err { + case ipns.ErrUnrecognizedValidity: + // No EOL. + case nil: + ttEol := time.Until(eol) + if ttEol < 0 { + // It *was* valid when we first resolved it. + ttl = 0 + } else if ttEol < ttl { + ttl = ttEol + } + default: + return 0, err + } + + return ttl, nil +} diff --git a/namesys/ipns_resolver_test.go b/namesys/ipns_resolver_test.go new file mode 100644 index 000000000..c7dcd3b97 --- /dev/null +++ b/namesys/ipns_resolver_test.go @@ -0,0 +1,131 @@ +package namesys + +import ( + "context" + "testing" + "time" + + ipns "github.com/ipfs/boxo/ipns" + "github.com/ipfs/boxo/path" + "github.com/ipfs/boxo/routing/offline" + "github.com/ipfs/go-cid" + ds "github.com/ipfs/go-datastore" + dssync "github.com/ipfs/go-datastore/sync" + record "github.com/libp2p/go-libp2p-record" + tnet "github.com/libp2p/go-libp2p-testing/net" + "github.com/libp2p/go-libp2p/core/routing" + "github.com/stretchr/testify/require" +) + +type noFailValidator struct{} + +func (v noFailValidator) Validate(key string, value []byte) error { + return nil +} + +func (v noFailValidator) Select(key string, values [][]byte) (int, error) { + return 0, nil +} + +func TestResolver(t *testing.T) { + t.Parallel() + + pathCat := path.FromCid(cid.MustParse("bafkqabddmf2au")) + pathDog := path.FromCid(cid.MustParse("bafkqabden5tqu")) + + makeResolverDependencies := func() (tnet.Identity, ipns.Name, ds.Datastore, routing.ValueStore) { + ds := dssync.MutexWrap(ds.NewMapDatastore()) + id := tnet.RandIdentityOrFatal(t) + r := offline.NewOfflineRouter(ds, record.NamespacedValidator{ + "ipns": ipns.Validator{}, // No need for KeyBook, as records created by NameSys include PublicKey for RSA. + "pk": record.PublicKeyValidator{}, + }) + + return id, ipns.NameFromPeer(id.ID()), ds, r + } + + t.Run("Publish and resolve", func(t *testing.T) { + t.Parallel() + + id, name, ds, r := makeResolverDependencies() + resolver := NewIPNSResolver(r) + publisher := NewIPNSPublisher(r, ds) + + err := publisher.Publish(context.Background(), id.PrivateKey(), pathCat) + require.NoError(t, err) + + res, err := resolver.Resolve(context.Background(), name.AsPath()) + require.NoError(t, err) + require.Equal(t, pathCat, res.Path) + }) + + t.Run("Resolve does not return expired record", func(t *testing.T) { + t.Parallel() + + id, name, ds, r := makeResolverDependencies() + resolver := NewIPNSResolver(r) + + // Create a "bad" publisher that allows to publish expired records. + publisher := NewIPNSPublisher(offline.NewOfflineRouter(ds, record.NamespacedValidator{ + "ipns": noFailValidator{}, + "pk": record.PublicKeyValidator{}, + }), ds) + + // Publish expired. + eol := time.Now().Add(time.Hour * -1) + err := publisher.Publish(context.Background(), id.PrivateKey(), pathCat, PublishWithEOL(eol)) + require.NoError(t, err) + + // Expect to not be able to resolve. + _, err = resolver.Resolve(context.Background(), name.AsPath()) + require.ErrorIs(t, err, ErrResolveFailed) + }) + + t.Run("Resolve prefers non-expired record", func(t *testing.T) { + t.Parallel() + + id, name, ds, r := makeResolverDependencies() + resolver := NewIPNSResolver(r) + + // Create a "bad" publisher that allows to publish expired records. + publisher := NewIPNSPublisher(offline.NewOfflineRouter(ds, record.NamespacedValidator{ + "ipns": noFailValidator{}, + "pk": record.PublicKeyValidator{}, + }), ds) + + // Publish expired. + eol := time.Now().Add(time.Hour * -1) + err := publisher.Publish(context.Background(), id.PrivateKey(), pathCat, PublishWithEOL(eol)) + require.NoError(t, err) + + // Publish new. + err = publisher.Publish(context.Background(), id.PrivateKey(), pathDog) + require.NoError(t, err) + + // Expect new. + res, err := resolver.Resolve(context.Background(), name.AsPath()) + require.NoError(t, err) + require.Equal(t, pathDog, res.Path) + }) + + t.Run("Resolve prefers newer record", func(t *testing.T) { + t.Parallel() + + id, name, ds, r := makeResolverDependencies() + resolver := NewIPNSResolver(r) + publisher := NewIPNSPublisher(r, ds) + + // Publish one... + err := publisher.Publish(context.Background(), id.PrivateKey(), pathCat, PublishWithEOL(time.Now().Add(time.Hour*2))) + require.NoError(t, err) + + // Publish two... + err = publisher.Publish(context.Background(), id.PrivateKey(), pathDog, PublishWithEOL(time.Now().Add(time.Hour*5))) + require.NoError(t, err) + + // Should receive newer! + res, err := resolver.Resolve(context.Background(), name.AsPath()) + require.NoError(t, err) + require.Equal(t, pathDog, res.Path) + }) +} diff --git a/namesys/namesys.go b/namesys/namesys.go index 381e5bff1..00b1f4d2d 100644 --- a/namesys/namesys.go +++ b/namesys/namesys.go @@ -20,7 +20,7 @@ import ( "time" lru "github.com/hashicorp/golang-lru/v2" - opts "github.com/ipfs/boxo/coreiface/options/namesys" + "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" @@ -32,35 +32,38 @@ import ( madns "github.com/multiformats/go-multiaddr-dns" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "go.uber.org/multierr" ) -// mpns (a multi-protocol NameSystem) implements generic IPFS naming. +// namesys is a multi-protocol [NameSystem] that implements generic IPFS naming. +// It uses several [Resolver]s: // -// Uses several Resolvers: -// (a) IPFS routing naming: SFS-like PKI names. -// (b) dns domains: resolves using links in DNS TXT records +// 1. IPFS routing naming: SFS-like PKI names. +// 2. dns domains: resolves using links in DNS TXT records // -// It can only publish to: (a) IPFS routing naming. -type mpns struct { +// It can only publish to: 1. IPFS routing naming. +type namesys struct { ds ds.Datastore dnsResolver, ipnsResolver resolver ipnsPublisher Publisher - staticMap map[string]path.Path - cache *lru.Cache[string, any] + staticMap map[string]*cacheEntry + cache *lru.Cache[string, cacheEntry] } -type Option func(*mpns) error +var _ NameSystem = &namesys{} + +type Option func(*namesys) error // WithCache is an option that instructs the name system to use a (LRU) cache of the given size. func WithCache(size int) Option { - return func(ns *mpns) error { + return func(ns *namesys) error { if size <= 0 { return fmt.Errorf("invalid cache size %d; must be > 0", size) } - cache, err := lru.New[string, any](size) + cache, err := lru.New[string, cacheEntry](size) if err != nil { return err } @@ -70,33 +73,34 @@ func WithCache(size int) Option { } } -// WithDNSResolver is an option that supplies a custom DNS resolver to use instead of the system -// default. +// WithDNSResolver is an option that supplies a custom DNS resolver to use instead +// of the system default. func WithDNSResolver(rslv madns.BasicResolver) Option { - return func(ns *mpns) error { + return func(ns *namesys) error { ns.dnsResolver = NewDNSResolver(rslv.LookupTXT) return nil } } -// WithDatastore is an option that supplies a datastore to use instead of an in-memory map datastore. The datastore is used to store published IPNS records and make them available for querying. +// WithDatastore is an option that supplies a datastore to use instead of an in-memory map datastore. +// The datastore is used to store published IPNS Records and make them available for querying. func WithDatastore(ds ds.Datastore) Option { - return func(ns *mpns) error { + return func(ns *namesys) error { ns.ds = ds return nil } } -// NewNameSystem will construct the IPFS naming system based on Routing +// NewNameSystem constructs an IPFS [NameSystem] based on the given [routing.ValueStore]. func NewNameSystem(r routing.ValueStore, opts ...Option) (NameSystem, error) { - var staticMap map[string]path.Path + var staticMap map[string]*cacheEntry // Prewarm namesys cache with static records for deterministic tests and debugging. // Useful for testing things like DNSLink without real DNS lookup. // Example: // IPFS_NS_MAP="dnslink-test.example.com:/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am" if list := os.Getenv("IPFS_NS_MAP"); list != "" { - staticMap = make(map[string]path.Path) + staticMap = make(map[string]*cacheEntry) for _, pair := range strings.Split(list, ",") { mapping := strings.SplitN(pair, ":", 2) key := mapping[0] @@ -104,11 +108,11 @@ func NewNameSystem(r routing.ValueStore, opts ...Option) (NameSystem, error) { if err != nil { return nil, err } - staticMap[key] = value + staticMap[ipns.NamespacePrefix+key] = &cacheEntry{val: value, ttl: 0} } } - ns := &mpns{ + ns := &namesys{ staticMap: staticMap, } @@ -127,149 +131,109 @@ func NewNameSystem(r routing.ValueStore, opts ...Option) (NameSystem, error) { ns.dnsResolver = NewDNSResolver(madns.DefaultResolver.LookupTXT) } - ns.ipnsResolver = NewIpnsResolver(r) - ns.ipnsPublisher = NewIpnsPublisher(r, ns.ds) + ns.ipnsResolver = NewIPNSResolver(r) + ns.ipnsPublisher = NewIPNSPublisher(r, ns.ds) return ns, nil } -// DefaultResolverCacheTTL defines max ttl of a record placed in namesys cache. -const DefaultResolverCacheTTL = time.Minute - // Resolve implements Resolver. -func (ns *mpns) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) { - ctx, span := StartSpan(ctx, "MPNS.Resolve", trace.WithAttributes(attribute.String("Name", name))) +func (ns *namesys) Resolve(ctx context.Context, p path.Path, options ...ResolveOption) (Result, error) { + ctx, span := startSpan(ctx, "namesys.Resolve", trace.WithAttributes(attribute.Stringer("Path", p))) defer span.End() - if strings.HasPrefix(name, "/ipfs/") { - return path.NewPath(name) - } - - if !strings.HasPrefix(name, "/") { - return path.NewPath("/ipfs/" + name) - } - - return resolve(ctx, ns, name, opts.ProcessOpts(options)) + return resolve(ctx, ns, p, ProcessResolveOptions(options)) } -func (ns *mpns) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result { - ctx, span := StartSpan(ctx, "MPNS.ResolveAsync", trace.WithAttributes(attribute.String("Name", name))) +func (ns *namesys) ResolveAsync(ctx context.Context, p path.Path, options ...ResolveOption) <-chan AsyncResult { + ctx, span := startSpan(ctx, "namesys.ResolveAsync", trace.WithAttributes(attribute.Stringer("Path", p))) defer span.End() - if strings.HasPrefix(name, "/ipfs/") { - p, err := path.NewPath(name) - res := make(chan Result, 1) - res <- Result{p, err} - close(res) - return res - } - - if !strings.HasPrefix(name, "/") { - p, err := path.NewPath("/ipfs/" + name) - res := make(chan Result, 1) - res <- Result{p, err} - close(res) - return res - } - - return resolveAsync(ctx, ns, name, opts.ProcessOpts(options)) + return resolveAsync(ctx, ns, p, ProcessResolveOptions(options)) } // resolveOnce implements resolver. -func (ns *mpns) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult { - ctx, span := StartSpan(ctx, "MPNS.ResolveOnceAsync") +func (ns *namesys) resolveOnceAsync(ctx context.Context, p path.Path, options ResolveOptions) <-chan AsyncResult { + ctx, span := startSpan(ctx, "namesys.ResolveOnceAsync", trace.WithAttributes(attribute.Stringer("Path", p))) defer span.End() - out := make(chan onceResult, 1) - - if !strings.HasPrefix(name, ipnsPrefix) { - name = ipnsPrefix + name - } - segments := strings.SplitN(name, "/", 4) - if len(segments) < 3 || segments[0] != "" { - log.Debugf("invalid name syntax for %s", name) - out <- onceResult{err: ErrResolveFailed} + out := make(chan AsyncResult, 1) + if !p.Mutable() { + out <- AsyncResult{Path: p} close(out) return out } - key := segments[2] - - // Resolver selection: - // 1. if it is a PeerID/CID/multihash resolve through "ipns". - // 2. if it is a domain name, resolve through "dns" - - var res resolver - ipnsKey, err := peer.Decode(key) - // CIDs in IPNS are expected to have libp2p-key multicodec - // We ease the transition by returning a more meaningful error with a valid CID + segments := p.Segments() + resolvablePath, err := path.NewPathFromSegments(segments[0], segments[1]) if err != nil { - ipnsCid, cidErr := cid.Decode(key) - if cidErr == nil && ipnsCid.Version() == 1 && ipnsCid.Type() != cid.Libp2pKey { - fixedCid := cid.NewCidV1(cid.Libp2pKey, ipnsCid.Hash()).String() - codecErr := fmt.Errorf("peer ID represented as CIDv1 require libp2p-key multicodec: retry with /ipns/%s", fixedCid) - log.Debugf("RoutingResolver: could not convert public key hash %q to peer ID: %s\n", key, codecErr) - out <- onceResult{err: codecErr} - close(out) - return out - } - } - - cacheKey := key - if err == nil { - cacheKey = string(ipnsKey) + out <- AsyncResult{Err: err} + close(out) + return out } - if p, ok := ns.cacheGet(cacheKey); ok { - var err error - if len(segments) > 3 { - p, err = path.Join(p, segments[3]) - } + if resolvedBase, ttl, lastMod, ok := ns.cacheGet(resolvablePath.String()); ok { + p, err = joinPaths(resolvedBase, p) span.SetAttributes(attribute.Bool("CacheHit", true)) span.RecordError(err) - - out <- onceResult{value: p, err: err} + out <- AsyncResult{Path: p, TTL: ttl, LastMod: lastMod, Err: err} close(out) return out + } else { + span.SetAttributes(attribute.Bool("CacheHit", false)) } - span.SetAttributes(attribute.Bool("CacheHit", false)) - if err == nil { + // Resolver selection: + // 1. If it is an IPNS Name, resolve through IPNS. + // 2. if it is a domain name, resolve through DNSLink. + + var res resolver + if _, err := ipns.NameFromString(segments[1]); err == nil { res = ns.ipnsResolver - } else if _, ok := dns.IsDomainName(key); ok { + } else if _, ok := dns.IsDomainName(segments[1]); ok { res = ns.dnsResolver } else { - out <- onceResult{err: fmt.Errorf("invalid IPNS root: %q", key)} + // CIDs in IPNS are expected to have libp2p-key multicodec + // We ease the transition by returning a more meaningful error with a valid CID + ipnsCid, cidErr := cid.Decode(segments[1]) + if cidErr == nil && ipnsCid.Version() == 1 && ipnsCid.Type() != cid.Libp2pKey { + fixedCid := cid.NewCidV1(cid.Libp2pKey, ipnsCid.Hash()).String() + codecErr := fmt.Errorf("peer ID represented as CIDv1 require libp2p-key multicodec: retry with /ipns/%s", fixedCid) + log.Debugf("RoutingResolver: could not convert public key hash %q to peer ID: %s\n", segments[1], codecErr) + out <- AsyncResult{Err: codecErr} + } else { + out <- AsyncResult{Err: fmt.Errorf("cannot resolve: %q", resolvablePath.String())} + } + close(out) return out } - resCh := res.resolveOnceAsync(ctx, key, options) - var best onceResult + resCh := res.resolveOnceAsync(ctx, resolvablePath, options) + var best AsyncResult go func() { defer close(out) for { select { case res, ok := <-resCh: if !ok { - if best != (onceResult{}) { - ns.cacheSet(cacheKey, best.value, best.ttl) + if best != (AsyncResult{}) { + ns.cacheSet(resolvablePath.String(), best.Path, best.TTL, best.LastMod) } return } - if res.err == nil { + + if res.Err == nil { best = res } - p := res.value - err := res.err - ttl := res.ttl - // Attach rest of the path - if len(segments) > 3 { - p, err = path.Join(p, segments[3]) + p, err := joinPaths(res.Path, p) + if err != nil { + // res.Err may already be defined, so just combine them + res.Err = multierr.Combine(err, res.Err) } - emitOnceResult(ctx, out, onceResult{value: p, ttl: ttl, err: err}) + emitOnceResult(ctx, out, AsyncResult{Path: p, TTL: res.TTL, LastMod: res.LastMod, Err: res.Err}) case <-ctx.Done(): return } @@ -279,7 +243,7 @@ func (ns *mpns) resolveOnceAsync(ctx context.Context, name string, options opts. return out } -func emitOnceResult(ctx context.Context, outCh chan<- onceResult, r onceResult) { +func emitOnceResult(ctx context.Context, outCh chan<- AsyncResult, r AsyncResult) { select { case outCh <- r: case <-ctx.Done(): @@ -287,30 +251,35 @@ func emitOnceResult(ctx context.Context, outCh chan<- onceResult, r onceResult) } // Publish implements Publisher -func (ns *mpns) Publish(ctx context.Context, name ci.PrivKey, value path.Path, options ...opts.PublishOption) error { - ctx, span := StartSpan(ctx, "MPNS.Publish") +func (ns *namesys) Publish(ctx context.Context, name ci.PrivKey, value path.Path, options ...PublishOption) error { + ctx, span := startSpan(ctx, "namesys.Publish") defer span.End() // This is a bit hacky. We do this because the EOL is based on the current // time, but also needed in the end of the function. Therefore, we parse // the options immediately and add an option PublishWithEOL with the EOL // calculated in this moment. - publishOpts := opts.ProcessPublishOptions(options) - options = append(options, opts.PublishWithEOL(publishOpts.EOL)) + publishOpts := ProcessPublishOptions(options) + options = append(options, PublishWithEOL(publishOpts.EOL)) - id, err := peer.IDFromPrivateKey(name) + pid, err := peer.IDFromPrivateKey(name) if err != nil { span.RecordError(err) return err } - span.SetAttributes(attribute.String("ID", id.String())) + + ipnsName := ipns.NameFromPeer(pid) + cacheKey := ipnsName.String() + + span.SetAttributes(attribute.String("ID", pid.String())) if err := ns.ipnsPublisher.Publish(ctx, name, value, options...); err != nil { // Invalidate the cache. Publishing may _partially_ succeed but // still return an error. - ns.cacheInvalidate(string(id)) + ns.cacheInvalidate(cacheKey) span.RecordError(err) return err } + ttl := DefaultResolverCacheTTL if publishOpts.TTL >= 0 { ttl = publishOpts.TTL @@ -318,6 +287,20 @@ func (ns *mpns) Publish(ctx context.Context, name ci.PrivKey, value path.Path, o if ttEOL := time.Until(publishOpts.EOL); ttEOL < ttl { ttl = ttEOL } - ns.cacheSet(string(id), value, ttl) + ns.cacheSet(cacheKey, value, ttl, time.Now()) return nil } + +// Resolve is an utility function that takes a [NameSystem] and a [path.Path], and +// returns the result of [NameSystem.Resolve] for the given path. If the given namesys +// is nil, [ErrNoNamesys] is returned. +func Resolve(ctx context.Context, ns NameSystem, p path.Path) (Result, error) { + ctx, span := startSpan(ctx, "Resolve", trace.WithAttributes(attribute.Stringer("Path", p))) + defer span.End() + + if ns == nil { + return Result{}, ErrNoNamesys + } + + return ns.Resolve(ctx, p) +} diff --git a/namesys/namesys_cache.go b/namesys/namesys_cache.go new file mode 100644 index 000000000..fc8842e3b --- /dev/null +++ b/namesys/namesys_cache.go @@ -0,0 +1,78 @@ +package namesys + +import ( + "time" + + "github.com/ipfs/boxo/path" +) + +type cacheEntry struct { + val path.Path // is the value of this entry + ttl time.Duration // is the ttl of this entry + lastMod time.Time // is the last time this entry was modified + cacheEOL time.Time // is until when we keep this entry in cache +} + +func (ns *namesys) cacheGet(name string) (path.Path, time.Duration, time.Time, bool) { + // existence of optional mapping defined via IPFS_NS_MAP is checked first + if ns.staticMap != nil { + entry, ok := ns.staticMap[name] + if ok { + return entry.val, entry.ttl, entry.lastMod, true + } + } + + if ns.cache == nil { + return nil, 0, time.Now(), false + } + + entry, ok := ns.cache.Get(name) + if !ok { + return nil, 0, time.Now(), false + } + + if time.Now().Before(entry.cacheEOL) { + return entry.val, entry.ttl, entry.lastMod, true + } + + // We do not delete the entry from the cache. Removals are handled by the + // backing cache system. It is useful to keep it since cacheSet can use + // previously existing values to heuristically update a cache entry. + return nil, 0, time.Now(), false +} + +func (ns *namesys) cacheSet(name string, val path.Path, ttl time.Duration, lastMod time.Time) { + if ns.cache == nil || ttl <= 0 { + return + } + + // Set the current date if there's no lastMod. + if lastMod.IsZero() { + lastMod = time.Now() + } + + // If there's an already cached version with the same path, but + // different lastMod date, keep the oldest. + entry, ok := ns.cache.Get(name) + if ok && entry.val.String() == val.String() { + if lastMod.After(entry.lastMod) { + lastMod = entry.lastMod + } + } + + // Add automatically evicts previous entry, so it works for updating. + ns.cache.Add(name, cacheEntry{ + val: val, + ttl: ttl, + lastMod: lastMod, + cacheEOL: time.Now().Add(ttl), + }) +} + +func (ns *namesys) cacheInvalidate(name string) { + if ns.cache == nil { + return + } + + ns.cache.Remove(name) +} diff --git a/namesys/namesys_test.go b/namesys/namesys_test.go index 9e1d9f5f6..41fa0ce88 100644 --- a/namesys/namesys_test.go +++ b/namesys/namesys_test.go @@ -2,12 +2,9 @@ package namesys import ( "context" - "errors" - "fmt" "testing" "time" - opts "github.com/ipfs/boxo/coreiface/options/namesys" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" offroute "github.com/ipfs/boxo/routing/offline" @@ -16,33 +13,33 @@ import ( record "github.com/libp2p/go-libp2p-record" ci "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type mockResolver struct { entries map[string]string } -func testResolution(t *testing.T, resolver Resolver, name string, depth uint, expected string, expError error) { +func testResolution(t *testing.T, resolver Resolver, name string, depth uint, expected string, expectedTTL time.Duration, expectedError error) { t.Helper() - p, err := resolver.Resolve(context.Background(), name, opts.Depth(depth)) - if !errors.Is(err, expError) { - t.Fatal(fmt.Errorf( - "expected %s with a depth of %d to have a '%s' error, but got '%s'", - name, depth, expError, err)) - } + + ptr, err := path.NewPath(name) + require.NoError(t, err) + + res, err := resolver.Resolve(context.Background(), ptr, ResolveWithDepth(depth)) + require.ErrorIs(t, err, expectedError) + require.Equal(t, expectedTTL, res.TTL) if expected == "" { - assert.Nil(t, p, "%s with depth %d", name, depth) + require.Nil(t, res.Path, "%s with depth %d", name, depth) } else { - assert.Equal(t, p.String(), expected, "%s with depth %d", name, depth) + require.Equal(t, expected, res.Path.String(), "%s with depth %d", name, depth) } } -func (r *mockResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult { - p, err := path.NewPath(r.entries[name]) - out := make(chan onceResult, 1) - out <- onceResult{value: p, err: err} +func (r *mockResolver) resolveOnceAsync(ctx context.Context, p path.Path, options ResolveOptions) <-chan AsyncResult { + p, err := path.NewPath(r.entries[p.String()]) + out := make(chan AsyncResult, 1) + out <- AsyncResult{Path: p, Err: err} close(out) return out } @@ -50,12 +47,12 @@ func (r *mockResolver) resolveOnceAsync(ctx context.Context, name string, option func mockResolverOne() *mockResolver { return &mockResolver{ entries: map[string]string{ - "QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy": "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", - "QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n": "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", - "QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD": "/ipns/ipfs.io", - "QmQ4QZh8nrsczdUEwTyfBope4THUhqxqc1fx6qYhhzZQei": "/ipfs/QmP3ouCnU8NNLsW6261pAx2pNLV2E4dQoisB1sgda12Act", - "12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // ed25519+identity multihash - "bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // cidv1 in base32 with libp2p-key multicodec + "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy": "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", + "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n": "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", + "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD": "/ipns/ipfs.io", + "/ipns/QmQ4QZh8nrsczdUEwTyfBope4THUhqxqc1fx6qYhhzZQei": "/ipfs/QmP3ouCnU8NNLsW6261pAx2pNLV2E4dQoisB1sgda12Act", + "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // ed25519+identity multihash + "/ipns/bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", // cidv1 in base32 with libp2p-key multicodec }, } } @@ -63,123 +60,106 @@ func mockResolverOne() *mockResolver { func mockResolverTwo() *mockResolver { return &mockResolver{ entries: map[string]string{ - "ipfs.io": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", + "/ipns/ipfs.io": "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", }, } } func TestNamesysResolution(t *testing.T) { - r := &mpns{ + r := &namesys{ ipnsResolver: mockResolverOne(), dnsResolver: mockResolverTwo(), } - testResolution(t, r, "Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 1, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion) - testResolution(t, r, "/ipns/ipfs.io", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/ipfs.io", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) - testResolution(t, r, "/ipns/ipfs.io", 2, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion) - testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", opts.DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", nil) - testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 1, "/ipns/ipfs.io", ErrResolveRecursion) - testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 2, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) - testResolution(t, r, "/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 3, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", ErrResolveRecursion) - testResolution(t, r, "/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) - testResolution(t, r, "/ipns/bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", ErrResolveRecursion) + for _, testCase := range []struct { + name string + depth uint + expectedPath string + expectedTTL time.Duration + expectedError error + }{ + {"/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil}, + {"/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil}, + {"/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil}, + {"/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 1, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", 0, ErrResolveRecursion}, + {"/ipns/ipfs.io", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil}, + {"/ipns/ipfs.io", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 0, ErrResolveRecursion}, + {"/ipns/ipfs.io", 2, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", 0, ErrResolveRecursion}, + {"/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", DefaultDepthLimit, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj", 0, nil}, + {"/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 1, "/ipns/ipfs.io", 0, ErrResolveRecursion}, + {"/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 2, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 0, ErrResolveRecursion}, + {"/ipns/QmY3hE8xgFCjGcz6PHgnvJz5HZi1BaKRfPkn1ghZUcYMjD", 3, "/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy", 0, ErrResolveRecursion}, + {"/ipns/12D3KooWFB51PRY9BxcXSH6khFXw1BZeszeLDy7C8GciskqCTZn5", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 0, ErrResolveRecursion}, + {"/ipns/bafzbeickencdqw37dpz3ha36ewrh4undfjt2do52chtcky4rxkj447qhdm", 1, "/ipns/QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n", 0, ErrResolveRecursion}, + } { + t.Run(testCase.name, func(t *testing.T) { + testResolution(t, r, testCase.name, (testCase.depth), testCase.expectedPath, 0, testCase.expectedError) + }) + } +} + +func TestResolveIPNS(t *testing.T) { + ns := &namesys{ + ipnsResolver: mockResolverOne(), + dnsResolver: mockResolverTwo(), + } + + inputPath, err := path.NewPath("/ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy/a/b/c") + require.NoError(t, err) + + res, err := Resolve(context.Background(), ns, inputPath) + require.NoError(t, err) + require.Equal(t, "/ipfs/Qmcqtw8FfrVSBaRmbWwHxt3AuySBhJLcvmFYi3Lbc4xnwj/a/b/c", res.Path.String()) } func TestPublishWithCache0(t *testing.T) { dst := dssync.MutexWrap(ds.NewMapDatastore()) - priv, _, err := ci.GenerateKeyPair(ci.RSA, 2048) - if err != nil { - t.Fatal(err) - } - ps, err := pstoremem.NewPeerstore() - if err != nil { - t.Fatal(err) - } - pid, err := peer.IDFromPrivateKey(priv) - if err != nil { - t.Fatal(err) - } - err = ps.AddPrivKey(pid, priv) - if err != nil { - t.Fatal(err) - } + priv, _, err := ci.GenerateKeyPair(ci.RSA, 4096) + require.NoError(t, err) routing := offroute.NewOfflineRouter(dst, record.NamespacedValidator{ - "ipns": ipns.Validator{KeyBook: ps}, + "ipns": ipns.Validator{}, // No need for KeyBook, as records created by NameSys include PublicKey for RSA. "pk": record.PublicKeyValidator{}, }) nsys, err := NewNameSystem(routing, WithDatastore(dst)) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) // CID is arbitrary. p, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + err = nsys.Publish(context.Background(), priv, p) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) } func TestPublishWithTTL(t *testing.T) { dst := dssync.MutexWrap(ds.NewMapDatastore()) priv, _, err := ci.GenerateKeyPair(ci.RSA, 2048) - if err != nil { - t.Fatal(err) - } - ps, err := pstoremem.NewPeerstore() - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) + pid, err := peer.IDFromPrivateKey(priv) - if err != nil { - t.Fatal(err) - } - err = ps.AddPrivKey(pid, priv) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) routing := offroute.NewOfflineRouter(dst, record.NamespacedValidator{ - "ipns": ipns.Validator{KeyBook: ps}, + "ipns": ipns.Validator{}, // No need for KeyBook, as records created by NameSys include PublicKey for RSA. "pk": record.PublicKeyValidator{}, }) - nsys, err := NewNameSystem(routing, WithDatastore(dst), WithCache(128)) - if err != nil { - t.Fatal(err) - } + ns, err := NewNameSystem(routing, WithDatastore(dst), WithCache(128)) + require.NoError(t, err) // CID is arbitrary. p, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) ttl := 1 * time.Second eol := time.Now().Add(2 * time.Second) - err = nsys.Publish(context.Background(), priv, p, opts.PublishWithEOL(eol), opts.PublishWithTTL(ttl)) - if err != nil { - t.Fatal(err) - } - ientry, ok := nsys.(*mpns).cache.Get(string(pid)) - if !ok { - t.Fatal("cache get failed") - } - entry, ok := ientry.(cacheEntry) - if !ok { - t.Fatal("bad cache item returned") - } - if entry.eol.Sub(eol) > 10*time.Millisecond { - t.Fatalf("bad cache ttl: expected %s, got %s", eol, entry.eol) - } + err = ns.Publish(context.Background(), priv, p, PublishWithEOL(eol), PublishWithTTL(ttl)) + require.NoError(t, err) + + entry, ok := ns.(*namesys).cache.Get(ipns.NameFromPeer(pid).String()) + require.True(t, ok) + require.LessOrEqual(t, entry.cacheEOL.Sub(eol), 10*time.Millisecond) } diff --git a/namesys/publisher.go b/namesys/publisher.go deleted file mode 100644 index 9cb3ae66a..000000000 --- a/namesys/publisher.go +++ /dev/null @@ -1,287 +0,0 @@ -package namesys - -import ( - "context" - "strings" - "sync" - "time" - - opts "github.com/ipfs/boxo/coreiface/options/namesys" - "github.com/ipfs/boxo/ipns" - "github.com/ipfs/boxo/path" - ds "github.com/ipfs/go-datastore" - dsquery "github.com/ipfs/go-datastore/query" - "github.com/libp2p/go-libp2p/core/crypto" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/routing" - "github.com/whyrusleeping/base32" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -const ipnsPrefix = "/ipns/" - -// IpnsPublisher is capable of publishing and resolving names to the IPFS -// routing system. -type IpnsPublisher struct { - routing routing.ValueStore - ds ds.Datastore - - // Used to ensure we assign IPNS records *sequential* sequence numbers. - mu sync.Mutex -} - -// NewIpnsPublisher constructs a publisher for the IPFS Routing name system. -func NewIpnsPublisher(route routing.ValueStore, ds ds.Datastore) *IpnsPublisher { - if ds == nil { - panic("nil datastore") - } - return &IpnsPublisher{routing: route, ds: ds} -} - -// Publish implements Publisher. Accepts a keypair and a value, -// and publishes it out to the routing system -func (p *IpnsPublisher) Publish(ctx context.Context, k crypto.PrivKey, value path.Path, options ...opts.PublishOption) error { - log.Debugf("Publish %s", value) - - ctx, span := StartSpan(ctx, "IpnsPublisher.Publish", trace.WithAttributes(attribute.String("Value", value.String()))) - defer span.End() - - record, err := p.updateRecord(ctx, k, value, options...) - if err != nil { - return err - } - - return PutRecordToRouting(ctx, p.routing, k.GetPublic(), record) -} - -// IpnsDsKey returns a datastore key given an IPNS identifier (peer -// ID). Defines the storage key for IPNS records in the local datastore. -func IpnsDsKey(id peer.ID) ds.Key { - return ds.NewKey("/ipns/" + base32.RawStdEncoding.EncodeToString([]byte(id))) -} - -// ListPublished returns the latest IPNS records published by this node and -// their expiration times. -// -// This method will not search the routing system for records published by other -// nodes. -func (p *IpnsPublisher) ListPublished(ctx context.Context) (map[peer.ID]*ipns.Record, error) { - query, err := p.ds.Query(ctx, dsquery.Query{ - Prefix: ipnsPrefix, - }) - if err != nil { - return nil, err - } - defer query.Close() - - records := make(map[peer.ID]*ipns.Record) - for { - select { - case result, ok := <-query.Next(): - if !ok { - return records, nil - } - if result.Error != nil { - return nil, result.Error - } - rec, err := ipns.UnmarshalRecord(result.Value) - if err != nil { - // Might as well return what we can. - log.Error("found an invalid IPNS entry:", err) - continue - } - if !strings.HasPrefix(result.Key, ipnsPrefix) { - log.Errorf("datastore query for keys with prefix %s returned a key: %s", ipnsPrefix, result.Key) - continue - } - k := result.Key[len(ipnsPrefix):] - pid, err := base32.RawStdEncoding.DecodeString(k) - if err != nil { - log.Errorf("ipns ds key invalid: %s", result.Key) - continue - } - records[peer.ID(pid)] = rec - case <-ctx.Done(): - return nil, ctx.Err() - } - } -} - -// GetPublished returns the record this node has published corresponding to the -// given peer ID. -// -// If `checkRouting` is true and we have no existing record, this method will -// check the routing system for any existing records. -func (p *IpnsPublisher) GetPublished(ctx context.Context, id peer.ID, checkRouting bool) (*ipns.Record, error) { - ctx, cancel := context.WithTimeout(ctx, time.Second*30) - defer cancel() - - value, err := p.ds.Get(ctx, IpnsDsKey(id)) - switch err { - case nil: - case ds.ErrNotFound: - if !checkRouting { - return nil, nil - } - ipnskey := string(ipns.NameFromPeer(id).RoutingKey()) - value, err = p.routing.GetValue(ctx, ipnskey) - if err != nil { - // Not found or other network issue. Can't really do - // anything about this case. - if err != routing.ErrNotFound { - log.Debugf("error when determining the last published IPNS record for %s: %s", id, err) - } - - return nil, nil - } - default: - return nil, err - } - - return ipns.UnmarshalRecord(value) -} - -func (p *IpnsPublisher) updateRecord(ctx context.Context, k crypto.PrivKey, value path.Path, options ...opts.PublishOption) (*ipns.Record, error) { - id, err := peer.IDFromPrivateKey(k) - if err != nil { - return nil, err - } - - p.mu.Lock() - defer p.mu.Unlock() - - // get previous records sequence number - rec, err := p.GetPublished(ctx, id, true) - if err != nil { - return nil, err - } - - seqno := uint64(0) - if rec != nil { - seqno, err = rec.Sequence() - if err != nil { - return nil, err - } - - p, err := rec.Value() - if err != nil { - return nil, err - } - if value.String() != p.String() { - // Don't bother incrementing the sequence number unless the - // value changes. - seqno++ - } - } - - opts := opts.ProcessPublishOptions(options) - - // Create record - r, err := ipns.NewRecord(k, value, seqno, opts.EOL, opts.TTL, ipns.WithV1Compatibility(opts.CompatibleWithV1)) - if err != nil { - return nil, err - } - - data, err := ipns.MarshalRecord(r) - if err != nil { - return nil, err - } - - // Put the new record. - key := IpnsDsKey(id) - if err := p.ds.Put(ctx, key, data); err != nil { - return nil, err - } - if err := p.ds.Sync(ctx, key); err != nil { - return nil, err - } - return r, nil -} - -// PutRecordToRouting publishes the given entry using the provided ValueStore, -// keyed on the ID associated with the provided public key. The public key is -// also made available to the routing system so that entries can be verified. -func PutRecordToRouting(ctx context.Context, r routing.ValueStore, k crypto.PubKey, rec *ipns.Record) error { - ctx, span := StartSpan(ctx, "PutRecordToRouting") - defer span.End() - - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - errs := make(chan error, 2) // At most two errors (IPNS, and public key) - - id, err := peer.IDFromPublicKey(k) - if err != nil { - return err - } - - go func() { - errs <- PublishEntry(ctx, r, string(ipns.NameFromPeer(id).RoutingKey()), rec) - }() - - // Publish the public key if a public key cannot be extracted from the ID - // TODO: once v0.4.16 is widespread enough, we can stop doing this - // and at that point we can even deprecate the /pk/ namespace in the dht - // - // NOTE: This check actually checks if the public key has been embedded - // in the IPNS entry. This check is sufficient because we embed the - // public key in the IPNS entry if it can't be extracted from the ID. - if _, err := rec.PubKey(); err == nil { - go func() { - errs <- PublishPublicKey(ctx, r, PkKeyForID(id), k) - }() - - if err := waitOnErrChan(ctx, errs); err != nil { - return err - } - } - - return waitOnErrChan(ctx, errs) -} - -func waitOnErrChan(ctx context.Context, errs chan error) error { - select { - case err := <-errs: - return err - case <-ctx.Done(): - return ctx.Err() - } -} - -// PublishPublicKey stores the given public key in the ValueStore with the -// given key. -func PublishPublicKey(ctx context.Context, r routing.ValueStore, k string, pubk crypto.PubKey) error { - ctx, span := StartSpan(ctx, "PublishPublicKey", trace.WithAttributes(attribute.String("Key", k))) - defer span.End() - - log.Debugf("Storing pubkey at: %s", k) - pkbytes, err := crypto.MarshalPublicKey(pubk) - if err != nil { - return err - } - - // Store associated public key - return r.PutValue(ctx, k, pkbytes) -} - -// PublishEntry stores the given IpnsEntry in the ValueStore with the given -// ipnskey. -func PublishEntry(ctx context.Context, r routing.ValueStore, ipnskey string, rec *ipns.Record) error { - ctx, span := StartSpan(ctx, "PublishEntry", trace.WithAttributes(attribute.String("IPNSKey", ipnskey))) - defer span.End() - - data, err := ipns.MarshalRecord(rec) - if err != nil { - return err - } - - log.Debugf("Storing ipns entry at: %x", ipnskey) - // Store ipns entry at "/ipns/"+h(pubkey) - return r.PutValue(ctx, ipnskey, data) -} - -// PkKeyForID returns the public key routing key for the given peer ID. -func PkKeyForID(id peer.ID) string { - return "/pk/" + string(id) -} diff --git a/namesys/publisher_test.go b/namesys/publisher_test.go deleted file mode 100644 index 536e72771..000000000 --- a/namesys/publisher_test.go +++ /dev/null @@ -1,159 +0,0 @@ -package namesys - -import ( - "context" - "crypto/rand" - "testing" - "time" - - "github.com/ipfs/boxo/path" - - dshelp "github.com/ipfs/boxo/datastore/dshelp" - "github.com/ipfs/boxo/ipns" - mockrouting "github.com/ipfs/boxo/routing/mock" - ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" - testutil "github.com/libp2p/go-libp2p-testing/net" - ci "github.com/libp2p/go-libp2p/core/crypto" - "github.com/libp2p/go-libp2p/core/peer" - ma "github.com/multiformats/go-multiaddr" -) - -type identity struct { - testutil.PeerNetParams -} - -func (p *identity) ID() peer.ID { - return p.PeerNetParams.ID -} - -func (p *identity) Address() ma.Multiaddr { - return p.Addr -} - -func (p *identity) PrivateKey() ci.PrivKey { - return p.PrivKey -} - -func (p *identity) PublicKey() ci.PubKey { - return p.PubKey -} - -func testNamekeyPublisher(t *testing.T, keyType int, expectedErr error, expectedExistence bool) { - // Context - ctx := context.Background() - - // Private key - privKey, pubKey, err := ci.GenerateKeyPairWithReader(keyType, 2048, rand.Reader) - if err != nil { - t.Fatal(err) - } - - // ID - id, err := peer.IDFromPublicKey(pubKey) - if err != nil { - t.Fatal(err) - } - - // Value - value, err := path.NewPath("/ipfs/bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4") - if err != nil { - t.Fatal(err) - } - - // Seqnum - seqnum := uint64(0) - - // Eol - eol := time.Now().Add(24 * time.Hour) - - // Routing value store - p := testutil.PeerNetParams{ - ID: id, - PrivKey: privKey, - PubKey: pubKey, - Addr: testutil.ZeroLocalTCPAddress, - } - - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - serv := mockrouting.NewServer() - r := serv.ClientWithDatastore(context.Background(), &identity{p}, dstore) - - rec, err := ipns.NewRecord(privKey, value, seqnum, eol, 0) - if err != nil { - t.Fatal(err) - } - - err = PutRecordToRouting(ctx, r, pubKey, rec) - if err != nil { - t.Fatal(err) - } - - // Check for namekey existence in value store - namekey := PkKeyForID(id) - _, err = r.GetValue(ctx, namekey) - if err != expectedErr { - t.Fatal(err) - } - - // Also check datastore for completeness - key := dshelp.NewKeyFromBinary([]byte(namekey)) - exists, err := dstore.Has(ctx, key) - if err != nil { - t.Fatal(err) - } - - if exists != expectedExistence { - t.Fatal("Unexpected key existence in datastore") - } -} - -func TestRSAPublisher(t *testing.T) { - testNamekeyPublisher(t, ci.RSA, nil, true) -} - -func TestEd22519Publisher(t *testing.T) { - testNamekeyPublisher(t, ci.Ed25519, ds.ErrNotFound, false) -} - -func TestAsyncDS(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - rt := mockrouting.NewServer().Client(testutil.RandIdentityOrFatal(t)) - ds := &checkSyncDS{ - Datastore: ds.NewMapDatastore(), - syncKeys: make(map[ds.Key]struct{}), - } - publisher := NewIpnsPublisher(rt, ds) - - ipnsFakeID := testutil.RandIdentityOrFatal(t) - ipnsVal, err := path.NewPath("/ipns/foo.bar") - if err != nil { - t.Fatal(err) - } - - if err := publisher.Publish(ctx, ipnsFakeID.PrivateKey(), ipnsVal); err != nil { - t.Fatal(err) - } - - ipnsKey := IpnsDsKey(ipnsFakeID.ID()) - - for k := range ds.syncKeys { - if k.IsAncestorOf(ipnsKey) || k.Equal(ipnsKey) { - return - } - } - - t.Fatal("ipns key not synced") -} - -type checkSyncDS struct { - ds.Datastore - syncKeys map[ds.Key]struct{} -} - -func (d *checkSyncDS) Sync(ctx context.Context, prefix ds.Key) error { - d.syncKeys[prefix] = struct{}{} - return d.Datastore.Sync(ctx, prefix) -} diff --git a/namesys/republisher/repub.go b/namesys/republisher/repub.go index bb7a5e2b0..347f4cf5a 100644 --- a/namesys/republisher/repub.go +++ b/namesys/republisher/repub.go @@ -5,13 +5,15 @@ package republisher import ( "context" "errors" + "fmt" "time" - keystore "github.com/ipfs/boxo/keystore" + "github.com/ipfs/boxo/keystore" "github.com/ipfs/boxo/namesys" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" - opts "github.com/ipfs/boxo/coreiface/options/namesys" "github.com/ipfs/boxo/ipns" ds "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" @@ -21,24 +23,27 @@ import ( "github.com/libp2p/go-libp2p/core/peer" ) -var errNoEntry = errors.New("no previous entry") - -var log = logging.Logger("ipns-repub") +var ( + errNoEntry = errors.New("no previous entry") + log = logging.Logger("ipns/repub") +) -// DefaultRebroadcastInterval is the default interval at which we rebroadcast IPNS records -var DefaultRebroadcastInterval = time.Hour * 4 +const ( + // DefaultRebroadcastInterval is the default interval at which we rebroadcast IPNS records + DefaultRebroadcastInterval = time.Hour * 4 -// InitialRebroadcastDelay is the delay before first broadcasting IPNS records on start -var InitialRebroadcastDelay = time.Minute * 1 + // InitialRebroadcastDelay is the delay before first broadcasting IPNS records on start + InitialRebroadcastDelay = time.Minute * 1 -// FailureRetryInterval is the interval at which we retry IPNS records broadcasts (when they fail) -var FailureRetryInterval = time.Minute * 5 + // FailureRetryInterval is the interval at which we retry IPNS records broadcasts (when they fail) + FailureRetryInterval = time.Minute * 5 -// DefaultRecordLifetime is the default lifetime for IPNS records -const DefaultRecordLifetime = time.Hour * 24 + // DefaultRecordLifetime is the default lifetime for IPNS records + DefaultRecordLifetime = ipns.DefaultRecordLifetime +) // Republisher facilitates the regular publishing of all the IPNS records -// associated to keys in a Keystore. +// associated to keys in a [keystore.Keystore]. type Republisher struct { ns namesys.Publisher ds ds.Datastore @@ -51,7 +56,7 @@ type Republisher struct { RecordLifetime time.Duration } -// NewRepublisher creates a new Republisher +// NewRepublisher creates a new [Republisher] from the given options. func NewRepublisher(ns namesys.Publisher, ds ds.Datastore, self ic.PrivKey, ks keystore.Keystore) *Republisher { return &Republisher{ ns: ns, @@ -63,8 +68,7 @@ func NewRepublisher(ns namesys.Publisher, ds ds.Datastore, self ic.PrivKey, ks k } } -// Run starts the republisher facility. It can be stopped by stopping the -// provided proc. +// Run starts the republisher facility. It can be stopped by stopping the provided proc. func (rp *Republisher) Run(proc goprocess.Process) { timer := time.NewTimer(InitialRebroadcastDelay) defer timer.Stop() @@ -92,7 +96,7 @@ func (rp *Republisher) Run(proc goprocess.Process) { func (rp *Republisher) republishEntries(p goprocess.Process) error { ctx, cancel := context.WithCancel(gpctx.OnClosingContext(p)) defer cancel() - ctx, span := namesys.StartSpan(ctx, "Republisher.RepublishEntries") + ctx, span := startSpan(ctx, "Republisher.RepublishEntries") defer span.End() // TODO: Use rp.ipns.ListPublished(). We can't currently *do* that @@ -126,7 +130,7 @@ func (rp *Republisher) republishEntries(p goprocess.Process) error { } func (rp *Republisher) republishEntry(ctx context.Context, priv ic.PrivKey) error { - ctx, span := namesys.StartSpan(ctx, "Republisher.RepublishEntry") + ctx, span := startSpan(ctx, "Republisher.RepublishEntry") defer span.End() id, err := peer.IDFromPrivateKey(priv) if err != nil { @@ -137,7 +141,7 @@ func (rp *Republisher) republishEntry(ctx context.Context, priv ic.PrivKey) erro log.Debugf("republishing ipns entry for %s", id) // Look for it locally only - rec, err := rp.getLastIPNSRecord(ctx, id) + rec, err := rp.getLastIPNSRecord(ctx, ipns.NameFromPeer(id)) if err != nil { if err == errNoEntry { span.SetAttributes(attribute.Bool("NoEntry", true)) @@ -164,14 +168,14 @@ func (rp *Republisher) republishEntry(ctx context.Context, priv ic.PrivKey) erro if prevEol.After(eol) { eol = prevEol } - err = rp.ns.Publish(ctx, priv, p, opts.PublishWithEOL(eol)) + err = rp.ns.Publish(ctx, priv, p, namesys.PublishWithEOL(eol)) span.RecordError(err) return err } -func (rp *Republisher) getLastIPNSRecord(ctx context.Context, id peer.ID) (*ipns.Record, error) { +func (rp *Republisher) getLastIPNSRecord(ctx context.Context, name ipns.Name) (*ipns.Record, error) { // Look for it locally only - val, err := rp.ds.Get(ctx, namesys.IpnsDsKey(id)) + val, err := rp.ds.Get(ctx, namesys.IpnsDsKey(name)) switch err { case nil: case ds.ErrNotFound: @@ -182,3 +186,9 @@ func (rp *Republisher) getLastIPNSRecord(ctx context.Context, id peer.ID) (*ipns return ipns.UnmarshalRecord(val) } + +var tracer = otel.Tracer("boxo/namesys/republisher") + +func startSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + return tracer.Start(ctx, fmt.Sprintf("Namesys.%s", name)) +} diff --git a/namesys/republisher/repub_test.go b/namesys/republisher/repub_test.go index 6b5d2abf0..74297a41c 100644 --- a/namesys/republisher/repub_test.go +++ b/namesys/republisher/repub_test.go @@ -13,8 +13,8 @@ import ( host "github.com/libp2p/go-libp2p/core/host" peer "github.com/libp2p/go-libp2p/core/peer" routing "github.com/libp2p/go-libp2p/core/routing" + "github.com/stretchr/testify/require" - opts "github.com/ipfs/boxo/coreiface/options/namesys" "github.com/ipfs/boxo/ipns" "github.com/ipfs/boxo/path" ds "github.com/ipfs/go-datastore" @@ -27,7 +27,7 @@ import ( type mockNode struct { h host.Host - id string + id peer.ID privKey ic.PrivKey store ds.Batching dht *dht.IpfsDHT @@ -47,13 +47,11 @@ func getMockNode(t *testing.T, ctx context.Context) *mockNode { return rt, err }), ) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) return &mockNode{ h: h, - id: h.ID().Pretty(), + id: h.ID(), privKey: h.Peerstore().PrivKey(h.ID()), store: dstore, dht: idht, @@ -72,9 +70,7 @@ func TestRepublish(t *testing.T) { for i := 0; i < 10; i++ { n := getMockNode(t, ctx) ns, err := namesys.NewNameSystem(n.dht, namesys.WithDatastore(n.store)) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) nsystems = append(nsystems, ns) nodes = append(nodes, n) @@ -83,21 +79,18 @@ func TestRepublish(t *testing.T) { pinfo := host.InfoFromHost(nodes[0].h) for _, n := range nodes[1:] { - if err := n.h.Connect(ctx, *pinfo); err != nil { - t.Fatal(err) - } + err := n.h.Connect(ctx, *pinfo) + require.NoError(t, err) } // have one node publish a record that is valid for 1 second publisher := nodes[3] p, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) - rp := namesys.NewIpnsPublisher(publisher.dht, publisher.store) - name := "/ipns/" + publisher.id + rp := namesys.NewIPNSPublisher(publisher.dht, publisher.store) + name := ipns.NameFromPeer(publisher.id).AsPath() // Retry in case the record expires before we can fetch it. This can // happen when running the test on a slow machine. @@ -105,10 +98,8 @@ func TestRepublish(t *testing.T) { timeout := time.Second for { expiration = time.Now().Add(time.Second) - err := rp.Publish(ctx, publisher.privKey, p, opts.PublishWithEOL(expiration)) - if err != nil { - t.Fatal(err) - } + err := rp.Publish(ctx, publisher.privKey, p, namesys.PublishWithEOL(expiration)) + require.NoError(t, err) err = verifyResolution(nsystems, name, p) if err == nil { @@ -124,9 +115,8 @@ func TestRepublish(t *testing.T) { // Now wait a second, the records will be invalid and we should fail to resolve time.Sleep(timeout) - if err := verifyResolutionFails(nsystems, name); err != nil { - t.Fatal(err) - } + err = verifyResolutionFails(nsystems, name) + require.NoError(t, err) // The republishers that are contained within the nodes have their timeout set // to 12 hours. Instead of trying to tweak those, we're just going to pretend @@ -142,9 +132,8 @@ func TestRepublish(t *testing.T) { time.Sleep(time.Second * 2) // we should be able to resolve them now - if err := verifyResolution(nsystems, name, p); err != nil { - t.Fatal(err) - } + err = verifyResolution(nsystems, name, p) + require.NoError(t, err) } func TestLongEOLRepublish(t *testing.T) { @@ -158,9 +147,7 @@ func TestLongEOLRepublish(t *testing.T) { for i := 0; i < 10; i++ { n := getMockNode(t, ctx) ns, err := namesys.NewNameSystem(n.dht, namesys.WithDatastore(n.store)) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) nsystems = append(nsystems, ns) nodes = append(nodes, n) @@ -169,31 +156,24 @@ func TestLongEOLRepublish(t *testing.T) { pinfo := host.InfoFromHost(nodes[0].h) for _, n := range nodes[1:] { - if err := n.h.Connect(ctx, *pinfo); err != nil { - t.Fatal(err) - } + err := n.h.Connect(ctx, *pinfo) + require.NoError(t, err) } // have one node publish a record that is valid for 1 second publisher := nodes[3] - p, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid - if err != nil { - t.Fatal(err) - } + p, err := path.NewPath("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") + require.NoError(t, err) - rp := namesys.NewIpnsPublisher(publisher.dht, publisher.store) - name := "/ipns/" + publisher.id + rp := namesys.NewIPNSPublisher(publisher.dht, publisher.store) + name := ipns.NameFromPeer(publisher.id).AsPath() expiration := time.Now().Add(time.Hour) - err = rp.Publish(ctx, publisher.privKey, p, opts.PublishWithEOL(expiration)) - if err != nil { - t.Fatal(err) - } + err = rp.Publish(ctx, publisher.privKey, p, namesys.PublishWithEOL(expiration)) + require.NoError(t, err) err = verifyResolution(nsystems, name, p) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) // The republishers that are contained within the nodes have their timeout set // to 12 hours. Instead of trying to tweak those, we're just going to pretend @@ -209,28 +189,19 @@ func TestLongEOLRepublish(t *testing.T) { time.Sleep(time.Second * 2) err = verifyResolution(nsystems, name, p) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) - rec, err := getLastIPNSRecord(ctx, publisher.store, publisher.h.ID()) - if err != nil { - t.Fatal(err) - } + rec, err := getLastIPNSRecord(ctx, publisher.store, ipns.NameFromPeer(publisher.h.ID())) + require.NoError(t, err) finalEol, err := rec.Validity() - if err != nil { - t.Fatal(err) - } - - if !finalEol.Equal(expiration) { - t.Fatal("expiration time modified") - } + require.NoError(t, err) + require.Equal(t, expiration.UTC(), finalEol.UTC()) } -func getLastIPNSRecord(ctx context.Context, dstore ds.Datastore, id peer.ID) (*ipns.Record, error) { +func getLastIPNSRecord(ctx context.Context, dstore ds.Datastore, name ipns.Name) (*ipns.Record, error) { // Look for it locally only - val, err := dstore.Get(ctx, namesys.IpnsDsKey(id)) + val, err := dstore.Get(ctx, namesys.IpnsDsKey(name)) if err != nil { return nil, err } @@ -238,23 +209,23 @@ func getLastIPNSRecord(ctx context.Context, dstore ds.Datastore, id peer.ID) (*i return ipns.UnmarshalRecord(val) } -func verifyResolution(nsystems []namesys.NameSystem, key string, exp path.Path) error { +func verifyResolution(nsystems []namesys.NameSystem, key path.Path, exp path.Path) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() for _, n := range nsystems { - val, err := n.Resolve(ctx, key) + res, err := n.Resolve(ctx, key) if err != nil { return err } - if val.String() != exp.String() { + if res.Path.String() != exp.String() { return errors.New("resolved wrong record") } } return nil } -func verifyResolutionFails(nsystems []namesys.NameSystem, key string) error { +func verifyResolutionFails(nsystems []namesys.NameSystem, key path.Path) error { ctx, cancel := context.WithCancel(context.Background()) defer cancel() for _, n := range nsystems { diff --git a/namesys/resolve/resolve.go b/namesys/resolve/resolve.go deleted file mode 100644 index b01fd38f7..000000000 --- a/namesys/resolve/resolve.go +++ /dev/null @@ -1,56 +0,0 @@ -package resolve - -import ( - "context" - "errors" - "fmt" - - "github.com/ipfs/boxo/path" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/ipfs/boxo/namesys" -) - -// ErrNoNamesys is an explicit error for when an IPFS node doesn't -// (yet) have a name system -var ErrNoNamesys = errors.New( - "core/resolve: no Namesys on IpfsNode - can't resolve ipns entry") - -// ResolveIPNS resolves /ipns paths -func ResolveIPNS(ctx context.Context, nsys namesys.NameSystem, p path.Path) (path.Path, error) { - ctx, span := namesys.StartSpan(ctx, "ResolveIPNS", trace.WithAttributes(attribute.String("Path", p.String()))) - defer span.End() - - if p.Namespace() == path.IPNSNamespace { - // TODO(cryptix): we should be able to query the local cache for the path - if nsys == nil { - return nil, ErrNoNamesys - } - - seg := p.Segments() - - if len(seg) < 2 || seg[1] == "" { // just "/" without further segments - err := fmt.Errorf("invalid path %q: ipns path missing IPNS ID", p) - return nil, err - } - - extensions := seg[2:] - resolvable, err := path.NewPathFromSegments(seg[0], seg[1]) - if err != nil { - return nil, err - } - - respath, err := nsys.Resolve(ctx, resolvable.String()) - if err != nil { - return nil, err - } - - segments := append(respath.Segments(), extensions...) - p, err = path.NewPathFromSegments(segments...) - if err != nil { - return nil, err - } - } - return p, nil -} diff --git a/namesys/resolve_test.go b/namesys/resolve_test.go deleted file mode 100644 index 158a9a26c..000000000 --- a/namesys/resolve_test.go +++ /dev/null @@ -1,132 +0,0 @@ -package namesys - -import ( - "context" - "errors" - "testing" - "time" - - ipns "github.com/ipfs/boxo/ipns" - "github.com/ipfs/boxo/path" - mockrouting "github.com/ipfs/boxo/routing/mock" - ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" - tnet "github.com/libp2p/go-libp2p-testing/net" -) - -func TestRoutingResolve(t *testing.T) { - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - serv := mockrouting.NewServer() - id := tnet.RandIdentityOrFatal(t) - d := serv.ClientWithDatastore(context.Background(), id, dstore) - - resolver := NewIpnsResolver(d) - publisher := NewIpnsPublisher(d, dstore) - - identity := tnet.RandIdentityOrFatal(t) - - h, err := path.NewPath("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") - if err != nil { - t.Fatal(err) - } - - err = publisher.Publish(context.Background(), identity.PrivateKey(), h) - if err != nil { - t.Fatal(err) - } - - res, err := resolver.Resolve(context.Background(), identity.ID().Pretty()) - if err != nil { - t.Fatal(err) - } - - if res.String() != h.String() { - t.Fatal("Got back incorrect value.") - } -} - -func TestPrexistingExpiredRecord(t *testing.T) { - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - d := mockrouting.NewServer().ClientWithDatastore(context.Background(), tnet.RandIdentityOrFatal(t), dstore) - - resolver := NewIpnsResolver(d) - publisher := NewIpnsPublisher(d, dstore) - - identity := tnet.RandIdentityOrFatal(t) - - // Make an expired record and put it in the datastore - h, err := path.NewPath("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") - if err != nil { - t.Fatal(err) - } - eol := time.Now().Add(time.Hour * -1) - - entry, err := ipns.NewRecord(identity.PrivateKey(), h, 0, eol, 0) - if err != nil { - t.Fatal(err) - } - err = PutRecordToRouting(context.Background(), d, identity.PublicKey(), entry) - if err != nil { - t.Fatal(err) - } - - // Now, with an old record in the system already, try and publish a new one - err = publisher.Publish(context.Background(), identity.PrivateKey(), h) - if err != nil { - t.Fatal(err) - } - - err = verifyCanResolve(resolver, identity.ID().Pretty(), h) - if err != nil { - t.Fatal(err) - } -} - -func TestPrexistingRecord(t *testing.T) { - dstore := dssync.MutexWrap(ds.NewMapDatastore()) - d := mockrouting.NewServer().ClientWithDatastore(context.Background(), tnet.RandIdentityOrFatal(t), dstore) - - resolver := NewIpnsResolver(d) - publisher := NewIpnsPublisher(d, dstore) - - identity := tnet.RandIdentityOrFatal(t) - - // Make a good record and put it in the datastore - h, err := path.NewPath("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN") - if err != nil { - t.Fatal(err) - } - eol := time.Now().Add(time.Hour) - entry, err := ipns.NewRecord(identity.PrivateKey(), h, 0, eol, 0) - if err != nil { - t.Fatal(err) - } - err = PutRecordToRouting(context.Background(), d, identity.PublicKey(), entry) - if err != nil { - t.Fatal(err) - } - - // Now, with an old record in the system already, try and publish a new one - err = publisher.Publish(context.Background(), identity.PrivateKey(), h) - if err != nil { - t.Fatal(err) - } - - err = verifyCanResolve(resolver, identity.ID().Pretty(), h) - if err != nil { - t.Fatal(err) - } -} - -func verifyCanResolve(r Resolver, name string, exp path.Path) error { - res, err := r.Resolve(context.Background(), name) - if err != nil { - return err - } - - if res.String() != exp.String() { - return errors.New("got back wrong record") - } - - return nil -} diff --git a/namesys/routing.go b/namesys/routing.go deleted file mode 100644 index 1153341ab..000000000 --- a/namesys/routing.go +++ /dev/null @@ -1,147 +0,0 @@ -package namesys - -import ( - "context" - "strings" - "time" - - opts "github.com/ipfs/boxo/coreiface/options/namesys" - "github.com/ipfs/boxo/ipns" - "github.com/ipfs/boxo/path" - logging "github.com/ipfs/go-log/v2" - dht "github.com/libp2p/go-libp2p-kad-dht" - "github.com/libp2p/go-libp2p/core/peer" - "github.com/libp2p/go-libp2p/core/routing" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" -) - -var log = logging.Logger("namesys") - -// IpnsResolver implements NSResolver for the main IPFS SFS-like naming -type IpnsResolver struct { - routing routing.ValueStore -} - -// NewIpnsResolver constructs a name resolver using the IPFS Routing system -// to implement SFS-like naming on top. -func NewIpnsResolver(route routing.ValueStore) *IpnsResolver { - if route == nil { - panic("attempt to create resolver with nil routing system") - } - return &IpnsResolver{ - routing: route, - } -} - -// Resolve implements Resolver. -func (r *IpnsResolver) Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (path.Path, error) { - ctx, span := StartSpan(ctx, "IpnsResolver.Resolve", trace.WithAttributes(attribute.String("Name", name))) - defer span.End() - return resolve(ctx, r, name, opts.ProcessOpts(options)) -} - -// ResolveAsync implements Resolver. -func (r *IpnsResolver) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result { - ctx, span := StartSpan(ctx, "IpnsResolver.ResolveAsync", trace.WithAttributes(attribute.String("Name", name))) - defer span.End() - return resolveAsync(ctx, r, name, opts.ProcessOpts(options)) -} - -// resolveOnce implements resolver. Uses the IPFS routing system to -// resolve SFS-like names. -func (r *IpnsResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult { - ctx, span := StartSpan(ctx, "IpnsResolver.ResolveOnceAsync", trace.WithAttributes(attribute.String("Name", name))) - defer span.End() - - out := make(chan onceResult, 1) - log.Debugf("RoutingResolver resolving %s", name) - cancel := func() {} - - if options.DhtTimeout != 0 { - // Resolution must complete within the timeout - ctx, cancel = context.WithTimeout(ctx, options.DhtTimeout) - } - - name = strings.TrimPrefix(name, "/ipns/") - - pid, err := peer.Decode(name) - if err != nil { - log.Debugf("RoutingResolver: could not convert public key hash %s to peer ID: %s\n", name, err) - out <- onceResult{err: err} - close(out) - cancel() - return out - } - - // Use the routing system to get the name. - // Note that the DHT will call the ipns validator when retrieving - // the value, which in turn verifies the ipns record signature - ipnsKey := string(ipns.NameFromPeer(pid).RoutingKey()) - - vals, err := r.routing.SearchValue(ctx, ipnsKey, dht.Quorum(int(options.DhtRecordCount))) - if err != nil { - log.Debugf("RoutingResolver: dht get for name %s failed: %s", name, err) - out <- onceResult{err: err} - close(out) - cancel() - return out - } - - go func() { - defer cancel() - defer close(out) - ctx, span := StartSpan(ctx, "IpnsResolver.ResolveOnceAsync.Worker") - defer span.End() - - for { - select { - case val, ok := <-vals: - if !ok { - return - } - - rec, err := ipns.UnmarshalRecord(val) - if err != nil { - log.Debugf("RoutingResolver: could not unmarshal value for name %s: %s", name, err) - emitOnceResult(ctx, out, onceResult{err: err}) - return - } - - p, err := rec.Value() - if err != nil { - emitOnceResult(ctx, out, onceResult{err: err}) - return - } - - ttl := DefaultResolverCacheTTL - if recordTTL, err := rec.TTL(); err == nil { - ttl = recordTTL - } - - switch eol, err := rec.Validity(); err { - case ipns.ErrUnrecognizedValidity: - // No EOL. - case nil: - ttEol := time.Until(eol) - if ttEol < 0 { - // It *was* valid when we first resolved it. - ttl = 0 - } else if ttEol < ttl { - ttl = ttEol - } - default: - log.Errorf("encountered error when parsing EOL: %s", err) - emitOnceResult(ctx, out, onceResult{err: err}) - return - } - - emitOnceResult(ctx, out, onceResult{value: p, ttl: ttl}) - case <-ctx.Done(): - return - } - } - }() - - return out -} diff --git a/namesys/tracing.go b/namesys/tracing.go deleted file mode 100644 index 4ef84294a..000000000 --- a/namesys/tracing.go +++ /dev/null @@ -1,13 +0,0 @@ -package namesys - -import ( - "context" - "fmt" - - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/trace" -) - -func StartSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { - return otel.Tracer("go-namesys").Start(ctx, fmt.Sprintf("Namesys.%s", name)) -} diff --git a/namesys/utilities.go b/namesys/utilities.go new file mode 100644 index 000000000..cba7ee728 --- /dev/null +++ b/namesys/utilities.go @@ -0,0 +1,146 @@ +package namesys + +import ( + "context" + "fmt" + "strings" + + "github.com/ipfs/boxo/path" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/trace" +) + +type resolver interface { + resolveOnceAsync(context.Context, path.Path, ResolveOptions) <-chan AsyncResult +} + +// resolve is a helper for implementing Resolver.ResolveN using resolveOnce. +func resolve(ctx context.Context, r resolver, p path.Path, options ResolveOptions) (result Result, err error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + err = ErrResolveFailed + resCh := resolveAsync(ctx, r, p, options) + + for res := range resCh { + result.Path, result.TTL, result.LastMod, err = res.Path, res.TTL, res.LastMod, res.Err + if err != nil { + break + } + } + + return result, err +} + +func resolveAsync(ctx context.Context, r resolver, p path.Path, options ResolveOptions) <-chan AsyncResult { + ctx, span := startSpan(ctx, "ResolveAsync") + defer span.End() + + resCh := r.resolveOnceAsync(ctx, p, options) + depth := options.Depth + outCh := make(chan AsyncResult, 1) + + go func() { + defer close(outCh) + ctx, span := startSpan(ctx, "ResolveAsync.Worker") + defer span.End() + + var subCh <-chan AsyncResult + var cancelSub context.CancelFunc + defer func() { + if cancelSub != nil { + cancelSub() + } + }() + + for { + select { + case res, ok := <-resCh: + if !ok { + resCh = nil + break + } + + if res.Err != nil { + emitResult(ctx, outCh, res) + return + } + + log.Debugf("resolved %s to %s", p.String(), res.Path.String()) + + if !res.Path.Mutable() { + emitResult(ctx, outCh, res) + break + } + + if depth == 1 { + res.Err = ErrResolveRecursion + emitResult(ctx, outCh, res) + break + } + + subOpts := options + if subOpts.Depth > 1 { + subOpts.Depth-- + } + + var subCtx context.Context + if cancelSub != nil { + // Cancel previous recursive resolve since it won't be used anyways + cancelSub() + } + + subCtx, cancelSub = context.WithCancel(ctx) + _ = cancelSub + + subCh = resolveAsync(subCtx, r, res.Path, subOpts) + case res, ok := <-subCh: + if !ok { + subCh = nil + break + } + + // We don't bother returning here in case of context timeout as there is + // no good reason to do that, and we may still be able to emit a result + emitResult(ctx, outCh, res) + case <-ctx.Done(): + return + } + if resCh == nil && subCh == nil { + return + } + } + }() + return outCh +} + +func emitResult(ctx context.Context, outCh chan<- AsyncResult, r AsyncResult) { + select { + case outCh <- r: + case <-ctx.Done(): + } +} + +func joinPaths(resolvedBase, unresolvedPath path.Path) (path.Path, error) { + if resolvedBase == nil { + return nil, nil + } + + segments := unresolvedPath.Segments()[2:] + if strings.HasSuffix(unresolvedPath.String(), "/") { + segments = append(segments, "") + } + + // simple optimization + if len(segments) == 0 { + return resolvedBase, nil + } + + return path.Join(resolvedBase, segments...) +} + +var tracer = otel.Tracer("boxo/namesys") + +func startSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) { + return tracer.Start(ctx, fmt.Sprintf("Namesys.%s", name)) +} From a7e134e54ff9fdeca075a27b277c12e80459f3e9 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 18 Oct 2023 19:47:05 +0200 Subject: [PATCH 21/63] fix(gw): use real FQDN in DNSLink errors The old version was not very smart and wasted DNS lookup per every inlined DNSLink label (my-v--long-example-com) before retrying with un-inlined one (my.v-long.example.com). This version is optimized to avoid unnecessary DNS lookup for the golden path majority of inlined DNS names on subdomain will hit. It also ensures that in case of missing DNSLink, the un-inlined domain name is used in error path, which makes better UX. --- gateway/hostname.go | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/gateway/hostname.go b/gateway/hostname.go index 758d8499c..a986ef24f 100644 --- a/gateway/hostname.go +++ b/gateway/hostname.go @@ -165,15 +165,32 @@ func NewHostnameHandler(c Config, backend IPFSBackend, next http.Handler) http.H // can be loaded from a subdomain gateway with a wildcard // TLS cert if represented as a single DNS label: // https://my-v--long-example-com.ipns.dweb.link - if ns == "ipns" && !strings.Contains(rootID, ".") { - // if there is no TXT recordfor rootID - if !hasDNSLinkRecord(r.Context(), backend, rootID) { - // my-v--long-example-com → my.v-long.example.com - dnslinkFQDN := toDNSLinkFQDN(rootID) - if hasDNSLinkRecord(r.Context(), backend, dnslinkFQDN) { - // update path prefix to use real FQDN with DNSLink - pathPrefix = "/ipns/" + dnslinkFQDN - } + if ns == "ipns" && !strings.Contains(rootID, ".") && strings.Contains(rootID, "-") { + // If there are no '.' but '-' is present in rootID, we most + // likely have an inlined DNSLink (like my-v--long-example-com) + + // We un-inline and check for DNSLink presence on domain with '.' + // first to minimize the amount of DNS lookups: + // my-v--long-example-com → my.v-long.example.com + dnslinkFQDN := toDNSLinkFQDN(rootID) + + // Does _dnslink.my.v-long.example.com exist? + if hasDNSLinkRecord(r.Context(), backend, dnslinkFQDN) { + // Un-inlined DNS name has a valid DNSLink record. + // Update path prefix to use un-inlined FQDN in gateway processing. + pathPrefix = "/ipns/" + dnslinkFQDN // → /ipns/my.v-long.example.com + + } else if !hasDNSLinkRecord(r.Context(), backend, rootID) { + // Inspected _dnslink.my-v--long-example-com as a + // fallback, but it had no DNSLink record either. + + // At this point it is more likely the un-inlined + // dnslinkFQDN is what the end user wanted to load, so + // we switch to that. This ensures the error message + // about missing DNSLink will use the un-inlined FQDN, + // and not the inlined one. + pathPrefix = "/ipns/" + dnslinkFQDN + } } } From 40fb162d5e2bc2c55476566a5f9fb5563c617bb4 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Wed, 18 Oct 2023 22:59:29 +0200 Subject: [PATCH 22/63] feat(gw): improved dnslink inliner faster implementation + exported funcs for reuse outside of boxo/gateway --- gateway/hostname.go | 47 +++++++++++++++++++++++++---- gateway/hostname_test.go | 65 ++++++++++++++++++++++++++++++++++++++-- 2 files changed, 104 insertions(+), 8 deletions(-) diff --git a/gateway/hostname.go b/gateway/hostname.go index a986ef24f..6b485f0b4 100644 --- a/gateway/hostname.go +++ b/gateway/hostname.go @@ -172,7 +172,7 @@ func NewHostnameHandler(c Config, backend IPFSBackend, next http.Handler) http.H // We un-inline and check for DNSLink presence on domain with '.' // first to minimize the amount of DNS lookups: // my-v--long-example-com → my.v-long.example.com - dnslinkFQDN := toDNSLinkFQDN(rootID) + dnslinkFQDN := UninlineDNSLink(rootID) // Does _dnslink.my.v-long.example.com exist? if hasDNSLinkRecord(r.Context(), backend, dnslinkFQDN) { @@ -323,22 +323,59 @@ func isHTTPSRequest(r *http.Request) bool { // Converts a FQDN to DNS-safe representation that fits in 63 characters: // my.v-long.example.com → my-v--long-example-com -func toDNSLinkDNSLabel(fqdn string) (dnsLabel string, err error) { +// InlineDNSLink implements specification from https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header +func InlineDNSLink(fqdn string) (dnsLabel string, err error) { + /* What follows is an optimized version this three-liner: dnsLabel = strings.ReplaceAll(fqdn, "-", "--") dnsLabel = strings.ReplaceAll(dnsLabel, ".", "-") if len(dnsLabel) > dnsLabelMaxLength { return "", fmt.Errorf("DNSLink representation incompatible with DNS label length limit of 63: %s", dnsLabel) } return dnsLabel, nil + */ + result := make([]byte, 0, len(fqdn)) + for i := 0; i < len(fqdn); i++ { + char := fqdn[i] + if char == '-' { + result = append(result, '-', '-') + } else if char == '.' { + result = append(result, '-') + } else { + result = append(result, char) + } + } + if len(result) > dnsLabelMaxLength { + return "", fmt.Errorf("inlined DNSLink incompatible with DNS label length limit of 63: %q", result) + } + return string(result), nil } // Converts a DNS-safe representation of DNSLink FQDN to real FQDN: // my-v--long-example-com → my.v-long.example.com -func toDNSLinkFQDN(dnsLabel string) (fqdn string) { +// UninlineDNSLink implements specification from https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header +func UninlineDNSLink(dnsLabel string) (fqdn string) { + /* What follows is an optimized version this three-liner: fqdn = strings.ReplaceAll(dnsLabel, "--", "@") // @ placeholder is unused in DNS labels fqdn = strings.ReplaceAll(fqdn, "-", ".") fqdn = strings.ReplaceAll(fqdn, "@", "-") return fqdn + */ + result := make([]byte, 0, len(dnsLabel)) + for i := 0; i < len(dnsLabel); i++ { + if dnsLabel[i] == '-' { + if i+1 < len(dnsLabel) && dnsLabel[i+1] == '-' { + // Handle '--' by appending a single '-' + result = append(result, '-') + i++ + } else { + // Handle single '-' by appending '.' + result = append(result, '.') + } + } else { + result = append(result, dnsLabel[i]) + } + } + return string(result) } // Converts a hostname/path to a subdomain-based URL, if applicable. @@ -419,7 +456,7 @@ func toSubdomainURL(hostname, path string, r *http.Request, inlineDNSLink bool, // e.g. when ipfs-companion extension passes value from subdomain gateway // for further normalization: https://github.com/ipfs/ipfs-companion/issues/1278#issuecomment-1724550623 if ns == "ipns" && !strings.Contains(rootID, ".") && strings.Contains(rootID, "-") { - dnsLinkFqdn := toDNSLinkFQDN(rootID) // my-v--long-example-com → my.v-long.example.com + dnsLinkFqdn := UninlineDNSLink(rootID) // my-v--long-example-com → my.v-long.example.com if hasDNSLinkRecord(r.Context(), backend, dnsLinkFqdn) { // update path prefix to use real FQDN with DNSLink rootID = dnsLinkFqdn @@ -442,7 +479,7 @@ func toSubdomainURL(hostname, path string, r *http.Request, inlineDNSLink bool, // https://my-v--long-example-com.ipns.dweb.link if hasDNSLinkRecord(r.Context(), backend, rootID) { // my.v-long.example.com → my-v--long-example-com - dnsLabel, err := toDNSLinkDNSLabel(rootID) + dnsLabel, err := InlineDNSLink(rootID) if err != nil { return "", err } diff --git a/gateway/hostname_test.go b/gateway/hostname_test.go index 7a8d7a170..1a150facb 100644 --- a/gateway/hostname_test.go +++ b/gateway/hostname_test.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "strings" "testing" "github.com/ipfs/boxo/path" @@ -82,10 +83,13 @@ func TestToDNSLinkDNSLabel(t *testing.T) { err error }{ {"dnslink.long-name.example.com", "dnslink-long--name-example-com", nil}, - {"dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com", "", errors.New("DNSLink representation incompatible with DNS label length limit of 63: dnslink-too--long-f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o-example-com")}, + {"singlelabel", "singlelabel", nil}, + {"example.com", "example-com", nil}, + {"en.wikipedia-on-ipfs.org", "en-wikipedia--on--ipfs-org", nil}, + {"dnslink.too-long.f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o.example.com", "", errors.New(`inlined DNSLink incompatible with DNS label length limit of 63: "dnslink-too--long-f1siqrebi3vir8sab33hu5vcy008djegvay6atmz91ojesyjs8lx350b7y7i1nvyw2haytfukfyu2f2x4tocdrfa0zgij6p4zpl4u5o-example-com"`)}, } { t.Run(test.in, func(t *testing.T) { - out, err := toDNSLinkDNSLabel(test.in) + out, err := InlineDNSLink(test.in) require.Equal(t, test.out, out) require.Equal(t, test.err, err) }) @@ -99,11 +103,14 @@ func TestToDNSLinkFQDN(t *testing.T) { out string }{ {"singlelabel", "singlelabel"}, + {"no--tld", "no-tld"}, + {"example.com", "example.com"}, {"docs-ipfs-tech", "docs.ipfs.tech"}, + {"en-wikipedia--on--ipfs-org", "en.wikipedia-on-ipfs.org"}, {"dnslink-long--name-example-com", "dnslink.long-name.example.com"}, } { t.Run(test.in, func(t *testing.T) { - out := toDNSLinkFQDN(test.in) + out := UninlineDNSLink(test.in) require.Equal(t, test.out, out) }) } @@ -305,3 +312,55 @@ func TestKnownSubdomainDetails(t *testing.T) { }) } } + +const testInlinedDNSLinkA = "example-com" +const testInlinedDNSLinkB = "docs-ipfs-tech" +const testInlinedDNSLinkC = "en-wikipedia--on--ipfs-org" +const testDNSLinkA = "example.com" +const testDNSLinkB = "docs.ipfs.tech" +const testDNSLinkC = "en.wikipedia-on-ipfs.org" + +func inlineDNSLinkSimple(fqdn string) (dnsLabel string, err error) { + dnsLabel = strings.ReplaceAll(fqdn, "-", "--") + dnsLabel = strings.ReplaceAll(dnsLabel, ".", "-") + if len(dnsLabel) > dnsLabelMaxLength { + return "", fmt.Errorf("inlined DNSLink incompatible with DNS label length limit of 63: %q", dnsLabel) + } + return dnsLabel, nil +} +func uninlineDNSLinkSimple(dnsLabel string) (fqdn string) { + fqdn = strings.ReplaceAll(dnsLabel, "--", "@") // @ placeholder is unused in DNS labels + fqdn = strings.ReplaceAll(fqdn, "-", ".") + fqdn = strings.ReplaceAll(fqdn, "@", "-") + return fqdn +} + +func BenchmarkUninlineDNSLinkSimple(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = uninlineDNSLinkSimple(testInlinedDNSLinkA) + _ = uninlineDNSLinkSimple(testInlinedDNSLinkB) + _ = uninlineDNSLinkSimple(testInlinedDNSLinkC) + } +} +func BenchmarkUninlineDNSLink(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = UninlineDNSLink(testInlinedDNSLinkA) + _ = UninlineDNSLink(testInlinedDNSLinkB) + _ = UninlineDNSLink(testInlinedDNSLinkC) + } +} + +func BenchmarkInlineDNSLinkSimple(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = inlineDNSLinkSimple(testDNSLinkA) + _, _ = inlineDNSLinkSimple(testDNSLinkB) + _, _ = inlineDNSLinkSimple(testDNSLinkC) + } +} +func BenchmarkInlineDNSLink(b *testing.B) { + for i := 0; i < b.N; i++ { + _, _ = InlineDNSLink(testDNSLinkA) + _, _ = InlineDNSLink(testDNSLinkB) + _, _ = InlineDNSLink(testDNSLinkC) + } +} From dcee31d70e912a7624343372feaa442bc7aa9c60 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 25 May 2020 09:26:21 -0700 Subject: [PATCH 23/63] feat: implement peering service MVP for #6097 This feature will repeatedly reconnect (with a randomized exponential backoff) to peers in a set of "peered" peers. In the future, this should be extended to: 1. Include a CLI for modifying this list at runtime. 2. Include additional options for peers we want to _protect_ but not connect to. 3. Allow configuring timeouts, backoff, etc. 4. Allow groups? Possibly through textile threads. 5. Allow for runtime-only peering rules. 6. Different reconnect policies. But this MVP should be a significant step forward. This commit was moved from ipfs/kubo@978091a626a0e1f00a797fc4e2de99f4bfee943b --- peering/peering.go | 259 ++++++++++++++++++++++++++++++++++++++++ peering/peering_test.go | 6 + 2 files changed, 265 insertions(+) create mode 100644 peering/peering.go create mode 100644 peering/peering_test.go diff --git a/peering/peering.go b/peering/peering.go new file mode 100644 index 000000000..c543712c5 --- /dev/null +++ b/peering/peering.go @@ -0,0 +1,259 @@ +package peering + +import ( + "context" + "errors" + "math/rand" + "sync" + "time" + + "github.com/ipfs/go-log" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/multiformats/go-multiaddr" +) + +// maxBackoff is the maximum time between reconnect attempts. +const ( + maxBackoff = 10 * time.Minute + connmgrTag = "ipfs-peering" + // This needs to be sufficient to prevent two sides from simultaneously + // dialing. + initialDelay = 5 * time.Second +) + +var logger = log.Logger("peering") + +type state int + +const ( + stateInit state = iota + stateRunning + stateStopped +) + +// peerHandler keeps track of all state related to a specific "peering" peer. +type peerHandler struct { + peer peer.ID + host host.Host + ctx context.Context + cancel context.CancelFunc + + mu sync.Mutex + addrs []multiaddr.Multiaddr + timer *time.Timer + + nextDelay time.Duration +} + +func (ph *peerHandler) stop() { + ph.mu.Lock() + defer ph.mu.Unlock() + + if ph.timer != nil { + ph.timer.Stop() + ph.timer = nil + } +} + +func (ph *peerHandler) nextBackoff() time.Duration { + // calculate the timeout + if ph.nextDelay < maxBackoff { + ph.nextDelay += ph.nextDelay/2 + time.Duration(rand.Int63n(int64(ph.nextDelay))) + } + return ph.nextDelay +} + +func (ph *peerHandler) reconnect() { + // Try connecting + + ph.mu.Lock() + addrs := append(([]multiaddr.Multiaddr)(nil), ph.addrs...) + ph.mu.Unlock() + + logger.Debugw("reconnecting", "peer", ph.peer, "addrs", addrs) + + err := ph.host.Connect(ph.ctx, peer.AddrInfo{ID: ph.peer, Addrs: addrs}) + if err != nil { + logger.Debugw("failed to reconnect", "peer", ph.peer, "error", err) + // Ok, we failed. Extend the timeout. + ph.mu.Lock() + if ph.timer != nil { + // Only counts if the timer still exists. If not, a + // connection _was_ somehow established. + ph.timer.Reset(ph.nextBackoff()) + } + // Otherwise, someone else has stopped us so we can assume that + // we're either connected or someone else will start us. + ph.mu.Unlock() + } + + // Always call this. We could have connected since we processed the + // error. + ph.stopIfConnected() +} + +func (ph *peerHandler) stopIfConnected() { + ph.mu.Lock() + defer ph.mu.Unlock() + + if ph.timer != nil && ph.host.Network().Connectedness(ph.peer) == network.Connected { + logger.Debugw("successfully reconnected", "peer", ph.peer) + ph.timer.Stop() + ph.timer = nil + ph.nextDelay = initialDelay + } +} + +// startIfDisconnected is the inverse of stopIfConnected. +func (ph *peerHandler) startIfDisconnected() { + ph.mu.Lock() + defer ph.mu.Unlock() + + if ph.timer == nil && ph.host.Network().Connectedness(ph.peer) != network.Connected { + logger.Debugw("disconnected from peer", "peer", ph.peer) + // Always start with a short timeout so we can stagger things a bit. + ph.timer = time.AfterFunc(ph.nextBackoff(), ph.reconnect) + } +} + +// PeeringService maintains connections to specified peers, reconnecting on +// disconnect with a back-off. +type PeeringService struct { + host host.Host + + mu sync.RWMutex + peers map[peer.ID]*peerHandler + + ctx context.Context + cancel context.CancelFunc + state state +} + +// NewPeeringService constructs a new peering service. Peers can be added and +// removed immediately, but connections won't be formed until `Start` is called. +func NewPeeringService(host host.Host) *PeeringService { + ps := &PeeringService{host: host, peers: make(map[peer.ID]*peerHandler)} + ps.ctx, ps.cancel = context.WithCancel(context.Background()) + return ps +} + +// Start starts the peering service, connecting and maintaining connections to +// all registered peers. It returns an error if the service has already been +// stopped. +func (ps *PeeringService) Start() error { + ps.mu.Lock() + defer ps.mu.Unlock() + + switch ps.state { + case stateInit: + logger.Infow("starting") + case stateRunning: + return nil + case stateStopped: + return errors.New("already stopped") + } + ps.host.Network().Notify((*netNotifee)(ps)) + ps.state = stateRunning + for _, handler := range ps.peers { + go handler.startIfDisconnected() + } + return nil +} + +// Stop stops the peering service. +func (ps *PeeringService) Stop() error { + ps.cancel() + ps.host.Network().StopNotify((*netNotifee)(ps)) + + ps.mu.Lock() + defer ps.mu.Unlock() + + if ps.state == stateRunning { + logger.Infow("stopping") + for _, handler := range ps.peers { + handler.stop() + } + } + return nil +} + +// AddPeer adds a peer to the peering service. This function may be safely +// called at any time: before the service is started, while running, or after it +// stops. +// +// Add peer may also be called multiple times for the same peer. The new +// addresses will replace the old. +func (ps *PeeringService) AddPeer(info peer.AddrInfo) { + ps.mu.Lock() + defer ps.mu.Unlock() + + if handler, ok := ps.peers[info.ID]; ok { + logger.Infow("updating addresses", "peer", info.ID, "addrs", info.Addrs) + handler.addrs = info.Addrs + } else { + logger.Infow("peer added", "peer", info.ID, "addrs", info.Addrs) + ps.host.ConnManager().Protect(info.ID, connmgrTag) + + handler = &peerHandler{ + host: ps.host, + peer: info.ID, + addrs: info.Addrs, + nextDelay: initialDelay, + } + handler.ctx, handler.cancel = context.WithCancel(ps.ctx) + ps.peers[info.ID] = handler + if ps.state == stateRunning { + go handler.startIfDisconnected() + } + } +} + +// RemovePeer removes a peer from the peering service. This function may be +// safely called at any time: before the service is started, while running, or +// after it stops. +func (ps *PeeringService) RemovePeer(id peer.ID) { + ps.mu.Lock() + defer ps.mu.Unlock() + + if handler, ok := ps.peers[id]; ok { + logger.Infow("peer removed", "peer", id) + ps.host.ConnManager().Unprotect(id, connmgrTag) + + handler.stop() + handler.cancel() + delete(ps.peers, id) + } +} + +type netNotifee PeeringService + +func (nn *netNotifee) Connected(_ network.Network, c network.Conn) { + ps := (*PeeringService)(nn) + + p := c.RemotePeer() + ps.mu.RLock() + defer ps.mu.RUnlock() + + if handler, ok := ps.peers[p]; ok { + // use a goroutine to avoid blocking events. + go handler.stopIfConnected() + } +} +func (nn *netNotifee) Disconnected(_ network.Network, c network.Conn) { + ps := (*PeeringService)(nn) + + p := c.RemotePeer() + ps.mu.RLock() + defer ps.mu.RUnlock() + + if handler, ok := ps.peers[p]; ok { + // use a goroutine to avoid blocking events. + go handler.startIfDisconnected() + } +} +func (nn *netNotifee) OpenedStream(network.Network, network.Stream) {} +func (nn *netNotifee) ClosedStream(network.Network, network.Stream) {} +func (nn *netNotifee) Listen(network.Network, multiaddr.Multiaddr) {} +func (nn *netNotifee) ListenClose(network.Network, multiaddr.Multiaddr) {} diff --git a/peering/peering_test.go b/peering/peering_test.go new file mode 100644 index 000000000..0be08dcdc --- /dev/null +++ b/peering/peering_test.go @@ -0,0 +1,6 @@ +package peering + +import "testing" + +func TestPeeringService(t *testing.T) { +} From 679b513b0b3668ee856652847c0f56a33a3da758 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 25 May 2020 18:53:55 -0700 Subject: [PATCH 24/63] fix: doc comment location Co-authored-by: Will This commit was moved from ipfs/kubo@8e52c7fb2d04482b108489e7cafd95ec6ebb0375 --- peering/peering.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/peering/peering.go b/peering/peering.go index c543712c5..ef0aa4d2e 100644 --- a/peering/peering.go +++ b/peering/peering.go @@ -14,8 +14,8 @@ import ( "github.com/multiformats/go-multiaddr" ) -// maxBackoff is the maximum time between reconnect attempts. const ( + // maxBackoff is the maximum time between reconnect attempts. maxBackoff = 10 * time.Minute connmgrTag = "ipfs-peering" // This needs to be sufficient to prevent two sides from simultaneously From 72a2daffc9361a7fa447341da65c1b7dc38886aa Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 25 May 2020 19:11:13 -0700 Subject: [PATCH 25/63] fix: address peering service code feedback * better name for timer * cancel context from within stop This commit was moved from ipfs/kubo@0551c4dca843fe3a9665fdc49cd2dcecaeb85046 --- peering/peering.go | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/peering/peering.go b/peering/peering.go index ef0aa4d2e..9785b6555 100644 --- a/peering/peering.go +++ b/peering/peering.go @@ -40,20 +40,22 @@ type peerHandler struct { ctx context.Context cancel context.CancelFunc - mu sync.Mutex - addrs []multiaddr.Multiaddr - timer *time.Timer + mu sync.Mutex + addrs []multiaddr.Multiaddr + reconnectTimer *time.Timer nextDelay time.Duration } +// stop permanently stops the peer handler. func (ph *peerHandler) stop() { + ph.cancel() + ph.mu.Lock() defer ph.mu.Unlock() - - if ph.timer != nil { - ph.timer.Stop() - ph.timer = nil + if ph.reconnectTimer != nil { + ph.reconnectTimer.Stop() + ph.reconnectTimer = nil } } @@ -79,10 +81,10 @@ func (ph *peerHandler) reconnect() { logger.Debugw("failed to reconnect", "peer", ph.peer, "error", err) // Ok, we failed. Extend the timeout. ph.mu.Lock() - if ph.timer != nil { - // Only counts if the timer still exists. If not, a + if ph.reconnectTimer != nil { + // Only counts if the reconnectTimer still exists. If not, a // connection _was_ somehow established. - ph.timer.Reset(ph.nextBackoff()) + ph.reconnectTimer.Reset(ph.nextBackoff()) } // Otherwise, someone else has stopped us so we can assume that // we're either connected or someone else will start us. @@ -98,10 +100,10 @@ func (ph *peerHandler) stopIfConnected() { ph.mu.Lock() defer ph.mu.Unlock() - if ph.timer != nil && ph.host.Network().Connectedness(ph.peer) == network.Connected { + if ph.reconnectTimer != nil && ph.host.Network().Connectedness(ph.peer) == network.Connected { logger.Debugw("successfully reconnected", "peer", ph.peer) - ph.timer.Stop() - ph.timer = nil + ph.reconnectTimer.Stop() + ph.reconnectTimer = nil ph.nextDelay = initialDelay } } @@ -111,10 +113,10 @@ func (ph *peerHandler) startIfDisconnected() { ph.mu.Lock() defer ph.mu.Unlock() - if ph.timer == nil && ph.host.Network().Connectedness(ph.peer) != network.Connected { + if ph.reconnectTimer == nil && ph.host.Network().Connectedness(ph.peer) != network.Connected { logger.Debugw("disconnected from peer", "peer", ph.peer) // Always start with a short timeout so we can stagger things a bit. - ph.timer = time.AfterFunc(ph.nextBackoff(), ph.reconnect) + ph.reconnectTimer = time.AfterFunc(ph.nextBackoff(), ph.reconnect) } } @@ -222,7 +224,6 @@ func (ps *PeeringService) RemovePeer(id peer.ID) { ps.host.ConnManager().Unprotect(id, connmgrTag) handler.stop() - handler.cancel() delete(ps.peers, id) } } From e5a61649e264e7eb13a4abeb02ac2e6e12215f7d Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 25 May 2020 20:02:23 -0700 Subject: [PATCH 26/63] test: add unit test for peering service This commit was moved from ipfs/kubo@fe2b289d3002ed11a1dbcc0521bc42d33f0d46ed --- peering/peering_test.go | 135 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 134 insertions(+), 1 deletion(-) diff --git a/peering/peering_test.go b/peering/peering_test.go index 0be08dcdc..0d03aaf8e 100644 --- a/peering/peering_test.go +++ b/peering/peering_test.go @@ -1,6 +1,139 @@ package peering -import "testing" +import ( + "context" + "testing" + "time" + + "github.com/libp2p/go-libp2p" + connmgr "github.com/libp2p/go-libp2p-connmgr" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + + "github.com/stretchr/testify/require" +) + +func newNode(ctx context.Context, t *testing.T) host.Host { + h, err := libp2p.New( + ctx, + libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"), + // We'd like to set the connection manager low water to 0, but + // that would disable the connection manager. + libp2p.ConnectionManager(connmgr.NewConnManager(1, 100, 0)), + ) + require.NoError(t, err) + return h +} func TestPeeringService(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + h1 := newNode(ctx, t) + ps1 := NewPeeringService(h1) + + h2 := newNode(ctx, t) + h3 := newNode(ctx, t) + h4 := newNode(ctx, t) + + // peer 1 -> 2 + ps1.AddPeer(peer.AddrInfo{ID: h2.ID(), Addrs: h2.Addrs()}) + + // We haven't started so we shouldn't have any peers. + require.Never(t, func() bool { + return len(h1.Network().Peers()) > 0 + }, 100*time.Millisecond, 1*time.Second, "expected host 1 to have no peers") + + // Use p4 to take up the one slot we have in the connection manager. + for _, h := range []host.Host{h1, h2} { + require.NoError(t, h.Connect(ctx, peer.AddrInfo{ID: h4.ID(), Addrs: h4.Addrs()})) + h.ConnManager().TagPeer(h4.ID(), "sticky-peer", 1000) + } + + // Now start. + require.NoError(t, ps1.Start()) + // starting twice is fine. + require.NoError(t, ps1.Start()) + + // We should eventually connect. + require.Eventually(t, func() bool { + return h1.Network().Connectedness(h2.ID()) == network.Connected + }, 30*time.Second, 10*time.Millisecond) + + // Now explicitly connect to p3. + require.NoError(t, h1.Connect(ctx, peer.AddrInfo{ID: h3.ID(), Addrs: h3.Addrs()})) + require.Eventually(t, func() bool { + return h1.Network().Connectedness(h2.ID()) == network.Connected + }, 30*time.Second, 100*time.Millisecond) + + require.Len(t, h1.Network().Peers(), 3) + + // force a disconnect + h1.ConnManager().TrimOpenConns(ctx) + + // Should disconnect from p3. + require.Eventually(t, func() bool { + return h1.Network().Connectedness(h3.ID()) != network.Connected + }, 5*time.Second, 10*time.Millisecond) + + // Should remain connected to p2 + require.Never(t, func() bool { + return h1.Network().Connectedness(h2.ID()) != network.Connected + }, 5*time.Second, 1*time.Second) + + // Now force h2 to disconnect (we have an asymmetric peering). + conns := h2.Network().ConnsToPeer(h1.ID()) + require.NotEmpty(t, conns) + h2.ConnManager().TrimOpenConns(ctx) + + // All conns to peer should eventually close. + for _, c := range conns { + require.Eventually(t, func() bool { + s, err := c.NewStream() + if s != nil { + _ = s.Reset() + } + return err != nil + }, 5*time.Second, 10*time.Millisecond) + } + + // Should eventually re-connect. + require.Eventually(t, func() bool { + return h1.Network().Connectedness(h2.ID()) == network.Connected + }, 30*time.Second, 1*time.Second) + + // Unprotect 2 from 1. + ps1.RemovePeer(h2.ID()) + + // Trim connections. + h1.ConnManager().TrimOpenConns(ctx) + + // Should disconnect + require.Eventually(t, func() bool { + return h1.Network().Connectedness(h2.ID()) != network.Connected + }, 5*time.Second, 10*time.Millisecond) + + // Should never reconnect. + require.Never(t, func() bool { + return h1.Network().Connectedness(h2.ID()) == network.Connected + }, 20*time.Second, 1*time.Second) + + // Until added back + ps1.AddPeer(peer.AddrInfo{ID: h2.ID(), Addrs: h2.Addrs()}) + ps1.AddPeer(peer.AddrInfo{ID: h3.ID(), Addrs: h3.Addrs()}) + require.Eventually(t, func() bool { + return h1.Network().Connectedness(h2.ID()) == network.Connected + }, 30*time.Second, 1*time.Second) + require.Eventually(t, func() bool { + return h1.Network().Connectedness(h3.ID()) == network.Connected + }, 30*time.Second, 1*time.Second) + + // Should be able to repeatedly stop. + require.NoError(t, ps1.Stop()) + require.NoError(t, ps1.Stop()) + + // Adding and removing should work after stopping. + ps1.AddPeer(peer.AddrInfo{ID: h4.ID(), Addrs: h4.Addrs()}) + ps1.RemovePeer(h2.ID()) } From 4c8297f13a2380a9e3d7d52a0c9f553c3a146d50 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 25 May 2020 20:09:50 -0700 Subject: [PATCH 27/63] fix(peering): fix a race condition This commit was moved from ipfs/kubo@87a293f6801b176d9a947c3bf49c511e3dd98dce --- peering/peering.go | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/peering/peering.go b/peering/peering.go index 9785b6555..5f78a44f6 100644 --- a/peering/peering.go +++ b/peering/peering.go @@ -47,6 +47,24 @@ type peerHandler struct { nextDelay time.Duration } +// setAddrs sets the addresses for this peer. +func (ph *peerHandler) setAddrs(addrs []multiaddr.Multiaddr) { + // Not strictly necessary, but it helps to not trust the calling code. + addrCopy := make([]multiaddr.Multiaddr, len(addrs)) + copy(addrCopy, addrs) + + ph.mu.Lock() + defer ph.mu.Unlock() + ph.addrs = addrCopy +} + +// getAddrs returns a shared slice of addresses for this peer. Do not modify. +func (ph *peerHandler) getAddrs() []multiaddr.Multiaddr { + ph.mu.Lock() + defer ph.mu.Unlock() + return ph.addrs +} + // stop permanently stops the peer handler. func (ph *peerHandler) stop() { ph.cancel() @@ -69,11 +87,7 @@ func (ph *peerHandler) nextBackoff() time.Duration { func (ph *peerHandler) reconnect() { // Try connecting - - ph.mu.Lock() - addrs := append(([]multiaddr.Multiaddr)(nil), ph.addrs...) - ph.mu.Unlock() - + addrs := ph.getAddrs() logger.Debugw("reconnecting", "peer", ph.peer, "addrs", addrs) err := ph.host.Connect(ph.ctx, peer.AddrInfo{ID: ph.peer, Addrs: addrs}) @@ -193,7 +207,7 @@ func (ps *PeeringService) AddPeer(info peer.AddrInfo) { if handler, ok := ps.peers[info.ID]; ok { logger.Infow("updating addresses", "peer", info.ID, "addrs", info.Addrs) - handler.addrs = info.Addrs + handler.setAddrs(info.Addrs) } else { logger.Infow("peer added", "peer", info.ID, "addrs", info.Addrs) ps.host.ConnManager().Protect(info.ID, connmgrTag) From 8caeb2740c8c914bc0c64f7c1495bad9d24d426e Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 25 May 2020 20:20:13 -0700 Subject: [PATCH 28/63] fix: remove unecessary context This commit was moved from ipfs/kubo@17b3b02549ef96534b4d170674d4ccf174efb1d1 --- peering/peering.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/peering/peering.go b/peering/peering.go index 5f78a44f6..663f41017 100644 --- a/peering/peering.go +++ b/peering/peering.go @@ -141,18 +141,13 @@ type PeeringService struct { mu sync.RWMutex peers map[peer.ID]*peerHandler - - ctx context.Context - cancel context.CancelFunc - state state + state state } // NewPeeringService constructs a new peering service. Peers can be added and // removed immediately, but connections won't be formed until `Start` is called. func NewPeeringService(host host.Host) *PeeringService { - ps := &PeeringService{host: host, peers: make(map[peer.ID]*peerHandler)} - ps.ctx, ps.cancel = context.WithCancel(context.Background()) - return ps + return &PeeringService{host: host, peers: make(map[peer.ID]*peerHandler)} } // Start starts the peering service, connecting and maintaining connections to @@ -180,17 +175,18 @@ func (ps *PeeringService) Start() error { // Stop stops the peering service. func (ps *PeeringService) Stop() error { - ps.cancel() ps.host.Network().StopNotify((*netNotifee)(ps)) ps.mu.Lock() defer ps.mu.Unlock() - if ps.state == stateRunning { + switch ps.state { + case stateInit, stateRunning: logger.Infow("stopping") for _, handler := range ps.peers { handler.stop() } + ps.state = stateStopped } return nil } @@ -218,10 +214,16 @@ func (ps *PeeringService) AddPeer(info peer.AddrInfo) { addrs: info.Addrs, nextDelay: initialDelay, } - handler.ctx, handler.cancel = context.WithCancel(ps.ctx) + handler.ctx, handler.cancel = context.WithCancel(context.Background()) ps.peers[info.ID] = handler - if ps.state == stateRunning { + switch ps.state { + case stateRunning: go handler.startIfDisconnected() + case stateStopped: + // We still construct everything in this state because + // it's easier to reason about. But we should still free + // resources. + handler.cancel() } } } From ae892df69d97867c3902579b64272c561c3f53d0 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 25 May 2020 21:18:45 -0700 Subject: [PATCH 29/63] fix: really cap the max backoff at 10 minutes While preserving some randomness. And add a test. This commit was moved from ipfs/kubo@e10289a93d50e3cc80a5a3692b98391cc1aab62b --- peering/peering.go | 18 ++++++++++++++++-- peering/peering_test.go | 19 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/peering/peering.go b/peering/peering.go index 663f41017..ed0b43226 100644 --- a/peering/peering.go +++ b/peering/peering.go @@ -14,10 +14,17 @@ import ( "github.com/multiformats/go-multiaddr" ) +// Seed the random number generator. +// +// We don't need good randomness, but we do need randomness. const ( // maxBackoff is the maximum time between reconnect attempts. maxBackoff = 10 * time.Minute - connmgrTag = "ipfs-peering" + // The backoff will be cut off when we get within 10% of the actual max. + // If we go over the max, we'll adjust the delay down to a random value + // between 90-100% of the max backoff. + maxBackoffJitter = 10 // % + connmgrTag = "ipfs-peering" // This needs to be sufficient to prevent two sides from simultaneously // dialing. initialDelay = 5 * time.Second @@ -78,10 +85,17 @@ func (ph *peerHandler) stop() { } func (ph *peerHandler) nextBackoff() time.Duration { - // calculate the timeout if ph.nextDelay < maxBackoff { ph.nextDelay += ph.nextDelay/2 + time.Duration(rand.Int63n(int64(ph.nextDelay))) } + + // If we've gone over the max backoff, reduce it under the max. + if ph.nextDelay > maxBackoff { + ph.nextDelay = maxBackoff + // randomize the backoff a bit (10%). + ph.nextDelay -= time.Duration(rand.Int63n(int64(maxBackoff) * maxBackoffJitter / 100)) + } + return ph.nextDelay } diff --git a/peering/peering_test.go b/peering/peering_test.go index 0d03aaf8e..1f21b7816 100644 --- a/peering/peering_test.go +++ b/peering/peering_test.go @@ -137,3 +137,22 @@ func TestPeeringService(t *testing.T) { ps1.AddPeer(peer.AddrInfo{ID: h4.ID(), Addrs: h4.Addrs()}) ps1.RemovePeer(h2.ID()) } + +func TestNextBackoff(t *testing.T) { + minMaxBackoff := (100 - maxBackoffJitter) / 100 * maxBackoff + for x := 0; x < 1000; x++ { + ph := peerHandler{nextDelay: time.Second} + for min, max := time.Second*3/2, time.Second*5/2; min < minMaxBackoff; min, max = min*3/2, max*5/2 { + b := ph.nextBackoff() + if b > max || b < min { + t.Errorf("expected backoff %s to be between %s and %s", b, min, max) + } + } + for i := 0; i < 100; i++ { + b := ph.nextBackoff() + if b < minMaxBackoff || b > maxBackoff { + t.Fatal("failed to stay within max bounds") + } + } + } +} From a163799af28b50177571ac93e41654d8f11b538f Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Sat, 19 Dec 2020 16:16:00 +0700 Subject: [PATCH 30/63] update go-libp2p to v0.13.0 This commit was moved from ipfs/kubo@fcdf77c9b0fe33268ad6048d973143b98ddeb83a --- peering/peering_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/peering/peering_test.go b/peering/peering_test.go index 1f21b7816..3bfdd9d94 100644 --- a/peering/peering_test.go +++ b/peering/peering_test.go @@ -90,7 +90,7 @@ func TestPeeringService(t *testing.T) { // All conns to peer should eventually close. for _, c := range conns { require.Eventually(t, func() bool { - s, err := c.NewStream() + s, err := c.NewStream(context.Background()) if s != nil { _ = s.Reset() } From 7df505c4d1307455e05c4525caffb64c85815db0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Sat, 6 Feb 2021 17:58:18 +0000 Subject: [PATCH 31/63] peering: add logs before many-second waits This test takes a full minute to run, and I was honestly thinking my run of "go test -v" had simply hung, as I saw no output and no apparent resource usage. The least we can do is print a few log messages before the potentially long waits, to hint that we're still making progress. Each of these "Eventually" and "Never" calls ends up blocking the test for a few seconds at a time. This commit was moved from ipfs/kubo@5e0c8bbf28e830c8a81bc81eee3633c0a176c30d --- peering/peering_test.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/peering/peering_test.go b/peering/peering_test.go index 3bfdd9d94..a6ce1332b 100644 --- a/peering/peering_test.go +++ b/peering/peering_test.go @@ -57,11 +57,13 @@ func TestPeeringService(t *testing.T) { require.NoError(t, ps1.Start()) // We should eventually connect. + t.Logf("waiting for h1 to connect to h2") require.Eventually(t, func() bool { return h1.Network().Connectedness(h2.ID()) == network.Connected }, 30*time.Second, 10*time.Millisecond) - // Now explicitly connect to p3. + // Now explicitly connect to h3. + t.Logf("waiting for h1's connection to h3 to work") require.NoError(t, h1.Connect(ctx, peer.AddrInfo{ID: h3.ID(), Addrs: h3.Addrs()})) require.Eventually(t, func() bool { return h1.Network().Connectedness(h2.ID()) == network.Connected @@ -72,7 +74,8 @@ func TestPeeringService(t *testing.T) { // force a disconnect h1.ConnManager().TrimOpenConns(ctx) - // Should disconnect from p3. + // Should disconnect from h3. + t.Logf("waiting for h1's connection to h3 to disconnect") require.Eventually(t, func() bool { return h1.Network().Connectedness(h3.ID()) != network.Connected }, 5*time.Second, 10*time.Millisecond) @@ -88,6 +91,7 @@ func TestPeeringService(t *testing.T) { h2.ConnManager().TrimOpenConns(ctx) // All conns to peer should eventually close. + t.Logf("waiting for all connections to close") for _, c := range conns { require.Eventually(t, func() bool { s, err := c.NewStream(context.Background()) @@ -110,11 +114,13 @@ func TestPeeringService(t *testing.T) { h1.ConnManager().TrimOpenConns(ctx) // Should disconnect + t.Logf("waiting for h1 to disconnect from h2") require.Eventually(t, func() bool { return h1.Network().Connectedness(h2.ID()) != network.Connected }, 5*time.Second, 10*time.Millisecond) // Should never reconnect. + t.Logf("ensuring h1 is not connected to h2 again") require.Never(t, func() bool { return h1.Network().Connectedness(h2.ID()) == network.Connected }, 20*time.Second, 1*time.Second) @@ -122,6 +128,7 @@ func TestPeeringService(t *testing.T) { // Until added back ps1.AddPeer(peer.AddrInfo{ID: h2.ID(), Addrs: h2.Addrs()}) ps1.AddPeer(peer.AddrInfo{ID: h3.ID(), Addrs: h3.Addrs()}) + t.Logf("wait for h1 to connect to h2 and h3 again") require.Eventually(t, func() bool { return h1.Network().Connectedness(h2.ID()) == network.Connected }, 30*time.Second, 1*time.Second) From 772fe893485e576950c118ec36d64c93f2dea5a8 Mon Sep 17 00:00:00 2001 From: Takashi Matsuda Date: Sat, 15 May 2021 11:34:54 +0900 Subject: [PATCH 32/63] fix: the test of peering.PeeringService This commit fixes the issue that the test of peering.PeeringService must check the connection of h3 but did not. This line seemed to be unintended. This commit was moved from ipfs/kubo@b3a6de8c6d3fa1649794d7ec9aa2e96adda875bd --- peering/peering_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/peering/peering_test.go b/peering/peering_test.go index a6ce1332b..7ec42efb6 100644 --- a/peering/peering_test.go +++ b/peering/peering_test.go @@ -66,7 +66,7 @@ func TestPeeringService(t *testing.T) { t.Logf("waiting for h1's connection to h3 to work") require.NoError(t, h1.Connect(ctx, peer.AddrInfo{ID: h3.ID(), Addrs: h3.Addrs()})) require.Eventually(t, func() bool { - return h1.Network().Connectedness(h2.ID()) == network.Connected + return h1.Network().Connectedness(h3.ID()) == network.Connected }, 30*time.Second, 100*time.Millisecond) require.Len(t, h1.Network().Peers(), 3) From cf090e5c7aa22186eea2cd0c99844acff02d055f Mon Sep 17 00:00:00 2001 From: Takashi Matsuda Date: Wed, 15 Sep 2021 23:46:16 +0900 Subject: [PATCH 33/63] feature: 'ipfs swarm peering' command (#8147) * feat: added swarm peering command supporting add, ls and rm Co-authored-by: Steven Allen This commit was moved from ipfs/kubo@a651045c502435b5dfea0d928dc243a7c02b6b04 --- peering/peering.go | 11 +++++++++++ peering/peering_test.go | 6 ++++++ 2 files changed, 17 insertions(+) diff --git a/peering/peering.go b/peering/peering.go index ed0b43226..3146769a0 100644 --- a/peering/peering.go +++ b/peering/peering.go @@ -242,6 +242,17 @@ func (ps *PeeringService) AddPeer(info peer.AddrInfo) { } } +// ListPeers lists peers in the peering service. +func (ps *PeeringService) ListPeers() []peer.AddrInfo { + out := make([]peer.AddrInfo, 0, len(ps.peers)) + for id, addrs := range ps.peers { + ai := peer.AddrInfo{ID: id} + ai.Addrs = append(ai.Addrs, addrs.addrs...) + out = append(out, ai) + } + return out +} + // RemovePeer removes a peer from the peering service. This function may be // safely called at any time: before the service is started, while running, or // after it stops. diff --git a/peering/peering_test.go b/peering/peering_test.go index 7ec42efb6..cf91e637c 100644 --- a/peering/peering_test.go +++ b/peering/peering_test.go @@ -39,6 +39,7 @@ func TestPeeringService(t *testing.T) { // peer 1 -> 2 ps1.AddPeer(peer.AddrInfo{ID: h2.ID(), Addrs: h2.Addrs()}) + require.Contains(t, ps1.ListPeers(), peer.AddrInfo{ID: h2.ID(), Addrs: h2.Addrs()}) // We haven't started so we shouldn't have any peers. require.Never(t, func() bool { @@ -109,6 +110,7 @@ func TestPeeringService(t *testing.T) { // Unprotect 2 from 1. ps1.RemovePeer(h2.ID()) + require.NotContains(t, ps1.ListPeers(), peer.AddrInfo{ID: h2.ID(), Addrs: h2.Addrs()}) // Trim connections. h1.ConnManager().TrimOpenConns(ctx) @@ -127,7 +129,9 @@ func TestPeeringService(t *testing.T) { // Until added back ps1.AddPeer(peer.AddrInfo{ID: h2.ID(), Addrs: h2.Addrs()}) + require.Contains(t, ps1.ListPeers(), peer.AddrInfo{ID: h2.ID(), Addrs: h2.Addrs()}) ps1.AddPeer(peer.AddrInfo{ID: h3.ID(), Addrs: h3.Addrs()}) + require.Contains(t, ps1.ListPeers(), peer.AddrInfo{ID: h3.ID(), Addrs: h3.Addrs()}) t.Logf("wait for h1 to connect to h2 and h3 again") require.Eventually(t, func() bool { return h1.Network().Connectedness(h2.ID()) == network.Connected @@ -142,7 +146,9 @@ func TestPeeringService(t *testing.T) { // Adding and removing should work after stopping. ps1.AddPeer(peer.AddrInfo{ID: h4.ID(), Addrs: h4.Addrs()}) + require.Contains(t, ps1.ListPeers(), peer.AddrInfo{ID: h4.ID(), Addrs: h4.Addrs()}) ps1.RemovePeer(h2.ID()) + require.NotContains(t, ps1.ListPeers(), peer.AddrInfo{ID: h2.ID(), Addrs: h2.Addrs()}) } func TestNextBackoff(t *testing.T) { From c632397b89b37107bfded2386dfc89b4dda7640a Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 15 Sep 2021 18:59:08 +0200 Subject: [PATCH 34/63] fix: take the lock while listing peers This commit was moved from ipfs/kubo@92854db7aed4424fad117ceb4e13f64a80ff348b --- peering/peering.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/peering/peering.go b/peering/peering.go index 3146769a0..dbf955cde 100644 --- a/peering/peering.go +++ b/peering/peering.go @@ -244,6 +244,9 @@ func (ps *PeeringService) AddPeer(info peer.AddrInfo) { // ListPeers lists peers in the peering service. func (ps *PeeringService) ListPeers() []peer.AddrInfo { + ps.mu.RLock() + defer ps.mu.RUnlock() + out := make([]peer.AddrInfo, 0, len(ps.peers)) for id, addrs := range ps.peers { ai := peer.AddrInfo{ID: id} From 87d836ed36a581f88acd95380a38ee7888dd672e Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Mon, 29 Nov 2021 13:58:05 -0500 Subject: [PATCH 35/63] feat: go-libp2p 0.16, UnixFS autosharding and go-datastore with contexts (#8563) * plumb through go-datastore context changes * update go-libp2p to v0.16.0 * use LIBP2P_TCP_REUSEPORT instead of IPFS_REUSEPORT * use relay config * making deprecation notice match the go-ipfs-config key * docs(config): circuit relay v2 * docs(config): fix links and headers * feat(config): Internal.Libp2pForceReachability This switches to config that supports setting and reading Internal.Libp2pForceReachability OptionalString flag * use configuration option for static relays * chore: go-ipfs-config v0.18.0 https://github.com/ipfs/go-ipfs-config/releases/tag/v0.18.0 * feat: circuit v1 migration prompt when Swarm.EnableRelayHop is set (#8559) * exit when Swarm.EnableRelayHop is set * docs: Experimental.ShardingEnabled migration This ensures existing users of global sharding experiment get notified that the flag no longer works + that autosharding happens automatically. For people who NEED to keep the old behavior (eg. have no time to migrate today) there is a note about restoring it with `UnixFSShardingSizeThreshold`. * chore: add dag-jose code to the cid command output * add support for setting automatic unixfs sharding threshold from the config * test: have tests use low cutoff for sharding to mimic old behavior * test: change error message to match the current error * test: Add automatic sharding/unsharding tests (#8547) * test: refactored naming in the sharding sharness tests to make more sense * ci: set interop test executor to convenience image for Go1.16 + Node * ci: use interop master Co-authored-by: Marcin Rataj Co-authored-by: Marten Seemann Co-authored-by: Marcin Rataj Co-authored-by: Gus Eggert Co-authored-by: Lucas Molas This commit was moved from ipfs/kubo@52c177ced94a1dca6f2a440ba9f25a184ff75ddb --- peering/peering_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/peering/peering_test.go b/peering/peering_test.go index cf91e637c..27c9b7175 100644 --- a/peering/peering_test.go +++ b/peering/peering_test.go @@ -16,7 +16,6 @@ import ( func newNode(ctx context.Context, t *testing.T) host.Host { h, err := libp2p.New( - ctx, libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"), // We'd like to set the connection manager low water to 0, but // that would disable the connection manager. From a6fdacb8cd529a32556fae47b5371a454e1adce8 Mon Sep 17 00:00:00 2001 From: Marten Seemann Date: Fri, 8 Apr 2022 02:06:35 +0100 Subject: [PATCH 36/63] feat: opt-in Swarm.ResourceMgr (go-libp2p v0.18) (#8680) * update go-libp2p to v0.18.0 * initialize the resource manager * add resource manager stats/limit commands * load limit file when building resource manager * log absent limit file * write rcmgr to file when IPFS_DEBUG_RCMGR is set * fix: mark swarm limit|stats as experimental * feat(cfg): opt-in Swarm.ResourceMgr This ensures we can safely test the resource manager without impacting default behavior. - Resource manager is disabled by default - Default for Swarm.ResourceMgr.Enabled is false for now - Swarm.ResourceMgr.Limits allows user to tweak limits per specific scope in a way that is persisted across restarts - 'ipfs swarm limit system' outputs human-readable json - 'ipfs swarm limit system new-limits.json' sets new runtime limits (but does not change Swarm.ResourceMgr.Limits in the config) Conventions to make libp2p devs life easier: - 'IPFS_RCMGR=1 ipfs daemon' overrides the config and enables resource manager - 'limit.json' overrides implicit defaults from libp2p (if present) * docs(config): small tweaks * fix: skip libp2p.ResourceManager if disabled This ensures 'ipfs swarm limit|stats' work only when enabled. * fix: use NullResourceManager when disabled This reverts commit b19f7c9eca4cee4187f8cba3389dc2c930258512. after clarification feedback from https://github.com/ipfs/go-ipfs/pull/8680#discussion_r841680182 * style: rename IPFS_RCMGR to LIBP2P_RCMGR preexisting libp2p toggles use LIBP2P_ prefix * test: Swarm.ResourceMgr * fix: location of opt-in limit.json and rcmgr.json.gz Places these files inside of IPFS_PATH * Update docs/config.md * feat: expose rcmgr metrics when enabled (#8785) * add metrics for the resource manager * export protocol and service name in Prometheus metrics * fix: expose rcmgr metrics only when enabled Co-authored-by: Marcin Rataj * refactor: rcmgr_metrics.go * refactor: rcmgr_defaults.go This file defines implicit limit defaults used when Swarm.ResourceMgr.Enabled We keep vendored copy to ensure go-ipfs is not impacted when go-libp2p decides to change defaults in any of the future releases. * refactor: adjustedDefaultLimits Cleans up the way we initialize defaults and adds a fix for case when connection manager runs with high limits. It also hides `Swarm.ResourceMgr.Limits` until we have a better understanding what syntax makes sense. * chore: cleanup after a review * fix: restore go-ipld-prime v0.14.2 * fix: restore go-ds-flatfs v0.5.1 Co-authored-by: Lucas Molas Co-authored-by: Marcin Rataj This commit was moved from ipfs/kubo@514411bedbd7d22d277d31f13d8ad312224c3ad4 --- peering/peering_test.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/peering/peering_test.go b/peering/peering_test.go index 27c9b7175..09a54f2ce 100644 --- a/peering/peering_test.go +++ b/peering/peering_test.go @@ -6,20 +6,22 @@ import ( "time" "github.com/libp2p/go-libp2p" - connmgr "github.com/libp2p/go-libp2p-connmgr" "github.com/libp2p/go-libp2p-core/host" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p/p2p/net/connmgr" "github.com/stretchr/testify/require" ) -func newNode(ctx context.Context, t *testing.T) host.Host { +func newNode(t *testing.T) host.Host { + cm, err := connmgr.NewConnManager(1, 100, connmgr.WithGracePeriod(0)) + require.NoError(t, err) h, err := libp2p.New( libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"), // We'd like to set the connection manager low water to 0, but // that would disable the connection manager. - libp2p.ConnectionManager(connmgr.NewConnManager(1, 100, 0)), + libp2p.ConnectionManager(cm), ) require.NoError(t, err) return h @@ -29,12 +31,12 @@ func TestPeeringService(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - h1 := newNode(ctx, t) + h1 := newNode(t) ps1 := NewPeeringService(h1) - h2 := newNode(ctx, t) - h3 := newNode(ctx, t) - h4 := newNode(ctx, t) + h2 := newNode(t) + h3 := newNode(t) + h4 := newNode(t) // peer 1 -> 2 ps1.AddPeer(peer.AddrInfo{ID: h2.ID(), Addrs: h2.Addrs()}) From ab240dda9f1eb6ce98f9fb77a7e00c02293f4b81 Mon Sep 17 00:00:00 2001 From: Alvin Reyes Date: Sat, 11 Jun 2022 03:34:10 -0400 Subject: [PATCH 37/63] feat: add a public function on peering to get the state PR #9030 This commit was moved from ipfs/kubo@04651e395d6e7923f592bb2c04db872e00844ad6 --- peering/peering.go | 48 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/peering/peering.go b/peering/peering.go index dbf955cde..53b79bf75 100644 --- a/peering/peering.go +++ b/peering/peering.go @@ -4,6 +4,7 @@ import ( "context" "errors" "math/rand" + "strconv" "sync" "time" @@ -32,12 +33,25 @@ const ( var logger = log.Logger("peering") -type state int +type State uint + +func (s State) String() string { + switch s { + case StateInit: + return "init" + case StateRunning: + return "running" + case StateStopped: + return "stopped" + default: + return "unkown peering state: " + strconv.FormatUint(uint64(s), 10) + } +} const ( - stateInit state = iota - stateRunning - stateStopped + StateInit State = iota + StateRunning + StateStopped ) // peerHandler keeps track of all state related to a specific "peering" peer. @@ -155,7 +169,7 @@ type PeeringService struct { mu sync.RWMutex peers map[peer.ID]*peerHandler - state state + state State } // NewPeeringService constructs a new peering service. Peers can be added and @@ -172,35 +186,41 @@ func (ps *PeeringService) Start() error { defer ps.mu.Unlock() switch ps.state { - case stateInit: + case StateInit: logger.Infow("starting") - case stateRunning: + case StateRunning: return nil - case stateStopped: + case StateStopped: return errors.New("already stopped") } ps.host.Network().Notify((*netNotifee)(ps)) - ps.state = stateRunning + ps.state = StateRunning for _, handler := range ps.peers { go handler.startIfDisconnected() } return nil } +// GetState get the State of the PeeringService +func (ps *PeeringService) GetState() State { + ps.mu.RLock() + defer ps.mu.RUnlock() + return ps.state +} + // Stop stops the peering service. func (ps *PeeringService) Stop() error { ps.host.Network().StopNotify((*netNotifee)(ps)) - ps.mu.Lock() defer ps.mu.Unlock() switch ps.state { - case stateInit, stateRunning: + case StateInit, StateRunning: logger.Infow("stopping") for _, handler := range ps.peers { handler.stop() } - ps.state = stateStopped + ps.state = StateStopped } return nil } @@ -231,9 +251,9 @@ func (ps *PeeringService) AddPeer(info peer.AddrInfo) { handler.ctx, handler.cancel = context.WithCancel(context.Background()) ps.peers[info.ID] = handler switch ps.state { - case stateRunning: + case StateRunning: go handler.startIfDisconnected() - case stateStopped: + case StateStopped: // We still construct everything in this state because // it's easier to reason about. But we should still free // resources. From 1e06a3f0f6d5f7fb72b5c9121d4dfed24180a511 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Mon, 29 Aug 2022 13:55:00 +0200 Subject: [PATCH 38/63] chore: bump go-libp2p v0.22.0 & go1.18&go1.19 Fixes: #9225 This commit was moved from ipfs/kubo@196887cbe5fbcd41243c1dfb0db681a1cc2914ff --- peering/peering.go | 6 +++--- peering/peering_test.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/peering/peering.go b/peering/peering.go index 53b79bf75..00801626f 100644 --- a/peering/peering.go +++ b/peering/peering.go @@ -9,9 +9,9 @@ import ( "time" "github.com/ipfs/go-log" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" ) diff --git a/peering/peering_test.go b/peering/peering_test.go index 09a54f2ce..de07789c2 100644 --- a/peering/peering_test.go +++ b/peering/peering_test.go @@ -6,9 +6,9 @@ import ( "time" "github.com/libp2p/go-libp2p" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/p2p/net/connmgr" "github.com/stretchr/testify/require" From 0b1f863ee5fa9f5bec1a83ff406a3121506640e4 Mon Sep 17 00:00:00 2001 From: Kay Date: Thu, 17 Aug 2023 15:32:08 +0330 Subject: [PATCH 39/63] style: gofumpt and godot [skip changelog] (#10081) This commit was moved from ipfs/kubo@f12b372af9cc32975ff48397708fac3ec1f9966f --- peering/peering.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/peering/peering.go b/peering/peering.go index 00801626f..291d9491c 100644 --- a/peering/peering.go +++ b/peering/peering.go @@ -201,7 +201,7 @@ func (ps *PeeringService) Start() error { return nil } -// GetState get the State of the PeeringService +// GetState get the State of the PeeringService. func (ps *PeeringService) GetState() State { ps.mu.RLock() defer ps.mu.RUnlock() @@ -306,6 +306,7 @@ func (nn *netNotifee) Connected(_ network.Network, c network.Conn) { go handler.stopIfConnected() } } + func (nn *netNotifee) Disconnected(_ network.Network, c network.Conn) { ps := (*PeeringService)(nn) From 935391d207ea6298f31dce58aa48e3c914ff96ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alejandro=20Criado-P=C3=A9rez?= Date: Fri, 22 Sep 2023 13:08:26 +0200 Subject: [PATCH 40/63] docs: fix typos This commit was moved from ipfs/kubo@cc3c224c6278c4c62b8d732fefc54cf7246308d2 --- peering/peering.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/peering/peering.go b/peering/peering.go index 291d9491c..34647d63c 100644 --- a/peering/peering.go +++ b/peering/peering.go @@ -44,7 +44,7 @@ func (s State) String() string { case StateStopped: return "stopped" default: - return "unkown peering state: " + strconv.FormatUint(uint64(s), 10) + return "unknown peering state: " + strconv.FormatUint(uint64(s), 10) } } From 96679580181f5d8bbece9e4ec3f555db784018cd Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 28 Sep 2023 13:46:40 -0700 Subject: [PATCH 41/63] peering: migrate from Kubo --- CHANGELOG.md | 2 ++ go.mod | 2 +- peering/peering.go | 3 +-- peering/peering_test.go | 4 ++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 71971240a..8494bdfde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ The following emojis are used to highlight certain changes: * The gateway now sets a `Cache-Control` header for requests under the `/ipns/` namespace if the TTL for the corresponding IPNS Records or DNSLink entities is known. * `boxo/bitswap/client`: * A new `WithoutDuplicatedBlockStats()` option can be used with `bitswap.New` and `bsclient.New`. This disable accounting for duplicated blocks, which requires a `blockstore.Has()` lookup for every received block and thus, can impact performance. +* ✹ Migrated repositories into Boxo + * [`github.com/ipfs/kubo/peering`](https://pkg.go.dev/github.com/ipfs/kubo/peering) => [`./peering`](./peering) ### Changed diff --git a/go.mod b/go.mod index 2f4d53569..cb0b34af4 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/ipfs/go-ipld-cbor v0.0.6 github.com/ipfs/go-ipld-format v0.5.0 github.com/ipfs/go-ipld-legacy v0.2.1 + github.com/ipfs/go-log v1.0.5 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-metrics-interface v0.0.1 github.com/ipfs/go-peertaskqueue v0.8.1 @@ -107,7 +108,6 @@ require ( github.com/huin/goupnp v1.2.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.2 // indirect - github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-unixfs v0.4.5 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect diff --git a/peering/peering.go b/peering/peering.go index 34647d63c..cc549c369 100644 --- a/peering/peering.go +++ b/peering/peering.go @@ -209,7 +209,7 @@ func (ps *PeeringService) GetState() State { } // Stop stops the peering service. -func (ps *PeeringService) Stop() error { +func (ps *PeeringService) Stop() { ps.host.Network().StopNotify((*netNotifee)(ps)) ps.mu.Lock() defer ps.mu.Unlock() @@ -222,7 +222,6 @@ func (ps *PeeringService) Stop() error { } ps.state = StateStopped } - return nil } // AddPeer adds a peer to the peering service. This function may be safely diff --git a/peering/peering_test.go b/peering/peering_test.go index de07789c2..3d146e3e3 100644 --- a/peering/peering_test.go +++ b/peering/peering_test.go @@ -142,8 +142,8 @@ func TestPeeringService(t *testing.T) { }, 30*time.Second, 1*time.Second) // Should be able to repeatedly stop. - require.NoError(t, ps1.Stop()) - require.NoError(t, ps1.Stop()) + ps1.Stop() + ps1.Stop() // Adding and removing should work after stopping. ps1.AddPeer(peer.AddrInfo{ID: h4.ID(), Addrs: h4.Addrs()}) From 15de4ac8aed6d913ac9f002d63cec809e3ef569e Mon Sep 17 00:00:00 2001 From: Jorropo Date: Tue, 24 Oct 2023 19:22:11 +0200 Subject: [PATCH 42/63] docs: improve changelog for peering --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8494bdfde..c17a9c46e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ The following emojis are used to highlight certain changes: * A new `WithoutDuplicatedBlockStats()` option can be used with `bitswap.New` and `bsclient.New`. This disable accounting for duplicated blocks, which requires a `blockstore.Has()` lookup for every received block and thus, can impact performance. * ✹ Migrated repositories into Boxo * [`github.com/ipfs/kubo/peering`](https://pkg.go.dev/github.com/ipfs/kubo/peering) => [`./peering`](./peering) + A service which establish, overwatch and maintain long lived connections. ### Changed From 07d149341188a26a98547fe48e70bd6102e509df Mon Sep 17 00:00:00 2001 From: Jorropo Date: Tue, 24 Oct 2023 19:24:23 +0200 Subject: [PATCH 43/63] chore: remove outdated comment in peering --- peering/peering.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/peering/peering.go b/peering/peering.go index cc549c369..d1c54ead3 100644 --- a/peering/peering.go +++ b/peering/peering.go @@ -15,9 +15,6 @@ import ( "github.com/multiformats/go-multiaddr" ) -// Seed the random number generator. -// -// We don't need good randomness, but we do need randomness. const ( // maxBackoff is the maximum time between reconnect attempts. maxBackoff = 10 * time.Minute From 182e86f5bb9b89c0b17f23c7f5739463ffbdf5bb Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Sat, 28 Oct 2023 03:52:56 +0200 Subject: [PATCH 44/63] fix(gw): blocked content produces http error 410 this is the minimum we need right now to make content blocking from https://github.com/ipfs/kubo/pull/10161 return HTTP 410 on rule match --- gateway/errors.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/gateway/errors.go b/gateway/errors.go index 08b532e82..4487f0c2e 100644 --- a/gateway/errors.go +++ b/gateway/errors.go @@ -149,6 +149,8 @@ func webError(w http.ResponseWriter, r *http.Request, c *Config, err error, defa code = http.StatusBadRequest case isErrNotFound(err): code = http.StatusNotFound + case isErrContentBlocked(err): + code = http.StatusGone case errors.Is(err, context.DeadlineExceeded): code = http.StatusGatewayTimeout } @@ -202,3 +204,10 @@ func isErrNotFound(err error) bool { } } } + +// isErrContentBlocked returns true for content filtering system errors +func isErrContentBlocked(err error) bool { + // TODO: we match error message to avoid pulling nopfs as a dependency + // Ref. https://github.com/ipfs-shipyard/nopfs/blob/cde3b5ba964c13e977f4a95f3bd8ca7d7710fbda/status.go#L87-L89 + return strings.Contains(err.Error(), "blocked and cannot be provided") +} From b0a265b5cdcc742c7606b03bd7910380932e3307 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Tue, 31 Oct 2023 11:45:28 +0100 Subject: [PATCH 45/63] fix(gw): duplicate blocks and range etags (#486) Co-authored-by: Henrique Dias --- gateway/gateway.go | 12 +++++----- gateway/handler_car.go | 45 +++++++++++++++++++++++-------------- gateway/handler_car_test.go | 24 ++++++++++++++++++++ 3 files changed, 59 insertions(+), 22 deletions(-) diff --git a/gateway/gateway.go b/gateway/gateway.go index 06d236d1f..28cf5d295 100644 --- a/gateway/gateway.go +++ b/gateway/gateway.go @@ -174,7 +174,7 @@ const ( ) // DuplicateBlocksPolicy represents the content type parameter 'dups' (IPIP-412) -type DuplicateBlocksPolicy int +type DuplicateBlocksPolicy uint8 const ( DuplicateBlocksUnspecified DuplicateBlocksPolicy = iota // 0 - implicit default @@ -183,14 +183,16 @@ const ( ) // NewDuplicateBlocksPolicy returns DuplicateBlocksPolicy based on the content type parameter 'dups' (IPIP-412) -func NewDuplicateBlocksPolicy(dupsValue string) DuplicateBlocksPolicy { +func NewDuplicateBlocksPolicy(dupsValue string) (DuplicateBlocksPolicy, error) { switch dupsValue { case "y": - return DuplicateBlocksIncluded + return DuplicateBlocksIncluded, nil case "n": - return DuplicateBlocksExcluded + return DuplicateBlocksExcluded, nil + case "": + return DuplicateBlocksUnspecified, nil } - return DuplicateBlocksUnspecified + return 0, fmt.Errorf("unsupported application/vnd.ipld.car content type dups parameter: %q", dupsValue) } func (d DuplicateBlocksPolicy) Bool() bool { diff --git a/gateway/handler_car.go b/gateway/handler_car.go index 1c670b045..21b90108c 100644 --- a/gateway/handler_car.go +++ b/gateway/handler_car.go @@ -2,6 +2,7 @@ package gateway import ( "context" + "encoding/binary" "fmt" "io" "net/http" @@ -166,20 +167,18 @@ func buildCarParams(r *http.Request, contentTypeParams map[string]string) (CarPa } // optional dups from IPIP-412 - if dups := NewDuplicateBlocksPolicy(contentTypeParams["dups"]); dups != DuplicateBlocksUnspecified { - switch dups { - case DuplicateBlocksExcluded, DuplicateBlocksIncluded: - params.Duplicates = dups - default: - return CarParams{}, fmt.Errorf("unsupported application/vnd.ipld.car content type dups parameter: %q", dups) - } - } else { + dups, err := NewDuplicateBlocksPolicy(contentTypeParams["dups"]) + if err != nil { + return CarParams{}, err + } + if dups == DuplicateBlocksUnspecified { // when duplicate block preference is not specified, we set it to // false, as this has always been the default behavior, we should // not break legacy clients, and responses to requests made via ?format=car // should benefit from block deduplication - params.Duplicates = DuplicateBlocksExcluded + dups = DuplicateBlocksExcluded } + params.Duplicates = dups return params, nil } @@ -226,31 +225,43 @@ func getCarRootCidAndLastSegment(imPath path.ImmutablePath) (cid.Cid, string, er } func getCarEtag(imPath path.ImmutablePath, params CarParams, rootCid cid.Cid) string { - data := imPath.String() + h := xxhash.New() + h.WriteString(imPath.String()) + // be careful with hashes here, we need boundaries and per entry salt, we don't want a request that has: + // - scope = dfs + // and: + // - order = dfs + // to result in the same hash because if we just do hash(scope + order) they would both yield hash("dfs"). if params.Scope != DagScopeAll { - data += string(params.Scope) + h.WriteString("\x00scope=") + h.WriteString(string(params.Scope)) } // 'order' from IPIP-412 impact Etag only if set to something else // than DFS (which is the implicit default) if params.Order != DagOrderDFS { - data += string(params.Order) + h.WriteString("\x00order=") + h.WriteString(string(params.Order)) } // 'dups' from IPIP-412 impact Etag only if 'y' - if dups := params.Duplicates.String(); dups == "y" { - data += dups + if dups := params.Duplicates; dups == DuplicateBlocksIncluded { + h.WriteString("\x00dups=y") } if params.Range != nil { if params.Range.From != 0 || params.Range.To != nil { - data += strconv.FormatInt(params.Range.From, 10) + h.WriteString("\x00range=") + var b [8]byte + binary.LittleEndian.PutUint64(b[:], uint64(params.Range.From)) + h.Write(b[:]) if params.Range.To != nil { - data += strconv.FormatInt(*params.Range.To, 10) + binary.LittleEndian.PutUint64(b[:], uint64(*params.Range.To)) + h.Write(b[:]) } } } - suffix := strconv.FormatUint(xxhash.Sum64([]byte(data)), 32) + suffix := strconv.FormatUint(h.Sum64(), 32) return `W/"` + rootCid.String() + ".car." + suffix + `"` } diff --git a/gateway/handler_car_test.go b/gateway/handler_car_test.go index efbad2b22..da2d16255 100644 --- a/gateway/handler_car_test.go +++ b/gateway/handler_car_test.go @@ -110,6 +110,30 @@ func TestCarParams(t *testing.T) { require.Equal(t, test.expectedDuplicates.String(), params.Duplicates.String()) } }) + + t.Run("buildCarParams from Accept header: order and dups parsing (invalid)", func(t *testing.T) { + t.Parallel() + + // below ensure the implicit default (DFS and no duplicates) is correctly inferred + // from the value read from Accept header + tests := []string{ + "application/vnd.ipld.car; dups=invalid", + "application/vnd.ipld.car; order=invalid", + "application/vnd.ipld.car; order=dfs; dups=invalid", + "application/vnd.ipld.car; order=invalid; dups=y", + } + for _, test := range tests { + r := mustNewRequest(t, http.MethodGet, "http://example.com/", nil) + r.Header.Set("Accept", test) + + mediaType, formatParams, err := customResponseFormat(r) + assert.NoError(t, err) + assert.Equal(t, carResponseFormat, mediaType) + + _, err = buildCarParams(r, formatParams) + assert.ErrorContains(t, err, "unsupported application/vnd.ipld.car content type") + } + }) } func TestContentTypeFromCarParams(t *testing.T) { From 46c47bcaa48021c80c5526cc236179d426ba9cc8 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 31 Oct 2023 15:55:48 +0100 Subject: [PATCH 46/63] docs: add some more projects in readme (#499) --- README.md | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index eec46e0e3..bd824f512 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,7 @@ Boxo powers [Kubo](https://github.com/ipfs/kubo), which is [the most popular IPF so its code has been battle-tested on the IPFS network for years, and is well-understood by the community. ### Motivation + **TL;DR** The goal of this repo is to help people build things. Previously users struggled to find existing useful code or to figure out how to use what they did find. We observed many running Kubo and using its HTTP RPC API. This repo aims to do better. We're taking the libraries that many were already effectively relying on in production and making them more easily discoverable and usable. The maintainers primarily aim to help people trying to build with IPFS in Go that were previously either giving up or relying on the [Kubo HTTP RPC API](https://docs.ipfs.tech/reference/kubo/rpc/). Some of these people will end up being better served by IPFS tooling in other languages (e.g., Javascript, Rust, Java, Python), but for those who are either looking to write in Go or to leverage the set of IPFS tooling we already have in Go we’d like to make their lives easier. @@ -73,6 +74,7 @@ Boxo is not exhaustive nor comprehensive--there are plenty of useful IPFS protoc More details can also be found in the [Rationale FAQ](./docs/FAQ.md#rationale-faq). ## Scope + ### What kind of components does Boxo have? Boxo includes high-quality components useful for interacting with IPFS protocols, public and private IPFS networks, and content-addressed data, such as: @@ -86,20 +88,23 @@ Boxo includes high-quality components useful for interacting with IPFS protocols Boxo aims to provide a cohesive interface into these components. Note that not all of the underlying components necessarily reside in this respository. ### Does Boxo == IPFS? -No. This repo houses some IPFS functionality written in Go that has been useful in practice, and is maintained by a group that has long term commitments to the IPFS project -### Is everything related to IPFS in the Go ecosystem in this repo? +No. This repo houses some IPFS functionality written in Go that has been useful in practice, and is maintained by a group that has long term commitments to the IPFS project -No. Not everything related to IPFS is intended to be in Boxo. View it as a starter toolbox (potentially among multiple). If you’d like to build an IPFS implementation with Go, here are some tools you might want that are maintained by a group that has long term commitments to the IPFS project. There are certainly repos that others maintain that aren't included here (e.g., ipfs/go-car) which are still useful to IPFS implementations. It's expected and fine for new IPFS functionality to be developed that won't be part of Boxo. +### Is everything related to IPFS in the Go ecosystem in this repo? +No. Not everything related to IPFS is intended to be in Boxo. View it as a starter toolbox (potentially among multiple). If you’d like to build an IPFS implementation with Go, here are some tools you might want that are maintained by a group that has long term commitments to the IPFS project. There are certainly repos that others maintain that aren't included here (e.g., ipfs/go-car) which are still useful to IPFS implementations. It's expected and fine for new IPFS functionality to be developed that won't be part of Boxo. ## Consuming + ### Getting started + See [examples](./examples/README.md). If you are migrating to Boxo, see [Migrating to Boxo](#migrating-to-boxo). ### Migrating to Boxo + Many Go modules under github.com/ipfs have moved here. Boxo provides a tool to ease this migration, which does most of the work for you: * `cd` into the root directory of your module (where the `go.mod` file is) @@ -116,10 +121,13 @@ We recommend upgrading to v0.8.0 first, and _then_ upgrading to the latest Boxo If you encounter any challenges, please [open an issue](https://github.com/ipfs/boxo/issues/new/choose) and Boxo maintainers will help you. ### Deprecations & Breaking Changes + See [RELEASE.md](./RELEASE.md). ## Development + ### Should I add my IPFS component to Boxo? + We happily accept external contributions! However, Boxo maintains a high quality bar, so code accepted into Boxo must meet some minimum maintenance criteria: * Actively maintained @@ -137,6 +145,7 @@ We happily accept external contributions! However, Boxo maintains a high quality If you have some experimental component that you think would benefit the IPFS community, we suggest you build the component in your own repository until it's clear that there's community demand for it, and then open an issue/PR in this repository to discuss including it in Boxo. ### Release Process + See [RELEASE.md](./RELEASE.md). ### Why is the code coverage so bad? @@ -144,30 +153,38 @@ See [RELEASE.md](./RELEASE.md). The code coverage of this repo is not currently representative of the actual test coverage of this code. Much of the code in this repo is currently covered by integration tests in [Kubo](https://github.com/ipfs/kubo). We are in the process of moving those tests here, and as that continues the code coverage will significantly increase. ## General + ### Help If you have questions, feel free to open an issue. You can also find the Boxo maintainers in [Filecoin Slack](https://filecoin.io/slack/) at #Boxo-maintainers. (If you would like to engage via IPFS Discord or ipfs.io Matrix, please drop into the #ipfs-implementers channel/room or file an issue, and we'll get bridging from #Boxo-maintainers to these other chat platforms.) ### What is the response time for issues or PRs filed? -TODO: fill this in. New issues and PRs to this repo are usually looked at on a weekly basis as part of [Kubo triage](https://pl-strflt.notion.site/Kubo-Issue-Triage-Notes-7d4983e8cf294e07b3cc51b0c60ede9a). + +New issues and PRs to this repo are usually looked at on a weekly basis as part of [Kubo triage](https://pl-strflt.notion.site/Kubo-Issue-Triage-Notes-7d4983e8cf294e07b3cc51b0c60ede9a). However, the response time may vary. ### What are some projects that depend on this project? -The exhaustive list is https://github.com/ipfs/boxo/network/dependents. Some notable projects include: + +The exhaustive list is https://github.com/ipfs/boxo/network/dependents. Some notable projects include: + 1. [Kubo](https://github.com/ipfs/kubo), an IPFS implementation in Go 2. [Lotus](https://github.com/filecoin-project/lotus), a Filecoin implementation in Go -3. [Bifrost Gateway](https://github.com/ipfs/bifrost-gateway), a dedicated IPFS gateway +6. [rainbow](https://github.com/ipfs/rainbow), a specialized IPFS gateway 4. [ipfs-check](https://github.com/ipfs-shipyard/ipfs-check), checks IPFS data availability +5. [someguy](https://github.com/ipfs-shipyard/someguy), a dedicated Delegated Routing V1 server and client +3. [Bifrost Gateway](https://github.com/ipfs/bifrost-gateway), a dedicated IPFS Gateway daemon backed by a remote datastore ### Governance and Access -See [CODEOWNERS](./docs/CODEOWNERS) for the current maintainers list. Governance for graduating additional maintainers hasn't been established. Repo permissions are all managed through [ipfs/github-mgmt](https://github.com/ipfs/github-mgmt). + +See [CODEOWNERS](./docs/CODEOWNERS) for the current maintainers list. Governance for graduating additional maintainers hasn't been established. Repo permissions are all managed through [ipfs/github-mgmt](https://github.com/ipfs/github-mgmt). ### Why is this named "Boxo"? + See https://github.com/ipfs/boxo/issues/215. ### Additional Docs & FAQs + See [the wiki](https://github.com/ipfs/boxo/wiki). ### License [SPDX-License-Identifier: Apache-2.0 OR MIT](LICENSE.md) - From fc8ea49959730abb4076cf89654c004482393153 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 3 Apr 2019 03:44:32 +0200 Subject: [PATCH 47/63] Cleanup core package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Ɓukasz Magiera This commit was moved from ipfs/kubo@d35dac70f0b0b47c7813c6380deb8974179bdb32 --- core/bootstrap/bootstrap.go | 243 +++++++++++++++++++++++++++++++ core/bootstrap/bootstrap_test.go | 56 +++++++ 2 files changed, 299 insertions(+) create mode 100644 core/bootstrap/bootstrap.go create mode 100644 core/bootstrap/bootstrap_test.go diff --git a/core/bootstrap/bootstrap.go b/core/bootstrap/bootstrap.go new file mode 100644 index 000000000..e6b4f826d --- /dev/null +++ b/core/bootstrap/bootstrap.go @@ -0,0 +1,243 @@ +package bootstrap + +import ( + "context" + "errors" + "fmt" + "io" + "math/rand" + "sync" + "time" + + config "github.com/ipfs/go-ipfs-config" + logging "github.com/ipfs/go-log" + "github.com/jbenet/goprocess" + "github.com/jbenet/goprocess/context" + "github.com/jbenet/goprocess/periodic" + "github.com/libp2p/go-libp2p-host" + "github.com/libp2p/go-libp2p-loggables" + "github.com/libp2p/go-libp2p-net" + "github.com/libp2p/go-libp2p-peer" + "github.com/libp2p/go-libp2p-peerstore" + "github.com/libp2p/go-libp2p-routing" + + "github.com/ipfs/go-ipfs/thirdparty/math2" +) + +var log = logging.Logger("bootstrap") + +// ErrNotEnoughBootstrapPeers signals that we do not have enough bootstrap +// peers to bootstrap correctly. +var ErrNotEnoughBootstrapPeers = errors.New("not enough bootstrap peers to bootstrap") + +// BootstrapConfig specifies parameters used in an IpfsNode's network +// bootstrapping process. +type BootstrapConfig struct { + // MinPeerThreshold governs whether to bootstrap more connections. If the + // node has less open connections than this number, it will open connections + // to the bootstrap nodes. From there, the routing system should be able + // to use the connections to the bootstrap nodes to connect to even more + // peers. Routing systems like the IpfsDHT do so in their own Bootstrap + // process, which issues random queries to find more peers. + MinPeerThreshold int + + // Period governs the periodic interval at which the node will + // attempt to bootstrap. The bootstrap process is not very expensive, so + // this threshold can afford to be small (<=30s). + Period time.Duration + + // ConnectionTimeout determines how long to wait for a bootstrap + // connection attempt before cancelling it. + ConnectionTimeout time.Duration + + // BootstrapPeers is a function that returns a set of bootstrap peers + // for the bootstrap process to use. This makes it possible for clients + // to control the peers the process uses at any moment. + BootstrapPeers func() []peerstore.PeerInfo +} + +// DefaultBootstrapConfig specifies default sane parameters for bootstrapping. +var DefaultBootstrapConfig = BootstrapConfig{ + MinPeerThreshold: 4, + Period: 30 * time.Second, + ConnectionTimeout: (30 * time.Second) / 3, // Perod / 3 +} + +func BootstrapConfigWithPeers(pis []peerstore.PeerInfo) BootstrapConfig { + cfg := DefaultBootstrapConfig + cfg.BootstrapPeers = func() []peerstore.PeerInfo { + return pis + } + return cfg +} + +// Bootstrap kicks off IpfsNode bootstrapping. This function will periodically +// check the number of open connections and -- if there are too few -- initiate +// connections to well-known bootstrap peers. It also kicks off subsystem +// bootstrapping (i.e. routing). +func Bootstrap(id peer.ID, host host.Host, rt routing.IpfsRouting, cfg BootstrapConfig) (io.Closer, error) { + + // make a signal to wait for one bootstrap round to complete. + doneWithRound := make(chan struct{}) + + if len(cfg.BootstrapPeers()) == 0 { + // We *need* to bootstrap but we have no bootstrap peers + // configured *at all*, inform the user. + log.Warning("no bootstrap nodes configured: go-ipfs may have difficulty connecting to the network") + } + + // the periodic bootstrap function -- the connection supervisor + periodic := func(worker goprocess.Process) { + ctx := goprocessctx.OnClosingContext(worker) + defer log.EventBegin(ctx, "periodicBootstrap", id).Done() + + if err := bootstrapRound(ctx, host, cfg); err != nil { + log.Event(ctx, "bootstrapError", id, loggables.Error(err)) + log.Debugf("%s bootstrap error: %s", id, err) + } + + <-doneWithRound + } + + // kick off the node's periodic bootstrapping + proc := periodicproc.Tick(cfg.Period, periodic) + proc.Go(periodic) // run one right now. + + // kick off Routing.Bootstrap + if rt != nil { + ctx := goprocessctx.OnClosingContext(proc) + if err := rt.Bootstrap(ctx); err != nil { + proc.Close() + return nil, err + } + } + + doneWithRound <- struct{}{} + close(doneWithRound) // it no longer blocks periodic + return proc, nil +} + +func bootstrapRound(ctx context.Context, host host.Host, cfg BootstrapConfig) error { + + ctx, cancel := context.WithTimeout(ctx, cfg.ConnectionTimeout) + defer cancel() + id := host.ID() + + // get bootstrap peers from config. retrieving them here makes + // sure we remain observant of changes to client configuration. + peers := cfg.BootstrapPeers() + // determine how many bootstrap connections to open + connected := host.Network().Peers() + if len(connected) >= cfg.MinPeerThreshold { + log.Event(ctx, "bootstrapSkip", id) + log.Debugf("%s core bootstrap skipped -- connected to %d (> %d) nodes", + id, len(connected), cfg.MinPeerThreshold) + return nil + } + numToDial := cfg.MinPeerThreshold - len(connected) + + // filter out bootstrap nodes we are already connected to + var notConnected []peerstore.PeerInfo + for _, p := range peers { + if host.Network().Connectedness(p.ID) != net.Connected { + notConnected = append(notConnected, p) + } + } + + // if connected to all bootstrap peer candidates, exit + if len(notConnected) < 1 { + log.Debugf("%s no more bootstrap peers to create %d connections", id, numToDial) + return ErrNotEnoughBootstrapPeers + } + + // connect to a random susbset of bootstrap candidates + randSubset := randomSubsetOfPeers(notConnected, numToDial) + + defer log.EventBegin(ctx, "bootstrapStart", id).Done() + log.Debugf("%s bootstrapping to %d nodes: %s", id, numToDial, randSubset) + return bootstrapConnect(ctx, host, randSubset) +} + +func bootstrapConnect(ctx context.Context, ph host.Host, peers []peerstore.PeerInfo) error { + if len(peers) < 1 { + return ErrNotEnoughBootstrapPeers + } + + errs := make(chan error, len(peers)) + var wg sync.WaitGroup + for _, p := range peers { + + // performed asynchronously because when performed synchronously, if + // one `Connect` call hangs, subsequent calls are more likely to + // fail/abort due to an expiring context. + // Also, performed asynchronously for dial speed. + + wg.Add(1) + go func(p peerstore.PeerInfo) { + defer wg.Done() + defer log.EventBegin(ctx, "bootstrapDial", ph.ID(), p.ID).Done() + log.Debugf("%s bootstrapping to %s", ph.ID(), p.ID) + + ph.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.PermanentAddrTTL) + if err := ph.Connect(ctx, p); err != nil { + log.Event(ctx, "bootstrapDialFailed", p.ID) + log.Debugf("failed to bootstrap with %v: %s", p.ID, err) + errs <- err + return + } + log.Event(ctx, "bootstrapDialSuccess", p.ID) + log.Infof("bootstrapped with %v", p.ID) + }(p) + } + wg.Wait() + + // our failure condition is when no connection attempt succeeded. + // So drain the errs channel, counting the results. + close(errs) + count := 0 + var err error + for err = range errs { + if err != nil { + count++ + } + } + if count == len(peers) { + return fmt.Errorf("failed to bootstrap. %s", err) + } + return nil +} + +func randomSubsetOfPeers(in []peerstore.PeerInfo, max int) []peerstore.PeerInfo { + n := math2.IntMin(max, len(in)) + var out []peerstore.PeerInfo + for _, val := range rand.Perm(len(in)) { + out = append(out, in[val]) + if len(out) >= n { + break + } + } + return out +} + +type Peers []config.BootstrapPeer + +func (bpeers Peers) ToPeerInfos() []peerstore.PeerInfo { + pinfos := make(map[peer.ID]*peerstore.PeerInfo) + for _, bootstrap := range bpeers { + pinfo, ok := pinfos[bootstrap.ID()] + if !ok { + pinfo = new(peerstore.PeerInfo) + pinfos[bootstrap.ID()] = pinfo + pinfo.ID = bootstrap.ID() + } + + pinfo.Addrs = append(pinfo.Addrs, bootstrap.Transport()) + } + + var peers []peerstore.PeerInfo + for _, pinfo := range pinfos { + peers = append(peers, *pinfo) + } + + return peers +} diff --git a/core/bootstrap/bootstrap_test.go b/core/bootstrap/bootstrap_test.go new file mode 100644 index 000000000..0c7799858 --- /dev/null +++ b/core/bootstrap/bootstrap_test.go @@ -0,0 +1,56 @@ +package bootstrap + +import ( + "fmt" + "testing" + + config "github.com/ipfs/go-ipfs-config" + pstore "github.com/libp2p/go-libp2p-peerstore" + testutil "github.com/libp2p/go-testutil" +) + +func TestSubsetWhenMaxIsGreaterThanLengthOfSlice(t *testing.T) { + var ps []pstore.PeerInfo + sizeofSlice := 100 + for i := 0; i < sizeofSlice; i++ { + pid, err := testutil.RandPeerID() + if err != nil { + t.Fatal(err) + } + + ps = append(ps, pstore.PeerInfo{ID: pid}) + } + out := randomSubsetOfPeers(ps, 2*sizeofSlice) + if len(out) != len(ps) { + t.Fail() + } +} + +func TestMultipleAddrsPerPeer(t *testing.T) { + var bsps []config.BootstrapPeer + for i := 0; i < 10; i++ { + pid, err := testutil.RandPeerID() + if err != nil { + t.Fatal(err) + } + + addr := fmt.Sprintf("/ip4/127.0.0.1/tcp/5001/ipfs/%s", pid.Pretty()) + bsp1, err := config.ParseBootstrapPeer(addr) + if err != nil { + t.Fatal(err) + } + + addr = fmt.Sprintf("/ip4/127.0.0.1/udp/5002/utp/ipfs/%s", pid.Pretty()) + bsp2, err := config.ParseBootstrapPeer(addr) + if err != nil { + t.Fatal(err) + } + + bsps = append(bsps, bsp1, bsp2) + } + + pinfos := Peers.ToPeerInfos(bsps) + if len(pinfos) != len(bsps)/2 { + t.Fatal("expected fewer peers") + } +} From d1e92682dc81cc7e87bfe32262e9644849d02c1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 8 Apr 2019 15:36:25 +0200 Subject: [PATCH 48/63] bootstrap: cleanup randomSubsetOfPeers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Ɓukasz Magiera This commit was moved from ipfs/kubo@0e6f8d4cc19ef5bb5d9c47e357a3ffb862b8b067 --- core/bootstrap/bootstrap.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/core/bootstrap/bootstrap.go b/core/bootstrap/bootstrap.go index e6b4f826d..d7c107690 100644 --- a/core/bootstrap/bootstrap.go +++ b/core/bootstrap/bootstrap.go @@ -20,8 +20,6 @@ import ( "github.com/libp2p/go-libp2p-peer" "github.com/libp2p/go-libp2p-peerstore" "github.com/libp2p/go-libp2p-routing" - - "github.com/ipfs/go-ipfs/thirdparty/math2" ) var log = logging.Logger("bootstrap") @@ -208,13 +206,13 @@ func bootstrapConnect(ctx context.Context, ph host.Host, peers []peerstore.PeerI } func randomSubsetOfPeers(in []peerstore.PeerInfo, max int) []peerstore.PeerInfo { - n := math2.IntMin(max, len(in)) - var out []peerstore.PeerInfo - for _, val := range rand.Perm(len(in)) { - out = append(out, in[val]) - if len(out) >= n { - break - } + if max > len(in) { + max = len(in) + } + + out := make([]peerstore.PeerInfo, max) + for i, val := range rand.Perm(len(in))[:max] { + out[i] = in[val] } return out } From 3f93896efef2bcbaec60933deb6756e26f44f077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Kripalani?= Date: Tue, 28 May 2019 17:21:57 +0100 Subject: [PATCH 49/63] migrate to go-libp2p-core. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #6391 License: MIT Signed-off-by: RaĂșl Kripalani This commit was moved from ipfs/kubo@e8c2852179b03a95c8d198895b246b1e3ffaeed8 --- core/bootstrap/bootstrap.go | 38 ++++++++++++++++---------------- core/bootstrap/bootstrap_test.go | 12 +++++----- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/core/bootstrap/bootstrap.go b/core/bootstrap/bootstrap.go index d7c107690..b2cf1a811 100644 --- a/core/bootstrap/bootstrap.go +++ b/core/bootstrap/bootstrap.go @@ -14,12 +14,12 @@ import ( "github.com/jbenet/goprocess" "github.com/jbenet/goprocess/context" "github.com/jbenet/goprocess/periodic" - "github.com/libp2p/go-libp2p-host" + "github.com/libp2p/go-libp2p-core/host" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/peerstore" + "github.com/libp2p/go-libp2p-core/routing" "github.com/libp2p/go-libp2p-loggables" - "github.com/libp2p/go-libp2p-net" - "github.com/libp2p/go-libp2p-peer" - "github.com/libp2p/go-libp2p-peerstore" - "github.com/libp2p/go-libp2p-routing" ) var log = logging.Logger("bootstrap") @@ -51,7 +51,7 @@ type BootstrapConfig struct { // BootstrapPeers is a function that returns a set of bootstrap peers // for the bootstrap process to use. This makes it possible for clients // to control the peers the process uses at any moment. - BootstrapPeers func() []peerstore.PeerInfo + BootstrapPeers func() []peer.AddrInfo } // DefaultBootstrapConfig specifies default sane parameters for bootstrapping. @@ -61,9 +61,9 @@ var DefaultBootstrapConfig = BootstrapConfig{ ConnectionTimeout: (30 * time.Second) / 3, // Perod / 3 } -func BootstrapConfigWithPeers(pis []peerstore.PeerInfo) BootstrapConfig { +func BootstrapConfigWithPeers(pis []peer.AddrInfo) BootstrapConfig { cfg := DefaultBootstrapConfig - cfg.BootstrapPeers = func() []peerstore.PeerInfo { + cfg.BootstrapPeers = func() []peer.AddrInfo { return pis } return cfg @@ -73,7 +73,7 @@ func BootstrapConfigWithPeers(pis []peerstore.PeerInfo) BootstrapConfig { // check the number of open connections and -- if there are too few -- initiate // connections to well-known bootstrap peers. It also kicks off subsystem // bootstrapping (i.e. routing). -func Bootstrap(id peer.ID, host host.Host, rt routing.IpfsRouting, cfg BootstrapConfig) (io.Closer, error) { +func Bootstrap(id peer.ID, host host.Host, rt routing.Routing, cfg BootstrapConfig) (io.Closer, error) { // make a signal to wait for one bootstrap round to complete. doneWithRound := make(chan struct{}) @@ -135,9 +135,9 @@ func bootstrapRound(ctx context.Context, host host.Host, cfg BootstrapConfig) er numToDial := cfg.MinPeerThreshold - len(connected) // filter out bootstrap nodes we are already connected to - var notConnected []peerstore.PeerInfo + var notConnected []peer.AddrInfo for _, p := range peers { - if host.Network().Connectedness(p.ID) != net.Connected { + if host.Network().Connectedness(p.ID) != network.Connected { notConnected = append(notConnected, p) } } @@ -156,7 +156,7 @@ func bootstrapRound(ctx context.Context, host host.Host, cfg BootstrapConfig) er return bootstrapConnect(ctx, host, randSubset) } -func bootstrapConnect(ctx context.Context, ph host.Host, peers []peerstore.PeerInfo) error { +func bootstrapConnect(ctx context.Context, ph host.Host, peers []peer.AddrInfo) error { if len(peers) < 1 { return ErrNotEnoughBootstrapPeers } @@ -171,7 +171,7 @@ func bootstrapConnect(ctx context.Context, ph host.Host, peers []peerstore.PeerI // Also, performed asynchronously for dial speed. wg.Add(1) - go func(p peerstore.PeerInfo) { + go func(p peer.AddrInfo) { defer wg.Done() defer log.EventBegin(ctx, "bootstrapDial", ph.ID(), p.ID).Done() log.Debugf("%s bootstrapping to %s", ph.ID(), p.ID) @@ -205,12 +205,12 @@ func bootstrapConnect(ctx context.Context, ph host.Host, peers []peerstore.PeerI return nil } -func randomSubsetOfPeers(in []peerstore.PeerInfo, max int) []peerstore.PeerInfo { +func randomSubsetOfPeers(in []peer.AddrInfo, max int) []peer.AddrInfo { if max > len(in) { max = len(in) } - out := make([]peerstore.PeerInfo, max) + out := make([]peer.AddrInfo, max) for i, val := range rand.Perm(len(in))[:max] { out[i] = in[val] } @@ -219,12 +219,12 @@ func randomSubsetOfPeers(in []peerstore.PeerInfo, max int) []peerstore.PeerInfo type Peers []config.BootstrapPeer -func (bpeers Peers) ToPeerInfos() []peerstore.PeerInfo { - pinfos := make(map[peer.ID]*peerstore.PeerInfo) +func (bpeers Peers) ToPeerInfos() []peer.AddrInfo { + pinfos := make(map[peer.ID]*peer.AddrInfo) for _, bootstrap := range bpeers { pinfo, ok := pinfos[bootstrap.ID()] if !ok { - pinfo = new(peerstore.PeerInfo) + pinfo = new(peer.AddrInfo) pinfos[bootstrap.ID()] = pinfo pinfo.ID = bootstrap.ID() } @@ -232,7 +232,7 @@ func (bpeers Peers) ToPeerInfos() []peerstore.PeerInfo { pinfo.Addrs = append(pinfo.Addrs, bootstrap.Transport()) } - var peers []peerstore.PeerInfo + var peers []peer.AddrInfo for _, pinfo := range pinfos { peers = append(peers, *pinfo) } diff --git a/core/bootstrap/bootstrap_test.go b/core/bootstrap/bootstrap_test.go index 0c7799858..e7e460b87 100644 --- a/core/bootstrap/bootstrap_test.go +++ b/core/bootstrap/bootstrap_test.go @@ -5,20 +5,20 @@ import ( "testing" config "github.com/ipfs/go-ipfs-config" - pstore "github.com/libp2p/go-libp2p-peerstore" - testutil "github.com/libp2p/go-testutil" + "github.com/libp2p/go-libp2p-core/peer" + "github.com/libp2p/go-libp2p-core/test" ) func TestSubsetWhenMaxIsGreaterThanLengthOfSlice(t *testing.T) { - var ps []pstore.PeerInfo + var ps []peer.AddrInfo sizeofSlice := 100 for i := 0; i < sizeofSlice; i++ { - pid, err := testutil.RandPeerID() + pid, err := test.RandPeerID() if err != nil { t.Fatal(err) } - ps = append(ps, pstore.PeerInfo{ID: pid}) + ps = append(ps, peer.AddrInfo{ID: pid}) } out := randomSubsetOfPeers(ps, 2*sizeofSlice) if len(out) != len(ps) { @@ -29,7 +29,7 @@ func TestSubsetWhenMaxIsGreaterThanLengthOfSlice(t *testing.T) { func TestMultipleAddrsPerPeer(t *testing.T) { var bsps []config.BootstrapPeer for i := 0; i < 10; i++ { - pid, err := testutil.RandPeerID() + pid, err := test.RandPeerID() if err != nil { t.Fatal(err) } From c73b0a6c3a919490452cc6cb2a52b6d5f9d749fe Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Fri, 31 May 2019 17:06:52 -0700 Subject: [PATCH 50/63] chore: deprecate go-ipfs-addr This commit was moved from ipfs/kubo@5d468e23ea315811018c3af46e0b29f4355f7d37 --- core/bootstrap/bootstrap.go | 24 ------------------------ core/bootstrap/bootstrap_test.go | 31 ------------------------------- 2 files changed, 55 deletions(-) diff --git a/core/bootstrap/bootstrap.go b/core/bootstrap/bootstrap.go index b2cf1a811..026690366 100644 --- a/core/bootstrap/bootstrap.go +++ b/core/bootstrap/bootstrap.go @@ -9,7 +9,6 @@ import ( "sync" "time" - config "github.com/ipfs/go-ipfs-config" logging "github.com/ipfs/go-log" "github.com/jbenet/goprocess" "github.com/jbenet/goprocess/context" @@ -216,26 +215,3 @@ func randomSubsetOfPeers(in []peer.AddrInfo, max int) []peer.AddrInfo { } return out } - -type Peers []config.BootstrapPeer - -func (bpeers Peers) ToPeerInfos() []peer.AddrInfo { - pinfos := make(map[peer.ID]*peer.AddrInfo) - for _, bootstrap := range bpeers { - pinfo, ok := pinfos[bootstrap.ID()] - if !ok { - pinfo = new(peer.AddrInfo) - pinfos[bootstrap.ID()] = pinfo - pinfo.ID = bootstrap.ID() - } - - pinfo.Addrs = append(pinfo.Addrs, bootstrap.Transport()) - } - - var peers []peer.AddrInfo - for _, pinfo := range pinfos { - peers = append(peers, *pinfo) - } - - return peers -} diff --git a/core/bootstrap/bootstrap_test.go b/core/bootstrap/bootstrap_test.go index e7e460b87..23128c31f 100644 --- a/core/bootstrap/bootstrap_test.go +++ b/core/bootstrap/bootstrap_test.go @@ -1,10 +1,8 @@ package bootstrap import ( - "fmt" "testing" - config "github.com/ipfs/go-ipfs-config" "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/test" ) @@ -25,32 +23,3 @@ func TestSubsetWhenMaxIsGreaterThanLengthOfSlice(t *testing.T) { t.Fail() } } - -func TestMultipleAddrsPerPeer(t *testing.T) { - var bsps []config.BootstrapPeer - for i := 0; i < 10; i++ { - pid, err := test.RandPeerID() - if err != nil { - t.Fatal(err) - } - - addr := fmt.Sprintf("/ip4/127.0.0.1/tcp/5001/ipfs/%s", pid.Pretty()) - bsp1, err := config.ParseBootstrapPeer(addr) - if err != nil { - t.Fatal(err) - } - - addr = fmt.Sprintf("/ip4/127.0.0.1/udp/5002/utp/ipfs/%s", pid.Pretty()) - bsp2, err := config.ParseBootstrapPeer(addr) - if err != nil { - t.Fatal(err) - } - - bsps = append(bsps, bsp1, bsp2) - } - - pinfos := Peers.ToPeerInfos(bsps) - if len(pinfos) != len(bsps)/2 { - t.Fatal("expected fewer peers") - } -} From c794bdba2ab1af0be7e6aa8646121f45435da0ee Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 16 Jan 2020 15:48:20 -0800 Subject: [PATCH 51/63] fix: migrate from deprecated warning function This commit was moved from ipfs/kubo@a53d48059bff98e3a48faf79103651ce301a7ab2 --- core/bootstrap/bootstrap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/bootstrap/bootstrap.go b/core/bootstrap/bootstrap.go index 026690366..e6ec29eea 100644 --- a/core/bootstrap/bootstrap.go +++ b/core/bootstrap/bootstrap.go @@ -80,7 +80,7 @@ func Bootstrap(id peer.ID, host host.Host, rt routing.Routing, cfg BootstrapConf if len(cfg.BootstrapPeers()) == 0 { // We *need* to bootstrap but we have no bootstrap peers // configured *at all*, inform the user. - log.Warning("no bootstrap nodes configured: go-ipfs may have difficulty connecting to the network") + log.Warn("no bootstrap nodes configured: go-ipfs may have difficulty connecting to the network") } // the periodic bootstrap function -- the connection supervisor From 1b4de817ffa71a73038020571c68abf6a72ecd84 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Thu, 16 Jan 2020 16:18:53 -0800 Subject: [PATCH 52/63] fix(tracing): remove event tracing We've deprecated this system and have yet to move to a new system. We might as well remove everything, switch to a new system, then deliberately trace the entire system. This commit was moved from ipfs/kubo@906f45edd9899352efba710e2f53978fc4b8c6e4 --- core/bootstrap/bootstrap.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/core/bootstrap/bootstrap.go b/core/bootstrap/bootstrap.go index e6ec29eea..4edc5ac84 100644 --- a/core/bootstrap/bootstrap.go +++ b/core/bootstrap/bootstrap.go @@ -18,7 +18,6 @@ import ( "github.com/libp2p/go-libp2p-core/peer" "github.com/libp2p/go-libp2p-core/peerstore" "github.com/libp2p/go-libp2p-core/routing" - "github.com/libp2p/go-libp2p-loggables" ) var log = logging.Logger("bootstrap") @@ -86,10 +85,8 @@ func Bootstrap(id peer.ID, host host.Host, rt routing.Routing, cfg BootstrapConf // the periodic bootstrap function -- the connection supervisor periodic := func(worker goprocess.Process) { ctx := goprocessctx.OnClosingContext(worker) - defer log.EventBegin(ctx, "periodicBootstrap", id).Done() if err := bootstrapRound(ctx, host, cfg); err != nil { - log.Event(ctx, "bootstrapError", id, loggables.Error(err)) log.Debugf("%s bootstrap error: %s", id, err) } @@ -126,7 +123,6 @@ func bootstrapRound(ctx context.Context, host host.Host, cfg BootstrapConfig) er // determine how many bootstrap connections to open connected := host.Network().Peers() if len(connected) >= cfg.MinPeerThreshold { - log.Event(ctx, "bootstrapSkip", id) log.Debugf("%s core bootstrap skipped -- connected to %d (> %d) nodes", id, len(connected), cfg.MinPeerThreshold) return nil @@ -150,7 +146,6 @@ func bootstrapRound(ctx context.Context, host host.Host, cfg BootstrapConfig) er // connect to a random susbset of bootstrap candidates randSubset := randomSubsetOfPeers(notConnected, numToDial) - defer log.EventBegin(ctx, "bootstrapStart", id).Done() log.Debugf("%s bootstrapping to %d nodes: %s", id, numToDial, randSubset) return bootstrapConnect(ctx, host, randSubset) } @@ -172,17 +167,14 @@ func bootstrapConnect(ctx context.Context, ph host.Host, peers []peer.AddrInfo) wg.Add(1) go func(p peer.AddrInfo) { defer wg.Done() - defer log.EventBegin(ctx, "bootstrapDial", ph.ID(), p.ID).Done() log.Debugf("%s bootstrapping to %s", ph.ID(), p.ID) ph.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.PermanentAddrTTL) if err := ph.Connect(ctx, p); err != nil { - log.Event(ctx, "bootstrapDialFailed", p.ID) log.Debugf("failed to bootstrap with %v: %s", p.ID, err) errs <- err return } - log.Event(ctx, "bootstrapDialSuccess", p.ID) log.Infof("bootstrapped with %v", p.ID) }(p) } From 735d89cb31885432ecf5270bedb7aeaf47f3e4fc Mon Sep 17 00:00:00 2001 From: Jorropo Date: Mon, 29 Aug 2022 13:55:00 +0200 Subject: [PATCH 53/63] chore: bump go-libp2p v0.22.0 & go1.18&go1.19 Fixes: #9225 This commit was moved from ipfs/kubo@196887cbe5fbcd41243c1dfb0db681a1cc2914ff --- core/bootstrap/bootstrap.go | 10 +++++----- core/bootstrap/bootstrap_test.go | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/core/bootstrap/bootstrap.go b/core/bootstrap/bootstrap.go index 4edc5ac84..daa0a44d3 100644 --- a/core/bootstrap/bootstrap.go +++ b/core/bootstrap/bootstrap.go @@ -13,11 +13,11 @@ import ( "github.com/jbenet/goprocess" "github.com/jbenet/goprocess/context" "github.com/jbenet/goprocess/periodic" - "github.com/libp2p/go-libp2p-core/host" - "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/peerstore" - "github.com/libp2p/go-libp2p-core/routing" + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/peerstore" + "github.com/libp2p/go-libp2p/core/routing" ) var log = logging.Logger("bootstrap") diff --git a/core/bootstrap/bootstrap_test.go b/core/bootstrap/bootstrap_test.go index 23128c31f..98a4a7827 100644 --- a/core/bootstrap/bootstrap_test.go +++ b/core/bootstrap/bootstrap_test.go @@ -3,8 +3,8 @@ package bootstrap import ( "testing" - "github.com/libp2p/go-libp2p-core/peer" - "github.com/libp2p/go-libp2p-core/test" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/test" ) func TestSubsetWhenMaxIsGreaterThanLengthOfSlice(t *testing.T) { From e1776db498e92e574169d9a731a145954cc88859 Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Thu, 25 May 2023 09:39:49 -0300 Subject: [PATCH 54/63] feat(bootstrap): save connected peers as backup bootstrap peers (#8856) * feat(bootstrap): save connected peers as backup temporary bootstrap ones * fix: do not add duplicated oldSavedPeers, not using tags, reuse randomizeList * test: add regression test * chore: add changelog --------- Co-authored-by: Henrique Dias Co-authored-by: Marcin Rataj This commit was moved from ipfs/kubo@63561f3baf63524ce7d147f67c0c4b4e0ddc5bc9 --- core/bootstrap/bootstrap.go | 229 ++++++++++++++++++++++++------- core/bootstrap/bootstrap_test.go | 6 +- 2 files changed, 183 insertions(+), 52 deletions(-) diff --git a/core/bootstrap/bootstrap.go b/core/bootstrap/bootstrap.go index daa0a44d3..b566e0e97 100644 --- a/core/bootstrap/bootstrap.go +++ b/core/bootstrap/bootstrap.go @@ -3,16 +3,16 @@ package bootstrap import ( "context" "errors" - "fmt" "io" "math/rand" "sync" + "sync/atomic" "time" logging "github.com/ipfs/go-log" "github.com/jbenet/goprocess" - "github.com/jbenet/goprocess/context" - "github.com/jbenet/goprocess/periodic" + goprocessctx "github.com/jbenet/goprocess/context" + periodicproc "github.com/jbenet/goprocess/periodic" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" @@ -50,13 +50,26 @@ type BootstrapConfig struct { // for the bootstrap process to use. This makes it possible for clients // to control the peers the process uses at any moment. BootstrapPeers func() []peer.AddrInfo + + // BackupBootstrapInterval governs the periodic interval at which the node will + // attempt to save connected nodes to use as temporary bootstrap peers. + BackupBootstrapInterval time.Duration + + // MaxBackupBootstrapSize controls the maximum number of peers we're saving + // as backup bootstrap peers. + MaxBackupBootstrapSize int + + SaveBackupBootstrapPeers func(context.Context, []peer.AddrInfo) + LoadBackupBootstrapPeers func(context.Context) []peer.AddrInfo } // DefaultBootstrapConfig specifies default sane parameters for bootstrapping. var DefaultBootstrapConfig = BootstrapConfig{ - MinPeerThreshold: 4, - Period: 30 * time.Second, - ConnectionTimeout: (30 * time.Second) / 3, // Perod / 3 + MinPeerThreshold: 4, + Period: 30 * time.Second, + ConnectionTimeout: (30 * time.Second) / 3, // Perod / 3 + BackupBootstrapInterval: 1 * time.Hour, + MaxBackupBootstrapSize: 20, } func BootstrapConfigWithPeers(pis []peer.AddrInfo) BootstrapConfig { @@ -90,6 +103,9 @@ func Bootstrap(id peer.ID, host host.Host, rt routing.Routing, cfg BootstrapConf log.Debugf("%s bootstrap error: %s", id, err) } + // Exit the first call (triggered independently by `proc.Go`, not `Tick`) + // only after being done with the *single* Routing.Bootstrap call. Following + // periodic calls (`Tick`) will not block on this. <-doneWithRound } @@ -108,9 +124,100 @@ func Bootstrap(id peer.ID, host host.Host, rt routing.Routing, cfg BootstrapConf doneWithRound <- struct{}{} close(doneWithRound) // it no longer blocks periodic + + startSavePeersAsTemporaryBootstrapProc(cfg, host, proc) + return proc, nil } +// Aside of the main bootstrap process we also run a secondary one that saves +// connected peers as a backup measure if we can't connect to the official +// bootstrap ones. These peers will serve as *temporary* bootstrap nodes. +func startSavePeersAsTemporaryBootstrapProc(cfg BootstrapConfig, host host.Host, bootstrapProc goprocess.Process) { + savePeersFn := func(worker goprocess.Process) { + ctx := goprocessctx.OnClosingContext(worker) + + if err := saveConnectedPeersAsTemporaryBootstrap(ctx, host, cfg); err != nil { + log.Debugf("saveConnectedPeersAsTemporaryBootstrap error: %s", err) + } + } + savePeersProc := periodicproc.Tick(cfg.BackupBootstrapInterval, savePeersFn) + + // When the main bootstrap process ends also terminate the 'save connected + // peers' ones. Coupling the two seems the easiest way to handle this backup + // process without additional complexity. + go func() { + <-bootstrapProc.Closing() + savePeersProc.Close() + }() + + // Run the first round now (after the first bootstrap process has finished) + // as the SavePeersPeriod can be much longer than bootstrap. + savePeersProc.Go(savePeersFn) +} + +func saveConnectedPeersAsTemporaryBootstrap(ctx context.Context, host host.Host, cfg BootstrapConfig) error { + // Randomize the list of connected peers, we don't prioritize anyone. + connectedPeers := randomizeList(host.Network().Peers()) + + bootstrapPeers := cfg.BootstrapPeers() + backupPeers := make([]peer.AddrInfo, 0, cfg.MaxBackupBootstrapSize) + + // Choose peers to save and filter out the ones that are already bootstrap nodes. + for _, p := range connectedPeers { + found := false + for _, bootstrapPeer := range bootstrapPeers { + if p == bootstrapPeer.ID { + found = true + break + } + } + if !found { + backupPeers = append(backupPeers, peer.AddrInfo{ + ID: p, + Addrs: host.Network().Peerstore().Addrs(p), + }) + } + + if len(backupPeers) >= cfg.MaxBackupBootstrapSize { + break + } + } + + // If we didn't reach the target number use previously stored connected peers. + if len(backupPeers) < cfg.MaxBackupBootstrapSize { + oldSavedPeers := cfg.LoadBackupBootstrapPeers(ctx) + log.Debugf("missing %d peers to reach backup bootstrap target of %d, trying from previous list of %d saved peers", + cfg.MaxBackupBootstrapSize-len(backupPeers), cfg.MaxBackupBootstrapSize, len(oldSavedPeers)) + + // Add some of the old saved peers. Ensure we don't duplicate them. + for _, p := range oldSavedPeers { + found := false + for _, sp := range backupPeers { + if p.ID == sp.ID { + found = true + break + } + } + + if !found { + backupPeers = append(backupPeers, p) + } + + if len(backupPeers) >= cfg.MaxBackupBootstrapSize { + break + } + } + } + + cfg.SaveBackupBootstrapPeers(ctx, backupPeers) + log.Debugf("saved %d peers (of %d target) as bootstrap backup in the config", len(backupPeers), cfg.MaxBackupBootstrapSize) + return nil +} + +// Connect to as many peers needed to reach the BootstrapConfig.MinPeerThreshold. +// Peers can be original bootstrap or temporary ones (drawn from a list of +// persisted previously connected peers). func bootstrapRound(ctx context.Context, host host.Host, cfg BootstrapConfig) error { ctx, cancel := context.WithTimeout(ctx, cfg.ConnectionTimeout) @@ -127,35 +234,58 @@ func bootstrapRound(ctx context.Context, host host.Host, cfg BootstrapConfig) er id, len(connected), cfg.MinPeerThreshold) return nil } - numToDial := cfg.MinPeerThreshold - len(connected) + numToDial := cfg.MinPeerThreshold - len(connected) // numToDial > 0 - // filter out bootstrap nodes we are already connected to - var notConnected []peer.AddrInfo - for _, p := range peers { - if host.Network().Connectedness(p.ID) != network.Connected { - notConnected = append(notConnected, p) + if len(peers) > 0 { + numToDial -= int(peersConnect(ctx, host, peers, numToDial, true)) + if numToDial <= 0 { + return nil } } - // if connected to all bootstrap peer candidates, exit - if len(notConnected) < 1 { - log.Debugf("%s no more bootstrap peers to create %d connections", id, numToDial) - return ErrNotEnoughBootstrapPeers + log.Debugf("not enough bootstrap peers to fill the remaining target of %d connections, trying backup list", numToDial) + + tempBootstrapPeers := cfg.LoadBackupBootstrapPeers(ctx) + if len(tempBootstrapPeers) > 0 { + numToDial -= int(peersConnect(ctx, host, tempBootstrapPeers, numToDial, false)) + if numToDial <= 0 { + return nil + } } - // connect to a random susbset of bootstrap candidates - randSubset := randomSubsetOfPeers(notConnected, numToDial) + log.Debugf("tried both original bootstrap peers and temporary ones but still missing target of %d connections", numToDial) - log.Debugf("%s bootstrapping to %d nodes: %s", id, numToDial, randSubset) - return bootstrapConnect(ctx, host, randSubset) + return ErrNotEnoughBootstrapPeers } -func bootstrapConnect(ctx context.Context, ph host.Host, peers []peer.AddrInfo) error { - if len(peers) < 1 { - return ErrNotEnoughBootstrapPeers - } +// Attempt to make `needed` connections from the `availablePeers` list. Mark +// peers as either `permanent` or temporary when adding them to the Peerstore. +// Return the number of connections completed. We eagerly over-connect in parallel, +// so we might connect to more than needed. +// (We spawn as many routines and attempt connections as the number of availablePeers, +// but this list comes from restricted sets of original or temporary bootstrap +// nodes which will keep it under a sane value.) +func peersConnect(ctx context.Context, ph host.Host, availablePeers []peer.AddrInfo, needed int, permanent bool) uint64 { + peers := randomizeList(availablePeers) + + // Monitor the number of connections and stop if we reach the target. + var connected uint64 + ctx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { + for { + select { + case <-ctx.Done(): + return + case <-time.After(1 * time.Second): + if int(atomic.LoadUint64(&connected)) >= needed { + cancel() + return + } + } + } + }() - errs := make(chan error, len(peers)) var wg sync.WaitGroup for _, p := range peers { @@ -164,45 +294,46 @@ func bootstrapConnect(ctx context.Context, ph host.Host, peers []peer.AddrInfo) // fail/abort due to an expiring context. // Also, performed asynchronously for dial speed. + if int(atomic.LoadUint64(&connected)) >= needed { + cancel() + break + } + wg.Add(1) go func(p peer.AddrInfo) { defer wg.Done() + + // Skip addresses belonging to a peer we're already connected to. + // (Not a guarantee but a best-effort policy.) + if ph.Network().Connectedness(p.ID) == network.Connected { + return + } log.Debugf("%s bootstrapping to %s", ph.ID(), p.ID) - ph.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.PermanentAddrTTL) if err := ph.Connect(ctx, p); err != nil { - log.Debugf("failed to bootstrap with %v: %s", p.ID, err) - errs <- err + if ctx.Err() != context.Canceled { + log.Debugf("failed to bootstrap with %v: %s", p.ID, err) + } return } + if permanent { + // We're connecting to an original bootstrap peer, mark it as + // a permanent address (Connect will register it as TempAddrTTL). + ph.Peerstore().AddAddrs(p.ID, p.Addrs, peerstore.PermanentAddrTTL) + } + log.Infof("bootstrapped with %v", p.ID) + atomic.AddUint64(&connected, 1) }(p) } wg.Wait() - // our failure condition is when no connection attempt succeeded. - // So drain the errs channel, counting the results. - close(errs) - count := 0 - var err error - for err = range errs { - if err != nil { - count++ - } - } - if count == len(peers) { - return fmt.Errorf("failed to bootstrap. %s", err) - } - return nil + return connected } -func randomSubsetOfPeers(in []peer.AddrInfo, max int) []peer.AddrInfo { - if max > len(in) { - max = len(in) - } - - out := make([]peer.AddrInfo, max) - for i, val := range rand.Perm(len(in))[:max] { +func randomizeList[T any](in []T) []T { + out := make([]T, len(in)) + for i, val := range rand.Perm(len(in)) { out[i] = in[val] } return out diff --git a/core/bootstrap/bootstrap_test.go b/core/bootstrap/bootstrap_test.go index 98a4a7827..39490a474 100644 --- a/core/bootstrap/bootstrap_test.go +++ b/core/bootstrap/bootstrap_test.go @@ -7,9 +7,9 @@ import ( "github.com/libp2p/go-libp2p/core/test" ) -func TestSubsetWhenMaxIsGreaterThanLengthOfSlice(t *testing.T) { +func TestRandomizeAddressList(t *testing.T) { var ps []peer.AddrInfo - sizeofSlice := 100 + sizeofSlice := 10 for i := 0; i < sizeofSlice; i++ { pid, err := test.RandPeerID() if err != nil { @@ -18,7 +18,7 @@ func TestSubsetWhenMaxIsGreaterThanLengthOfSlice(t *testing.T) { ps = append(ps, peer.AddrInfo{ID: pid}) } - out := randomSubsetOfPeers(ps, 2*sizeofSlice) + out := randomizeList(ps) if len(out) != len(ps) { t.Fail() } From 4ca63d99ca6bd765b33efda62802a2b64664293a Mon Sep 17 00:00:00 2001 From: Kay Date: Thu, 17 Aug 2023 15:32:08 +0330 Subject: [PATCH 55/63] style: gofumpt and godot [skip changelog] (#10081) This commit was moved from ipfs/kubo@f12b372af9cc32975ff48397708fac3ec1f9966f --- core/bootstrap/bootstrap.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/bootstrap/bootstrap.go b/core/bootstrap/bootstrap.go index b566e0e97..ed95d74e1 100644 --- a/core/bootstrap/bootstrap.go +++ b/core/bootstrap/bootstrap.go @@ -85,7 +85,6 @@ func BootstrapConfigWithPeers(pis []peer.AddrInfo) BootstrapConfig { // connections to well-known bootstrap peers. It also kicks off subsystem // bootstrapping (i.e. routing). func Bootstrap(id peer.ID, host host.Host, rt routing.Routing, cfg BootstrapConfig) (io.Closer, error) { - // make a signal to wait for one bootstrap round to complete. doneWithRound := make(chan struct{}) @@ -219,7 +218,6 @@ func saveConnectedPeersAsTemporaryBootstrap(ctx context.Context, host host.Host, // Peers can be original bootstrap or temporary ones (drawn from a list of // persisted previously connected peers). func bootstrapRound(ctx context.Context, host host.Host, cfg BootstrapConfig) error { - ctx, cancel := context.WithTimeout(ctx, cfg.ConnectionTimeout) defer cancel() id := host.ID() From a775656d698b277fff331547a8cc9165e5c648ef Mon Sep 17 00:00:00 2001 From: Andrew Gillis Date: Thu, 21 Sep 2023 09:29:38 -0700 Subject: [PATCH 56/63] core/bootstrap: fix panic without backup bootstrap peer functions (#10029) Fix panic when backup bootstrap peer load and save funcs are nil A panic occurs when the first bootstrap round runs is these functions are not assigned in the configuration: - `LoadBackupBootstrapPeers` - `SaveBackupBootstrapPeers` This fix assumes that it is acceptable for these functions to be nil, as it may be desirable to disable the backup peer load and save functionality. This commit was moved from ipfs/kubo@c46cbecb832b9a25f74a275b946b3a0ff3aefaba --- core/bootstrap/bootstrap.go | 50 ++++++++++++-- core/bootstrap/bootstrap_test.go | 114 +++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 7 deletions(-) diff --git a/core/bootstrap/bootstrap.go b/core/bootstrap/bootstrap.go index ed95d74e1..acd7ef672 100644 --- a/core/bootstrap/bootstrap.go +++ b/core/bootstrap/bootstrap.go @@ -59,8 +59,8 @@ type BootstrapConfig struct { // as backup bootstrap peers. MaxBackupBootstrapSize int - SaveBackupBootstrapPeers func(context.Context, []peer.AddrInfo) - LoadBackupBootstrapPeers func(context.Context) []peer.AddrInfo + saveBackupBootstrapPeers func(context.Context, []peer.AddrInfo) + loadBackupBootstrapPeers func(context.Context) []peer.AddrInfo } // DefaultBootstrapConfig specifies default sane parameters for bootstrapping. @@ -72,14 +72,41 @@ var DefaultBootstrapConfig = BootstrapConfig{ MaxBackupBootstrapSize: 20, } -func BootstrapConfigWithPeers(pis []peer.AddrInfo) BootstrapConfig { +// BootstrapConfigWithPeers creates a default BootstrapConfig configured with +// the specified peers, and optional functions to load and save backup peers. +func BootstrapConfigWithPeers(pis []peer.AddrInfo, options ...func(*BootstrapConfig)) BootstrapConfig { cfg := DefaultBootstrapConfig cfg.BootstrapPeers = func() []peer.AddrInfo { return pis } + for _, opt := range options { + opt(&cfg) + } return cfg } +// WithBackupPeers configures functions to load and save backup bootstrap peers. +func WithBackupPeers(load func(context.Context) []peer.AddrInfo, save func(context.Context, []peer.AddrInfo)) func(*BootstrapConfig) { + if save == nil && load != nil || save != nil && load == nil { + panic("both load and save backup bootstrap peers functions must be defined") + } + return func(cfg *BootstrapConfig) { + cfg.loadBackupBootstrapPeers = load + cfg.saveBackupBootstrapPeers = save + } +} + +// BackupPeers returns the load and save backup peers functions. +func (cfg *BootstrapConfig) BackupPeers() (func(context.Context) []peer.AddrInfo, func(context.Context, []peer.AddrInfo)) { + return cfg.loadBackupBootstrapPeers, cfg.saveBackupBootstrapPeers +} + +// SetBackupPeers sets the load and save backup peers functions. +func (cfg *BootstrapConfig) SetBackupPeers(load func(context.Context) []peer.AddrInfo, save func(context.Context, []peer.AddrInfo)) { + opt := WithBackupPeers(load, save) + opt(cfg) +} + // Bootstrap kicks off IpfsNode bootstrapping. This function will periodically // check the number of open connections and -- if there are too few -- initiate // connections to well-known bootstrap peers. It also kicks off subsystem @@ -124,7 +151,11 @@ func Bootstrap(id peer.ID, host host.Host, rt routing.Routing, cfg BootstrapConf doneWithRound <- struct{}{} close(doneWithRound) // it no longer blocks periodic - startSavePeersAsTemporaryBootstrapProc(cfg, host, proc) + // If loadBackupBootstrapPeers is not nil then saveBackupBootstrapPeers + // must also not be nil. + if cfg.loadBackupBootstrapPeers != nil { + startSavePeersAsTemporaryBootstrapProc(cfg, host, proc) + } return proc, nil } @@ -185,7 +216,7 @@ func saveConnectedPeersAsTemporaryBootstrap(ctx context.Context, host host.Host, // If we didn't reach the target number use previously stored connected peers. if len(backupPeers) < cfg.MaxBackupBootstrapSize { - oldSavedPeers := cfg.LoadBackupBootstrapPeers(ctx) + oldSavedPeers := cfg.loadBackupBootstrapPeers(ctx) log.Debugf("missing %d peers to reach backup bootstrap target of %d, trying from previous list of %d saved peers", cfg.MaxBackupBootstrapSize-len(backupPeers), cfg.MaxBackupBootstrapSize, len(oldSavedPeers)) @@ -209,7 +240,7 @@ func saveConnectedPeersAsTemporaryBootstrap(ctx context.Context, host host.Host, } } - cfg.SaveBackupBootstrapPeers(ctx, backupPeers) + cfg.saveBackupBootstrapPeers(ctx, backupPeers) log.Debugf("saved %d peers (of %d target) as bootstrap backup in the config", len(backupPeers), cfg.MaxBackupBootstrapSize) return nil } @@ -241,9 +272,14 @@ func bootstrapRound(ctx context.Context, host host.Host, cfg BootstrapConfig) er } } + if cfg.loadBackupBootstrapPeers == nil { + log.Debugf("not enough bootstrap peers to fill the remaining target of %d connections", numToDial) + return ErrNotEnoughBootstrapPeers + } + log.Debugf("not enough bootstrap peers to fill the remaining target of %d connections, trying backup list", numToDial) - tempBootstrapPeers := cfg.LoadBackupBootstrapPeers(ctx) + tempBootstrapPeers := cfg.loadBackupBootstrapPeers(ctx) if len(tempBootstrapPeers) > 0 { numToDial -= int(peersConnect(ctx, host, tempBootstrapPeers, numToDial, false)) if numToDial <= 0 { diff --git a/core/bootstrap/bootstrap_test.go b/core/bootstrap/bootstrap_test.go index 39490a474..d933379d4 100644 --- a/core/bootstrap/bootstrap_test.go +++ b/core/bootstrap/bootstrap_test.go @@ -1,8 +1,14 @@ package bootstrap import ( + "context" + "crypto/rand" + "reflect" "testing" + "time" + "github.com/libp2p/go-libp2p" + "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/test" ) @@ -23,3 +29,111 @@ func TestRandomizeAddressList(t *testing.T) { t.Fail() } } + +func TestLoadAndSaveOptions(t *testing.T) { + loadFunc := func(_ context.Context) []peer.AddrInfo { return nil } + saveFunc := func(_ context.Context, _ []peer.AddrInfo) {} + + bootCfg := BootstrapConfigWithPeers(nil, WithBackupPeers(loadFunc, saveFunc)) + load, save := bootCfg.BackupPeers() + if load == nil { + t.Fatal("load function not assigned") + } + if reflect.ValueOf(load).Pointer() != reflect.ValueOf(loadFunc).Pointer() { + t.Fatal("load not assigned correct function") + } + if save == nil { + t.Fatal("save function not assigned") + } + if reflect.ValueOf(save).Pointer() != reflect.ValueOf(saveFunc).Pointer() { + t.Fatal("save not assigned correct function") + } + + assertPanics(t, "with only load func", func() { + BootstrapConfigWithPeers(nil, WithBackupPeers(loadFunc, nil)) + }) + + assertPanics(t, "with only save func", func() { + BootstrapConfigWithPeers(nil, WithBackupPeers(nil, saveFunc)) + }) + + bootCfg = BootstrapConfigWithPeers(nil, WithBackupPeers(nil, nil)) + load, save = bootCfg.BackupPeers() + if load != nil || save != nil { + t.Fatal("load and save functions should both be nil") + } +} + +func TestSetBackupPeers(t *testing.T) { + loadFunc := func(_ context.Context) []peer.AddrInfo { return nil } + saveFunc := func(_ context.Context, _ []peer.AddrInfo) {} + + bootCfg := DefaultBootstrapConfig + bootCfg.SetBackupPeers(loadFunc, saveFunc) + load, save := bootCfg.BackupPeers() + if load == nil { + t.Fatal("load function not assigned") + } + if reflect.ValueOf(load).Pointer() != reflect.ValueOf(loadFunc).Pointer() { + t.Fatal("load not assigned correct function") + } + if save == nil { + t.Fatal("save function not assigned") + } + if reflect.ValueOf(save).Pointer() != reflect.ValueOf(saveFunc).Pointer() { + t.Fatal("save not assigned correct function") + } + + assertPanics(t, "with only load func", func() { + bootCfg.SetBackupPeers(loadFunc, nil) + }) + + assertPanics(t, "with only save func", func() { + bootCfg.SetBackupPeers(nil, saveFunc) + }) + + bootCfg.SetBackupPeers(nil, nil) + load, save = bootCfg.BackupPeers() + if load != nil || save != nil { + t.Fatal("load and save functions should both be nil") + } +} + +func TestNoTempPeersLoadAndSave(t *testing.T) { + period := 500 * time.Millisecond + bootCfg := BootstrapConfigWithPeers(nil) + bootCfg.MinPeerThreshold = 2 + bootCfg.Period = period + + priv, pub, err := crypto.GenerateEd25519Key(rand.Reader) + if err != nil { + t.Fatal(err) + } + peerID, err := peer.IDFromPublicKey(pub) + if err != nil { + t.Fatal(err) + } + p2pHost, err := libp2p.New(libp2p.Identity(priv)) + if err != nil { + t.Fatal(err) + } + + bootstrapper, err := Bootstrap(peerID, p2pHost, nil, bootCfg) + if err != nil { + t.Fatal(err) + } + + time.Sleep(4 * period) + bootstrapper.Close() + +} + +func assertPanics(t *testing.T, name string, f func()) { + defer func() { + if r := recover(); r == nil { + t.Errorf("%s: did not panic as expected", name) + } + }() + + f() +} From 24ee956efbf5fe037dfa00e5d1c0f9da023885c7 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Wed, 6 Sep 2023 00:32:10 +0200 Subject: [PATCH 57/63] perf: make bootstrap saves O(N) This commit was moved from ipfs/kubo@66590e350f6dbdf9da77e9548a91f9ce96d803dd --- core/bootstrap/bootstrap.go | 39 +++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/core/bootstrap/bootstrap.go b/core/bootstrap/bootstrap.go index acd7ef672..5cde50371 100644 --- a/core/bootstrap/bootstrap.go +++ b/core/bootstrap/bootstrap.go @@ -192,22 +192,24 @@ func saveConnectedPeersAsTemporaryBootstrap(ctx context.Context, host host.Host, bootstrapPeers := cfg.BootstrapPeers() backupPeers := make([]peer.AddrInfo, 0, cfg.MaxBackupBootstrapSize) + foundPeers := make(map[peer.ID]struct{}, cfg.MaxBackupBootstrapSize+len(bootstrapPeers)) + + // Don't record bootstrap peers + for _, b := range bootstrapPeers { + foundPeers[b.ID] = struct{}{} + } // Choose peers to save and filter out the ones that are already bootstrap nodes. for _, p := range connectedPeers { - found := false - for _, bootstrapPeer := range bootstrapPeers { - if p == bootstrapPeer.ID { - found = true - break - } - } - if !found { - backupPeers = append(backupPeers, peer.AddrInfo{ - ID: p, - Addrs: host.Network().Peerstore().Addrs(p), - }) + if _, found := foundPeers[p]; found { + continue } + foundPeers[p] = struct{}{} + + backupPeers = append(backupPeers, peer.AddrInfo{ + ID: p, + Addrs: host.Network().Peerstore().Addrs(p), + }) if len(backupPeers) >= cfg.MaxBackupBootstrapSize { break @@ -222,17 +224,12 @@ func saveConnectedPeersAsTemporaryBootstrap(ctx context.Context, host host.Host, // Add some of the old saved peers. Ensure we don't duplicate them. for _, p := range oldSavedPeers { - found := false - for _, sp := range backupPeers { - if p.ID == sp.ID { - found = true - break - } + if _, found := foundPeers[p.ID]; found { + continue } + foundPeers[p.ID] = struct{}{} - if !found { - backupPeers = append(backupPeers, p) - } + backupPeers = append(backupPeers, p) if len(backupPeers) >= cfg.MaxBackupBootstrapSize { break From a29dc5a758a53ad2a172c34c5cfb9cae82d55969 Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 28 Sep 2023 18:39:02 -0700 Subject: [PATCH 58/63] Move bootstrap from core to top level directory --- {core/bootstrap => bootstrap}/bootstrap.go | 0 {core/bootstrap => bootstrap}/bootstrap_test.go | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {core/bootstrap => bootstrap}/bootstrap.go (100%) rename {core/bootstrap => bootstrap}/bootstrap_test.go (100%) diff --git a/core/bootstrap/bootstrap.go b/bootstrap/bootstrap.go similarity index 100% rename from core/bootstrap/bootstrap.go rename to bootstrap/bootstrap.go diff --git a/core/bootstrap/bootstrap_test.go b/bootstrap/bootstrap_test.go similarity index 100% rename from core/bootstrap/bootstrap_test.go rename to bootstrap/bootstrap_test.go From 48b732db106798edfc1ee2f55459fd6ad182f668 Mon Sep 17 00:00:00 2001 From: gammazero Date: Thu, 28 Sep 2023 18:44:00 -0700 Subject: [PATCH 59/63] Update CHANGELOG and mod tidy --- CHANGELOG.md | 1 + go.sum | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c17a9c46e..c4d8baad4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The following emojis are used to highlight certain changes: * ✹ Migrated repositories into Boxo * [`github.com/ipfs/kubo/peering`](https://pkg.go.dev/github.com/ipfs/kubo/peering) => [`./peering`](./peering) A service which establish, overwatch and maintain long lived connections. + * github.com/ipfs/kubo/core/bootstrap => ./bootstrap ### Changed diff --git a/go.sum b/go.sum index 8beabe300..25af3632d 100644 --- a/go.sum +++ b/go.sum @@ -307,6 +307,7 @@ github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOan github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc= github.com/jbenet/go-cienv v0.1.0/go.mod h1:TqNnHUmJgXau0nCzC7kXWeotg3J9W34CUv5Djy1+FlA= github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk= github.com/jbenet/go-temp-err-catcher v0.1.0/go.mod h1:0kJRvmDZXNMIiJirNPEYfhpPwbGVtZVWC34vc5WLsDk= From 133a4650e25082f5bff7577690800ea5bd5e18e0 Mon Sep 17 00:00:00 2001 From: gammazero Date: Wed, 25 Oct 2023 14:55:34 -0700 Subject: [PATCH 60/63] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4d8baad4..f334cca8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,7 +24,8 @@ The following emojis are used to highlight certain changes: * ✹ Migrated repositories into Boxo * [`github.com/ipfs/kubo/peering`](https://pkg.go.dev/github.com/ipfs/kubo/peering) => [`./peering`](./peering) A service which establish, overwatch and maintain long lived connections. - * github.com/ipfs/kubo/core/bootstrap => ./bootstrap + * [`github.com/ipfs/kubo/core/bootstrap`](https://pkg.go.dev/github.com/ipfs/kubo/core/bootstrap) => [`./bootstrap](./bootstrap) + A service that maintains connections to a number of bootstrap peers. ### Changed From 45426dbd133ae7eaa473d4bc3e5eff88ea6d113f Mon Sep 17 00:00:00 2001 From: Andrew Gillis Date: Wed, 25 Oct 2023 20:55:49 -0700 Subject: [PATCH 61/63] Update bootstrap/bootstrap.go Co-authored-by: Jorropo --- bootstrap/bootstrap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go index 5cde50371..347d98797 100644 --- a/bootstrap/bootstrap.go +++ b/bootstrap/bootstrap.go @@ -67,7 +67,7 @@ type BootstrapConfig struct { var DefaultBootstrapConfig = BootstrapConfig{ MinPeerThreshold: 4, Period: 30 * time.Second, - ConnectionTimeout: (30 * time.Second) / 3, // Perod / 3 + ConnectionTimeout: (30 * time.Second) / 3, // Period / 3 BackupBootstrapInterval: 1 * time.Hour, MaxBackupBootstrapSize: 20, } From 66c76e5aa17b9d6a4f0a364e8517048d14a60bcc Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 1 Nov 2023 10:23:56 +0100 Subject: [PATCH 62/63] docs: update changelog for 0.14.0 --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f334cca8a..ab7b58ded 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,18 @@ The following emojis are used to highlight certain changes: ### Added +### Changed + +### Removed + +### Fixed + +### Security + +## [v0.14.0] + +### Added + * `boxo/gateway`: * A new `WithResolver(...)` option can be used with `NewBlocksBackend(...)` allowing the user to pass their custom `Resolver` implementation. * The gateway now sets a `Cache-Control` header for requests under the `/ipns/` namespace if the TTL for the corresponding IPNS Records or DNSLink entities is known. From 50f0e2c6e631a03f493fc8ddd60b7830e95d8bec Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Wed, 1 Nov 2023 10:25:11 +0100 Subject: [PATCH 63/63] chore: bump version to 0.14.0 --- version.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.json b/version.json index b947c058b..7dd7b578a 100644 --- a/version.json +++ b/version.json @@ -1,3 +1,3 @@ { - "version": "v0.13.1" + "version": "v0.14.0" }