From d29bc1aa30f7c8648935adb6a06abf35d059e29a Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Fri, 25 Sep 2020 22:26:44 +0200 Subject: [PATCH 1/3] fix(gateway): correct breadcrumbs on dnslink site This commit was moved from ipfs/kubo@cd1feb3af4e42a835870aca96243b55760ab5f04 --- gateway/core/corehttp/gateway_handler.go | 11 ++++++++--- gateway/core/corehttp/gateway_indexPage.go | 19 +++++++++++++++++-- gateway/core/corehttp/gateway_test.go | 8 +++++--- gateway/core/corehttp/hostname.go | 21 ++++++++++++++------- 4 files changed, 44 insertions(+), 15 deletions(-) diff --git a/gateway/core/corehttp/gateway_handler.go b/gateway/core/corehttp/gateway_handler.go index b9e7f144b..cff82fef7 100644 --- a/gateway/core/corehttp/gateway_handler.go +++ b/gateway/core/corehttp/gateway_handler.go @@ -379,8 +379,9 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request hash := resolvedPath.Cid().String() - // Storage for gateway URL to be used when linking to other rootIDs. This - // will be blank unless subdomain resolution is being used for this request. + // Gateway root URL to be used when linking to other rootIDs. + // This will be blank unless subdomain or DNSLink resolution is being used + // for this request. var gwURL string // Get gateway hostname and build gateway URL. @@ -396,11 +397,15 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request Listing: dirListing, Size: size, Path: urlPath, - Breadcrumbs: breadcrumbs(urlPath), + Breadcrumbs: breadcrumbs(urlPath, gwURL), BackLink: backLink, Hash: hash, } + // TODO: remove logging below + // tplDataJSON, _ := json.MarshalIndent(tplData, "", " ") + // fmt.Println(string(tplDataJSON)) + err = listingTemplate.Execute(w, tplData) if err != nil { internalWebError(w, err) diff --git a/gateway/core/corehttp/gateway_indexPage.go b/gateway/core/corehttp/gateway_indexPage.go index c9a948708..5cbf33bec 100644 --- a/gateway/core/corehttp/gateway_indexPage.go +++ b/gateway/core/corehttp/gateway_indexPage.go @@ -34,7 +34,7 @@ type breadcrumb struct { Path string } -func breadcrumbs(urlPath string) []breadcrumb { +func breadcrumbs(urlPath string, gwRootURL string) []breadcrumb { var ret []breadcrumb p, err := ipfspath.ParsePath(urlPath) @@ -42,8 +42,9 @@ func breadcrumbs(urlPath string) []breadcrumb { // No breadcrumbs, fallback to bare Path in template return ret } - segs := p.Segments() + ns := segs[0] + contentRoot := segs[1] for i, seg := range segs { if i == 0 { ret = append(ret, breadcrumb{Name: seg}) @@ -55,6 +56,20 @@ func breadcrumbs(urlPath string) []breadcrumb { } } + // Drop the /ipns/ prefix from breadcrumb Paths when directory listing + // on a DNSLink website (loaded due to Host header in HTTP request). + // Necessary because gwRootURL won't have a public gateway mounted. + if ns == "ipns" && (("//" + contentRoot) == gwRootURL) { + prefix := "/ipns/" + contentRoot + for i, crumb := range ret { + if strings.HasPrefix(crumb.Path, prefix) { + ret[i].Path = strings.Replace(crumb.Path, prefix, "", 1) + } + } + // Make contentRoot breadcrumb link to the website root + ret[1].Path = "/" + } + return ret } diff --git a/gateway/core/corehttp/gateway_test.go b/gateway/core/corehttp/gateway_test.go index f4d6d810d..46bf76a5e 100644 --- a/gateway/core/corehttp/gateway_test.go +++ b/gateway/core/corehttp/gateway_test.go @@ -391,6 +391,8 @@ func TestIPNSHostnameRedirect(t *testing.T) { } } +// Test directory listing on DNSLink website +// (scenario when Host header is the same as URL hostname) func TestIPNSHostnameBacklinks(t *testing.T) { ns := mockNamesys{} ts, api, ctx := newTestServerAndNode(t, ns) @@ -445,7 +447,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { s := string(body) t.Logf("body: %s\n", string(body)) - if !matchPathOrBreadcrumbs(s, "/ipns/example.net/foo? #<'") { + if !matchPathOrBreadcrumbs(s, "/ipns/example.net/foo? #<'") { t.Fatalf("expected a path in directory listing") } if !strings.Contains(s, "") { @@ -511,7 +513,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { s = string(body) t.Logf("body: %s\n", string(body)) - if !matchPathOrBreadcrumbs(s, "/ipns/example.net/foo? #<'/bar") { + if !matchPathOrBreadcrumbs(s, "/ipns/example.net/foo? #<'/bar") { t.Fatalf("expected a path in directory listing") } if !strings.Contains(s, "") { @@ -545,7 +547,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { s = string(body) t.Logf("body: %s\n", string(body)) - if !matchPathOrBreadcrumbs(s, "/ipns/example.net") { + if !matchPathOrBreadcrumbs(s, "/ipns/example.net") { t.Fatalf("expected a path in directory listing") } if !strings.Contains(s, "") { diff --git a/gateway/core/corehttp/hostname.go b/gateway/core/corehttp/hostname.go index 8b2666afb..4a531a4d3 100644 --- a/gateway/core/corehttp/hostname.go +++ b/gateway/core/corehttp/hostname.go @@ -129,7 +129,7 @@ func HostnameOption() ServeOption { if !gw.NoDNSLink && isDNSLinkRequest(r.Context(), coreAPI, host) { // rewrite path and handle as DNSLink r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path - childMux.ServeHTTP(w, r) + childMux.ServeHTTP(w, withHostnameContext(r, host)) return } @@ -143,10 +143,6 @@ func HostnameOption() ServeOption { if gw, hostname, ns, rootID, ok := knownSubdomainDetails(host, knownGateways); ok { // Looks like we're using a known gateway in subdomain mode. - // Add gateway hostname context for linking to other root ids. - // Example: localhost/ipfs/{cid} - ctx := context.WithValue(r.Context(), "gw-hostname", hostname) - // Assemble original path prefix. pathPrefix := "/" + ns + "/" + rootID @@ -201,7 +197,7 @@ func HostnameOption() ServeOption { r.URL.Path = pathPrefix + r.URL.Path // Serve path request - childMux.ServeHTTP(w, r.WithContext(ctx)) + childMux.ServeHTTP(w, withHostnameContext(r, hostname)) return } // We don't have a known gateway. Fallback on DNSLink lookup @@ -213,7 +209,7 @@ func HostnameOption() ServeOption { if !cfg.Gateway.NoDNSLink && isDNSLinkRequest(r.Context(), coreAPI, host) { // rewrite path and handle as DNSLink r.URL.Path = "/ipns/" + stripPort(host) + r.URL.Path - childMux.ServeHTTP(w, r) + childMux.ServeHTTP(w, withHostnameContext(r, host)) return } @@ -234,6 +230,17 @@ type wildcardHost struct { spec *config.GatewaySpec } +// Extends request context to include hostname of a canonical gateway root +// (subdomain root or dnslink fqdn) +func withHostnameContext(r *http.Request, hostname string) *http.Request { + // This is required for links on directory listing pages to work correctly + // on subdomain and dnslink gateways. While DNSlink could read value from + // Host header, subdomain gateways have more comples rules (knownSubdomainDetails) + // More: https://github.com/ipfs/dir-index-html/issues/42 + ctx := context.WithValue(r.Context(), "gw-hostname", hostname) + return r.WithContext(ctx) +} + func prepareKnownGateways(publicGateways map[string]*config.GatewaySpec) gatewayHosts { var hosts gatewayHosts From 0872de957ebe8c8ad0dd0fe06526747f763a7808 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Mon, 28 Sep 2020 14:09:29 +0200 Subject: [PATCH 2/3] fix(gw): links in CID column on dir listing This switches go-ipfs to dir-index-html after https://github.com/ipfs/dir-index-html/pull/43 got merged to master This commit was moved from ipfs/kubo@c94bd768d2f755c8825674e72539d49a82503d12 --- gateway/core/corehttp/gateway_handler.go | 5 ++++- gateway/core/corehttp/gateway_indexPage.go | 23 ++++++++++++++++------ 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/gateway/core/corehttp/gateway_handler.go b/gateway/core/corehttp/gateway_handler.go index cff82fef7..e531640da 100644 --- a/gateway/core/corehttp/gateway_handler.go +++ b/gateway/core/corehttp/gateway_handler.go @@ -391,13 +391,16 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request gwURL = "" } + dnslink := hasDNSLinkOrigin(gwURL, urlPath) + // See comment above where originalUrlPath is declared. tplData := listingTemplateData{ GatewayURL: gwURL, + DNSLink: dnslink, Listing: dirListing, Size: size, Path: urlPath, - Breadcrumbs: breadcrumbs(urlPath, gwURL), + Breadcrumbs: breadcrumbs(urlPath, dnslink), BackLink: backLink, Hash: hash, } diff --git a/gateway/core/corehttp/gateway_indexPage.go b/gateway/core/corehttp/gateway_indexPage.go index 5cbf33bec..f735aa0e6 100644 --- a/gateway/core/corehttp/gateway_indexPage.go +++ b/gateway/core/corehttp/gateway_indexPage.go @@ -13,6 +13,7 @@ import ( // structs for directory listing type listingTemplateData struct { GatewayURL string + DNSLink bool Listing []directoryItem Size string Path string @@ -34,7 +35,7 @@ type breadcrumb struct { Path string } -func breadcrumbs(urlPath string, gwRootURL string) []breadcrumb { +func breadcrumbs(urlPath string, dnslinkOrigin bool) []breadcrumb { var ret []breadcrumb p, err := ipfspath.ParsePath(urlPath) @@ -43,7 +44,6 @@ func breadcrumbs(urlPath string, gwRootURL string) []breadcrumb { return ret } segs := p.Segments() - ns := segs[0] contentRoot := segs[1] for i, seg := range segs { if i == 0 { @@ -56,10 +56,11 @@ func breadcrumbs(urlPath string, gwRootURL string) []breadcrumb { } } - // Drop the /ipns/ prefix from breadcrumb Paths when directory listing - // on a DNSLink website (loaded due to Host header in HTTP request). - // Necessary because gwRootURL won't have a public gateway mounted. - if ns == "ipns" && (("//" + contentRoot) == gwRootURL) { + // Drop the /ipns/ prefix from breadcrumb Paths when directory + // listing on a DNSLink website (loaded due to Host header in HTTP + // request). Necessary because the hostname most likely won't have a + // public gateway mounted. + if dnslinkOrigin { prefix := "/ipns/" + contentRoot for i, crumb := range ret { if strings.HasPrefix(crumb.Path, prefix) { @@ -77,6 +78,16 @@ func shortHash(hash string) string { return (hash[0:4] + "\u2026" + hash[len(hash)-4:]) } +// helper to detect DNSLink website context +// (when hostname from gwURL is matching /ipns/ in path) +func hasDNSLinkOrigin(gwURL string, path string) bool { + if gwURL != "" { + dnslinkRoot := strings.Replace(gwURL, "//", "/ipns/", 1) + return strings.HasPrefix(path, dnslinkRoot) + } + return false +} + var listingTemplate *template.Template func init() { From 60703d471be14206fbe4f76134eedfd1f146a491 Mon Sep 17 00:00:00 2001 From: Marcin Rataj Date: Tue, 29 Sep 2020 02:27:28 +0200 Subject: [PATCH 3/3] test(gw): add t0115-gateway-dir-listing.sh to sharness This adds proper end-to-end tests for directory listing on Gateway port that protects us against regressions oni each gw type: - path gateway - subdomain gateway - dnslink website gateway Tests cover: - etag/unicode support - breadcrumbs - file name column - hash column This commit was moved from ipfs/kubo@3ed46d995fee555f924b9163e491e37845ad8818 --- gateway/core/corehttp/gateway_handler.go | 4 ---- gateway/core/corehttp/gateway_indexPage.go | 4 ++-- gateway/core/corehttp/gateway_test.go | 12 +++++++++++- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/gateway/core/corehttp/gateway_handler.go b/gateway/core/corehttp/gateway_handler.go index e531640da..9af249520 100644 --- a/gateway/core/corehttp/gateway_handler.go +++ b/gateway/core/corehttp/gateway_handler.go @@ -405,10 +405,6 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request Hash: hash, } - // TODO: remove logging below - // tplDataJSON, _ := json.MarshalIndent(tplData, "", " ") - // fmt.Println(string(tplDataJSON)) - err = listingTemplate.Execute(w, tplData) if err != nil { internalWebError(w, err) diff --git a/gateway/core/corehttp/gateway_indexPage.go b/gateway/core/corehttp/gateway_indexPage.go index f735aa0e6..6e1722116 100644 --- a/gateway/core/corehttp/gateway_indexPage.go +++ b/gateway/core/corehttp/gateway_indexPage.go @@ -82,8 +82,8 @@ func shortHash(hash string) string { // (when hostname from gwURL is matching /ipns/ in path) func hasDNSLinkOrigin(gwURL string, path string) bool { if gwURL != "" { - dnslinkRoot := strings.Replace(gwURL, "//", "/ipns/", 1) - return strings.HasPrefix(path, dnslinkRoot) + fqdn := stripPort(strings.TrimPrefix(gwURL, "//")) + return strings.HasPrefix(path, "/ipns/"+fqdn) } return false } diff --git a/gateway/core/corehttp/gateway_test.go b/gateway/core/corehttp/gateway_test.go index 46bf76a5e..f98b4a773 100644 --- a/gateway/core/corehttp/gateway_test.go +++ b/gateway/core/corehttp/gateway_test.go @@ -393,6 +393,8 @@ func TestIPNSHostnameRedirect(t *testing.T) { // Test directory listing on DNSLink website // (scenario when Host header is the same as URL hostname) +// This is basic regression test: additional end-to-end tests +// can be found in test/sharness/t0115-gateway-dir-listing.sh func TestIPNSHostnameBacklinks(t *testing.T) { ns := mockNamesys{} ts, api, ctx := newTestServerAndNode(t, ns) @@ -439,7 +441,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) { t.Fatal(err) } - // expect correct backlinks + // expect correct links body, err := ioutil.ReadAll(res.Body) if err != nil { t.Fatalf("error reading response: %s", err) @@ -456,6 +458,10 @@ func TestIPNSHostnameBacklinks(t *testing.T) { if !strings.Contains(s, "") { t.Fatalf("expected file in directory listing") } + if !strings.Contains(s, "") { t.Fatalf("expected file in directory listing") } + if !strings.Contains(s, "