Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(gateway): _redirects file support #8816

Closed
wants to merge 8 commits into from
Closed

feat(gateway): _redirects file support #8816

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6152fa0
- implement basic redirect
cbrake Jan 11, 2022
0395527
- Update getOrHeadHandler to dispatch to getOrHeadHandlerUnixFs for U…
Apr 1, 2022
39e82fb
- More descriptive sharness test names
Apr 4, 2022
ce9e6cf
- More error handling
Apr 5, 2022
6a855f8
Cleanup function names
Apr 7, 2022
b7422d7
Wrap error with valuable context before returning
Apr 7, 2022
c47eb7b
More cleanup
Apr 7, 2022
9ef1863
Separate redirects logic from legacy redirect code
Apr 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
294 changes: 134 additions & 160 deletions core/corehttp/gateway_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,13 @@ import (
"context"
"fmt"
"html/template"
"io"
"mime"
"net/http"
"net/url"
"os"
gopath "path"
"regexp"
"runtime/debug"
"strconv"
"strings"
"time"

Expand All @@ -28,6 +26,7 @@ import (
prometheus "github.com/prometheus/client_golang/prometheus"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
)

const (
Expand Down Expand Up @@ -274,134 +273,75 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
logger := log.With("from", r.RequestURI)
logger.Debug("http request received")

// X-Ipfs-Gateway-Prefix was removed (https://github.com/ipfs/go-ipfs/issues/7702)
// TODO: remove this after go-ipfs 0.13 ships
if prfx := r.Header.Get("X-Ipfs-Gateway-Prefix"); prfx != "" {
err := fmt.Errorf("X-Ipfs-Gateway-Prefix support was removed: https://github.com/ipfs/go-ipfs/issues/7702")
webError(w, "unsupported HTTP header", err, http.StatusBadRequest)
if handledUnsupportedHeaders(w, r) {
return
}

// ?uri query param support for requests produced by web browsers
// via navigator.registerProtocolHandler Web API
// https://developer.mozilla.org/en-US/docs/Web/API/Navigator/registerProtocolHandler
// TLDR: redirect /ipfs/?uri=ipfs%3A%2F%2Fcid%3Fquery%3Dval to /ipfs/cid?query=val
if uriParam := r.URL.Query().Get("uri"); uriParam != "" {
u, err := url.Parse(uriParam)
if err != nil {
webError(w, "failed to parse uri query parameter", err, http.StatusBadRequest)
return
}
if u.Scheme != "ipfs" && u.Scheme != "ipns" {
webError(w, "uri query parameter scheme must be ipfs or ipns", err, http.StatusBadRequest)
return
}
path := u.Path
if u.RawQuery != "" { // preserve query if present
path = path + "?" + u.RawQuery
}

redirectURL := gopath.Join("/", u.Scheme, u.Host, path)
logger.Debugw("uri param, redirect", "to", redirectURL, "status", http.StatusMovedPermanently)
http.Redirect(w, r, redirectURL, http.StatusMovedPermanently)
if handledProtocolHandlerRedirect(w, r, logger) {
return
}

// Service Worker registration request
if r.Header.Get("Service-Worker") == "script" {
// Disallow Service Worker registration on namespace roots
// https://github.com/ipfs/go-ipfs/issues/4025
matched, _ := regexp.MatchString(`^/ip[fn]s/[^/]+$`, r.URL.Path)
if matched {
err := fmt.Errorf("registration is not allowed for this scope")
webError(w, "navigator.serviceWorker", err, http.StatusBadRequest)
return
}
if handledInvalidServiceWorkerRegistration(w, r) {
return
}

contentPath := ipath.New(r.URL.Path)
if pathErr := contentPath.IsValid(); pathErr != nil {
if fixupSuperfluousNamespace(w, r.URL.Path, r.URL.RawQuery) {
// the error was due to redundant namespace, which we were able to fix
// by returning error/redirect page, nothing left to do here
logger.Debugw("redundant namespace; noop")
return
}
// unable to fix path, returning error
webError(w, "invalid ipfs path", pathErr, http.StatusBadRequest)
if handledSuperfluousNamespaces(w, r, contentPath, logger) {
return
}

// Detect when explicit Accept header or ?format parameter are present
responseFormat, formatParams, err := customResponseFormat(r)
if err != nil {
webError(w, "error while processing the Accept header", err, http.StatusBadRequest)
return
}
trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResponseFormat", responseFormat))

// For Unixfs, when a path can't be resolved we need to check for redirects and pretty 404 page files.
if responseFormat == "" {
logger.Debugw("dispatching to getOrHeadHandlerUnixfs")
i.getOrHeadHandlerUnixfs(w, r, begin, logger)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Had to dispatch to getOrHeadHandlerUnixfs here, to avoid any early 404s, which would then short circuit the _redirects lookup.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a bit invasive change, duplicates code paths which are mostly the same.
This makes it harder to maintain, reason about metrics etc.

Could we keep the old way where we had a single getOrHeadHandler?
See my idea in https://github.com/ipfs/go-ipfs/pull/8816/files#r840944672

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This link just takes me to my comment "Moved pretty 404 logic over to gateway_handler_unixfs.go". Is that what you intended to link me?

return
}

// Resolve path to the final DAG node for the ETag
resolvedPath, err := i.api.ResolvePath(r.Context(), contentPath)
trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResolvedPath", resolvedPath.String()))

switch err {
case nil:
case coreiface.ErrOffline:
webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusServiceUnavailable)
return
default:
// if Accept is text/html, see if ipfs-404.html is present
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Moved pretty 404 logic over to gateway_handler_unixfs.go

if i.servePretty404IfPresent(w, r, contentPath) {
logger.Debugw("serve pretty 404 if present")
return
}

webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusNotFound)
return
}

// Detect when explicit Accept header or ?format parameter are present
responseFormat, formatParams, err := customResponseFormat(r)
if err != nil {
webError(w, "error while processing the Accept header", err, http.StatusBadRequest)
if i.returnedNotModifiedForMatchingETag(w, r, resolvedPath) {
return
}
trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResponseFormat", responseFormat))
trace.SpanFromContext(r.Context()).SetAttributes(attribute.String("ResolvedPath", resolvedPath.String()))

// Finish early if client already has matching Etag
if r.Header.Get("If-None-Match") == getEtag(r, resolvedPath.Cid()) {
w.WriteHeader(http.StatusNotModified)
return
}

// Update the global metric of the time it takes to read the final root block of the requested resource
// NOTE: for legacy reasons this happens before we go into content-type specific code paths
_, err = i.api.Block().Get(r.Context(), resolvedPath)
if err != nil {
webError(w, "ipfs block get "+resolvedPath.Cid().String(), err, http.StatusInternalServerError)
if !i.updateFirstContentBlockMetrics(w, r, begin, contentPath, resolvedPath) {
return
}
ns := contentPath.Namespace()
timeToGetFirstContentBlock := time.Since(begin).Seconds()
i.unixfsGetMetric.WithLabelValues(ns).Observe(timeToGetFirstContentBlock) // deprecated, use firstContentBlockGetMetric instead
i.firstContentBlockGetMetric.WithLabelValues(ns).Observe(timeToGetFirstContentBlock)

// HTTP Headers
i.addUserHeaders(w) // ok, _now_ write user's headers.
w.Header().Set("X-Ipfs-Path", contentPath.String())

if rootCids, err := i.buildIpfsRootsHeader(contentPath.String(), r); err == nil {
w.Header().Set("X-Ipfs-Roots", rootCids)
} else { // this should never happen, as we resolved the contentPath already
webError(w, "error while resolving X-Ipfs-Roots", err, http.StatusInternalServerError)
if !i.setHeaders(w, r, contentPath) {
return
}

// Support custom response formats passed via ?format or Accept HTTP header
// Note that we handle Unixfs (e.g. responseFormat of "") above.
switch responseFormat {
case "": // The implicit response format is UnixFS
logger.Debugw("serving unixfs", "path", contentPath)
i.serveUnixFs(w, r, resolvedPath, contentPath, begin, logger)
return
case "application/vnd.ipld.raw":
logger.Debugw("serving raw block", "path", contentPath)
i.serveRawBlock(w, r, resolvedPath, contentPath, begin)
return
case "application/vnd.ipld.car":
logger.Debugw("serving car stream", "path", contentPath)
carVersion := formatParams["version"]
i.serveCar(w, r, resolvedPath, contentPath, carVersion, begin)
i.serveCar(w, r, resolvedPath, contentPath, carVersion, begin)
return
default: // catch-all for unsuported application/vnd.*
err := fmt.Errorf("unsupported format %q", responseFormat)
Expand All @@ -410,36 +350,6 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
}
}

func (i *gatewayHandler) servePretty404IfPresent(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) bool {
resolved404Path, ctype, err := i.searchUpTreeFor404(r, contentPath)
if err != nil {
return false
}

dr, err := i.api.Unixfs().Get(r.Context(), resolved404Path)
if err != nil {
return false
}
defer dr.Close()

f, ok := dr.(files.File)
if !ok {
return false
}

size, err := f.Size()
if err != nil {
return false
}

log.Debugw("using pretty 404 file", "path", contentPath)
w.Header().Set("Content-Type", ctype)
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
w.WriteHeader(http.StatusNotFound)
_, err = io.CopyN(w, f, size)
return err == nil
}

func (i *gatewayHandler) postHandler(w http.ResponseWriter, r *http.Request) {
p, err := i.api.Unixfs().Add(r.Context(), files.NewReaderFile(r.Body))
if err != nil {
Expand Down Expand Up @@ -809,48 +719,6 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string]
return "", nil, nil
}

func (i *gatewayHandler) searchUpTreeFor404(r *http.Request, contentPath ipath.Path) (ipath.Resolved, string, error) {
filename404, ctype, err := preferred404Filename(r.Header.Values("Accept"))
if err != nil {
return nil, "", err
}

pathComponents := strings.Split(contentPath.String(), "/")

for idx := len(pathComponents); idx >= 3; idx-- {
pretty404 := gopath.Join(append(pathComponents[0:idx], filename404)...)
parsed404Path := ipath.New("/" + pretty404)
if parsed404Path.IsValid() != nil {
break
}
resolvedPath, err := i.api.ResolvePath(r.Context(), parsed404Path)
if err != nil {
continue
}
return resolvedPath, ctype, nil
}

return nil, "", fmt.Errorf("no pretty 404 in any parent folder")
}

func preferred404Filename(acceptHeaders []string) (string, string, error) {
// If we ever want to offer a 404 file for a different content type
// then this function will need to parse q weightings, but for now
// the presence of anything matching HTML is enough.
for _, acceptHeader := range acceptHeaders {
accepted := strings.Split(acceptHeader, ",")
for _, spec := range accepted {
contentType := strings.SplitN(spec, ";", 1)[0]
switch contentType {
case "*/*", "text/*", "text/html":
return "ipfs-404.html", "text/html", nil
}
}
}

return "", "", fmt.Errorf("there is no 404 file for the requested content types")
}

// returns unquoted path with all special characters revealed as \u codes
func debugStr(path string) string {
q := fmt.Sprintf("%+q", path)
Expand Down Expand Up @@ -889,3 +757,109 @@ func fixupSuperfluousNamespace(w http.ResponseWriter, urlPath string, urlQuery s
ErrorMsg: fmt.Sprintf("invalid path: %q should be %q", urlPath, intendedPath.String()),
}) == nil
}

func handledUnsupportedHeaders(w http.ResponseWriter, r *http.Request) bool {
// X-Ipfs-Gateway-Prefix was removed (https://github.com/ipfs/go-ipfs/issues/7702)
// TODO: remove this after go-ipfs 0.13 ships
if prfx := r.Header.Get("X-Ipfs-Gateway-Prefix"); prfx != "" {
err := fmt.Errorf("X-Ipfs-Gateway-Prefix support was removed: https://github.com/ipfs/go-ipfs/issues/7702")
webError(w, "unsupported HTTP header", err, http.StatusBadRequest)
return true
} else {
return false
}
}

func handledProtocolHandlerRedirect(w http.ResponseWriter, r *http.Request, logger *zap.SugaredLogger) bool {
if uriParam := r.URL.Query().Get("uri"); uriParam != "" {
u, err := url.Parse(uriParam)
if err != nil {
webError(w, "failed to parse uri query parameter", err, http.StatusBadRequest)
return true
}
if u.Scheme != "ipfs" && u.Scheme != "ipns" {
webError(w, "uri query parameter scheme must be ipfs or ipns", err, http.StatusBadRequest)
return true
}
path := u.Path
if u.RawQuery != "" { // preserve query if present
path = path + "?" + u.RawQuery
}

redirectURL := gopath.Join("/", u.Scheme, u.Host, path)
logger.Debugw("uri param, redirect", "to", redirectURL, "status", http.StatusMovedPermanently)
http.Redirect(w, r, redirectURL, http.StatusMovedPermanently)
return true
}

return false
}

// Disallow Service Worker registration on namespace roots
// https://github.com/ipfs/go-ipfs/issues/4025
func handledInvalidServiceWorkerRegistration(w http.ResponseWriter, r *http.Request) bool {
if r.Header.Get("Service-Worker") == "script" {
matched, _ := regexp.MatchString(`^/ip[fn]s/[^/]+$`, r.URL.Path)
if matched {
err := fmt.Errorf("registration is not allowed for this scope")
webError(w, "navigator.serviceWorker", err, http.StatusBadRequest)
return true
}
}

return false
}

func handledSuperfluousNamespaces(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, logger *zap.SugaredLogger) bool {
if pathErr := contentPath.IsValid(); pathErr != nil {
if fixupSuperfluousNamespace(w, r.URL.Path, r.URL.RawQuery) {
// the error was due to redundant namespace, which we were able to fix
// by returning error/redirect page, nothing left to do here
logger.Debugw("redundant namespace; noop")
return true
}
// unable to fix path, returning error
webError(w, "invalid ipfs path", pathErr, http.StatusBadRequest)
return true
}
return false
}

func (i *gatewayHandler) returnedNotModifiedForMatchingETag(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved) bool {
// Finish early if client already has matching Etag
if r.Header.Get("If-None-Match") == getEtag(r, resolvedPath.Cid()) {
w.WriteHeader(http.StatusNotModified)
return true
}

return false
}

// Update the global metric of the time it takes to read the final root block of the requested resource
func (i *gatewayHandler) updateFirstContentBlockMetrics(w http.ResponseWriter, r *http.Request, begin time.Time, contentPath ipath.Path, resolvedPath ipath.Resolved) bool {
// NOTE: for legacy reasons this happens before we go into content-type specific code paths
_, err := i.api.Block().Get(r.Context(), resolvedPath)
if err != nil {
webError(w, "ipfs block get "+resolvedPath.Cid().String(), err, http.StatusInternalServerError)
return false
}
ns := contentPath.Namespace()
timeToGetFirstContentBlock := time.Since(begin).Seconds()
i.unixfsGetMetric.WithLabelValues(ns).Observe(timeToGetFirstContentBlock) // deprecated, use firstContentBlockGetMetric instead
i.firstContentBlockGetMetric.WithLabelValues(ns).Observe(timeToGetFirstContentBlock)
return true
}

func (i *gatewayHandler) setHeaders(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) bool {
i.addUserHeaders(w) // ok, _now_ write user's headers.
w.Header().Set("X-Ipfs-Path", contentPath.String())

if rootCids, err := i.buildIpfsRootsHeader(contentPath.String(), r); err == nil {
w.Header().Set("X-Ipfs-Roots", rootCids)
} else { // this should never happen, as we resolved the contentPath already
webError(w, "error while resolving X-Ipfs-Roots", err, http.StatusInternalServerError)
return false
}

return true
}
Loading