From 2dc6eb6e9adcec5f6764285def66822086560c55 Mon Sep 17 00:00:00 2001 From: Jes Cok Date: Tue, 20 Jun 2023 11:35:10 +0000 Subject: [PATCH 01/93] net/http: declare publicErr as a constant Do the same as the code above: "case err == errTooLarge", declare publicErr as a constant to avoid runtime calls. Change-Id: I50a9951232c70eff027b0da86c0bbb8bea51acbe GitHub-Last-Rev: 71d4458ded3a1e99a0d027ccca6c9d6269a1ab06 GitHub-Pull-Request: golang/go#60884 Reviewed-on: https://go-review.googlesource.com/c/go/+/504456 Reviewed-by: Damien Neil Reviewed-by: Ian Lance Taylor TryBot-Result: Gopher Robot Run-TryBot: Damien Neil Reviewed-by: Olif Oftimis --- server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server.go b/server.go index 8f63a902..29e862d8 100644 --- a/server.go +++ b/server.go @@ -1971,7 +1971,7 @@ func (c *conn) serve(ctx context.Context) { fmt.Fprintf(c.rwc, "HTTP/1.1 %d %s: %s%s%d %s: %s", v.code, StatusText(v.code), v.text, errorHeaders, v.code, StatusText(v.code), v.text) return } - publicErr := "400 Bad Request" + const publicErr = "400 Bad Request" fmt.Fprintf(c.rwc, "HTTP/1.1 "+publicErr+errorHeaders+publicErr) return } From 7c6c2e4ea98b0032324e3649602b42d3f1d853e1 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Mon, 24 Jul 2023 16:19:49 +0000 Subject: [PATCH 02/93] Revert "net/http: use Copy in ServeContent if CopyN not needed" This reverts CL 446276. Reason for revert: Causing surprising performance regression. Fixes #61530 Change-Id: Ic970f2e05d875b606ce274ea621f7e4c8c337481 Reviewed-on: https://go-review.googlesource.com/c/go/+/512615 Run-TryBot: Damien Neil Reviewed-by: Bryan Mills TryBot-Result: Gopher Robot --- fs.go | 9 ++------- fs_test.go | 43 ++----------------------------------------- 2 files changed, 4 insertions(+), 48 deletions(-) diff --git a/fs.go b/fs.go index 55094400..41e0b43a 100644 --- a/fs.go +++ b/fs.go @@ -349,13 +349,8 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, w.WriteHeader(code) - if r.Method != MethodHead { - if sendSize == size { - // use Copy in the non-range case to make use of WriterTo if available - io.Copy(w, sendContent) - } else { - io.CopyN(w, sendContent, sendSize) - } + if r.Method != "HEAD" { + io.CopyN(w, sendContent, sendSize) } } diff --git a/fs_test.go b/fs_test.go index e37e0f04..3fb9e012 100644 --- a/fs_test.go +++ b/fs_test.go @@ -924,7 +924,6 @@ func testServeContent(t *testing.T, mode testMode) { wantContentType string wantContentRange string wantStatus int - wantContent []byte } htmlModTime := mustStat(t, "testdata/index.html").ModTime() tests := map[string]testCase{ @@ -1140,24 +1139,6 @@ func testServeContent(t *testing.T, mode testMode) { wantStatus: 412, wantLastMod: htmlModTime.UTC().Format(TimeFormat), }, - "uses_writeTo_if_available_and_non-range": { - content: &panicOnNonWriterTo{seekWriterTo: strings.NewReader("foobar")}, - serveContentType: "text/plain; charset=utf-8", - wantContentType: "text/plain; charset=utf-8", - wantStatus: StatusOK, - wantContent: []byte("foobar"), - }, - "do_not_use_writeTo_for_range_requests": { - content: &panicOnWriterTo{ReadSeeker: strings.NewReader("foobar")}, - serveContentType: "text/plain; charset=utf-8", - reqHeader: map[string]string{ - "Range": "bytes=0-4", - }, - wantContentType: "text/plain; charset=utf-8", - wantContentRange: "bytes 0-4/6", - wantStatus: StatusPartialContent, - wantContent: []byte("fooba"), - }, } for testName, tt := range tests { var content io.ReadSeeker @@ -1171,8 +1152,7 @@ func testServeContent(t *testing.T, mode testMode) { } else { content = tt.content } - contentOut := &strings.Builder{} - for _, method := range []string{MethodGet, MethodHead} { + for _, method := range []string{"GET", "HEAD"} { //restore content in case it is consumed by previous method if content, ok := content.(*strings.Reader); ok { content.Seek(0, io.SeekStart) @@ -1198,8 +1178,7 @@ func testServeContent(t *testing.T, mode testMode) { if err != nil { t.Fatal(err) } - contentOut.Reset() - io.Copy(contentOut, res.Body) + io.Copy(io.Discard, res.Body) res.Body.Close() if res.StatusCode != tt.wantStatus { t.Errorf("test %q using %q: got status = %d; want %d", testName, method, res.StatusCode, tt.wantStatus) @@ -1213,28 +1192,10 @@ func testServeContent(t *testing.T, mode testMode) { if g, e := res.Header.Get("Last-Modified"), tt.wantLastMod; g != e { t.Errorf("test %q using %q: got last-modified = %q, want %q", testName, method, g, e) } - if g, e := contentOut.String(), tt.wantContent; e != nil && method == MethodGet && g != string(e) { - t.Errorf("test %q using %q: got unexpected content %q, want %q", testName, method, g, e) - } } } } -type seekWriterTo interface { - io.Seeker - io.WriterTo -} - -type panicOnNonWriterTo struct { - io.Reader - seekWriterTo -} - -type panicOnWriterTo struct { - io.ReadSeeker - io.WriterTo -} - // Issue 12991 func TestServerFileStatError(t *testing.T) { rec := httptest.NewRecorder() From e7c7b52b55d32bb501f7e952395a47d64ad22bf0 Mon Sep 17 00:00:00 2001 From: Jes Cok Date: Tue, 25 Jul 2023 23:10:34 +0000 Subject: [PATCH 03/93] all: use built-in clear to clear maps Change-Id: I7f4ac72fe3230d8b7486fab0c925015cefcbe355 GitHub-Last-Rev: 54455839b674f980fb6c3afceb433db4833d340e GitHub-Pull-Request: golang/go#61544 Reviewed-on: https://go-review.googlesource.com/c/go/+/512376 Reviewed-by: Ian Lance Taylor Run-TryBot: Keith Randall Reviewed-by: Bryan Mills Run-TryBot: Ian Lance Taylor Auto-Submit: Ian Lance Taylor TryBot-Result: Gopher Robot --- httputil/reverseproxy.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/httputil/reverseproxy.go b/httputil/reverseproxy.go index 2a76b0b8..719ab62d 100644 --- a/httputil/reverseproxy.go +++ b/httputil/reverseproxy.go @@ -461,10 +461,7 @@ func (p *ReverseProxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) { rw.WriteHeader(code) // Clear headers, it's not automatically done by ResponseWriter.WriteHeader() for 1xx responses - for k := range h { - delete(h, k) - } - + clear(h) return nil }, } From 581f2393119ad7620623c247fb4b995bda835a5b Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Sun, 23 Jul 2023 14:19:57 -0400 Subject: [PATCH 04/93] net/http: perform streaming body feature detection once per process As far a I can tell, there's currently no situation where this feature detection will report a different result per request, so default to doing once per process until there's evidence that doing it more often is worthwhile. Change-Id: I567d3dbd847af2f49f2e83cd9eb0ae61d82c1f83 Reviewed-on: https://go-review.googlesource.com/c/go/+/513459 Reviewed-by: Matthew Dempsky Run-TryBot: Johan Brandhorst-Satzkorn TryBot-Result: Gopher Robot Reviewed-by: Dmitri Shuralyov Reviewed-by: Johan Brandhorst-Satzkorn --- roundtrip_js.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roundtrip_js.go b/roundtrip_js.go index 2826383c..dd9efe51 100644 --- a/roundtrip_js.go +++ b/roundtrip_js.go @@ -12,6 +12,7 @@ import ( "io" "strconv" "strings" + "sync" "syscall/js" ) @@ -57,7 +58,7 @@ var jsFetchDisabled = js.Global().Get("process").Type() == js.TypeObject && // Determine whether the JS runtime supports streaming request bodies. // Courtesy: https://developer.chrome.com/articles/fetch-streaming-requests/#feature-detection -func supportsPostRequestStreams() bool { +var supportsPostRequestStreams = sync.OnceValue(func() bool { requestOpt := js.Global().Get("Object").New() requestBody := js.Global().Get("ReadableStream").New() @@ -85,7 +86,7 @@ func supportsPostRequestStreams() bool { hasContentTypeHeader := requestObject.Get("headers").Call("has", "Content-Type").Bool() return duplexCalled && !hasContentTypeHeader -} +}) // RoundTrip implements the RoundTripper interface using the WHATWG Fetch API. func (t *Transport) RoundTrip(req *Request) (*Response, error) { From 6374140a90ff011ee4d48fc9ef7da738f8243331 Mon Sep 17 00:00:00 2001 From: Eduard Bondarenko Date: Wed, 26 Jul 2023 16:28:28 +0000 Subject: [PATCH 05/93] net/http: fix doc comment on FormValue function This function checks Request.Form, which now includes values parsed from a PATCH request. Fixes #60585 Change-Id: Icb095d9ac2f8b0c5dbf313e507ed838cb941517f GitHub-Last-Rev: 3a477ea97e27f5b31d28085df75163fc13541c13 GitHub-Pull-Request: golang/go#61591 Reviewed-on: https://go-review.googlesource.com/c/go/+/513435 Reviewed-by: David Chase Reviewed-by: Damien Neil Run-TryBot: Damien Neil TryBot-Result: Gopher Robot Auto-Submit: Damien Neil --- request.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/request.go b/request.go index bd868373..a2e8373d 100644 --- a/request.go +++ b/request.go @@ -1339,7 +1339,7 @@ func (r *Request) ParseMultipartForm(maxMemory int64) error { } // FormValue returns the first value for the named component of the query. -// POST and PUT body parameters take precedence over URL query string values. +// POST, PUT, and PATCH body parameters take precedence over URL query string values. // FormValue calls ParseMultipartForm and ParseForm if necessary and ignores // any errors returned by these functions. // If key is not present, FormValue returns the empty string. @@ -1356,7 +1356,7 @@ func (r *Request) FormValue(key string) string { } // PostFormValue returns the first value for the named component of the POST, -// PATCH, or PUT request body. URL query parameters are ignored. +// PUT, or PATCH request body. URL query parameters are ignored. // PostFormValue calls ParseMultipartForm and ParseForm if necessary and ignores // any errors returned by these functions. // If key is not present, PostFormValue returns the empty string. From 4ed35642e7afdef65eb536a28a2985bd11c37f0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Fri, 7 Jul 2023 11:06:05 +0200 Subject: [PATCH 06/93] all: add a few more godoc links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Over the past few months as I read the standard library's documentation I kept finding spots where godoc links would have helped me. I kept adding to a stash of changes to fix them up bit by bit. The stash has grown big enough by now, and we're nearing a release, so I think it's time to merge to avoid git conflicts or bit rot. Note that a few sentences are slightly reworded, since "implements the Fooer interface" can just be "implements [Fooer]" now that the link provides all the context needed to the user. Change-Id: I01c31d3d3ff066d06aeb44f545f8dd0fb9a8d998 Reviewed-on: https://go-review.googlesource.com/c/go/+/508395 Run-TryBot: Daniel Martí TryBot-Result: Gopher Robot Reviewed-by: Michael Knyszek Reviewed-by: Ian Lance Taylor --- server.go | 51 ++++++++++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 27 deletions(-) diff --git a/server.go b/server.go index 29e862d8..0d75b877 100644 --- a/server.go +++ b/server.go @@ -61,16 +61,16 @@ var ( // A Handler responds to an HTTP request. // -// ServeHTTP should write reply headers and data to the ResponseWriter +// ServeHTTP should write reply headers and data to the [ResponseWriter] // and then return. Returning signals that the request is finished; it -// is not valid to use the ResponseWriter or read from the -// Request.Body after or concurrently with the completion of the +// is not valid to use the [ResponseWriter] or read from the +// [Request.Body] after or concurrently with the completion of the // ServeHTTP call. // // Depending on the HTTP client software, HTTP protocol version, and // any intermediaries between the client and the Go server, it may not -// be possible to read from the Request.Body after writing to the -// ResponseWriter. Cautious handlers should read the Request.Body +// be possible to read from the [Request.Body] after writing to the +// [ResponseWriter]. Cautious handlers should read the [Request.Body] // first, and then reply. // // Except for reading the body, handlers should not modify the @@ -82,7 +82,7 @@ var ( // and either closes the network connection or sends an HTTP/2 // RST_STREAM, depending on the HTTP protocol. To abort a handler so // the client sees an interrupted response but the server doesn't log -// an error, panic with the value ErrAbortHandler. +// an error, panic with the value [ErrAbortHandler]. type Handler interface { ServeHTTP(ResponseWriter, *Request) } @@ -90,15 +90,14 @@ type Handler interface { // A ResponseWriter interface is used by an HTTP handler to // construct an HTTP response. // -// A ResponseWriter may not be used after the Handler.ServeHTTP method -// has returned. +// A ResponseWriter may not be used after [Handler.ServeHTTP] has returned. type ResponseWriter interface { // Header returns the header map that will be sent by - // WriteHeader. The Header map also is the mechanism with which - // Handlers can set HTTP trailers. + // [ResponseWriter.WriteHeader]. The [Header] map also is the mechanism with which + // [Handler] implementations can set HTTP trailers. // - // Changing the header map after a call to WriteHeader (or - // Write) has no effect unless the HTTP status code was of the + // Changing the header map after a call to [ResponseWriter.WriteHeader] (or + // [ResponseWriter.Write]) has no effect unless the HTTP status code was of the // 1xx class or the modified headers are trailers. // // There are two ways to set Trailers. The preferred way is to @@ -107,9 +106,9 @@ type ResponseWriter interface { // trailer keys which will come later. In this case, those // keys of the Header map are treated as if they were // trailers. See the example. The second way, for trailer - // keys not known to the Handler until after the first Write, - // is to prefix the Header map keys with the TrailerPrefix - // constant value. See TrailerPrefix. + // keys not known to the [Handler] until after the first [ResponseWriter.Write], + // is to prefix the [Header] map keys with the [TrailerPrefix] + // constant value. // // To suppress automatic response headers (such as "Date"), set // their value to nil. @@ -117,11 +116,11 @@ type ResponseWriter interface { // Write writes the data to the connection as part of an HTTP reply. // - // If WriteHeader has not yet been called, Write calls + // If [ResponseWriter.WriteHeader] has not yet been called, Write calls // WriteHeader(http.StatusOK) before writing the data. If the Header // does not contain a Content-Type line, Write adds a Content-Type set // to the result of passing the initial 512 bytes of written data to - // DetectContentType. Additionally, if the total size of all written + // [DetectContentType]. Additionally, if the total size of all written // data is under a few KB and there are no Flush calls, the // Content-Length header is added automatically. // @@ -2567,14 +2566,12 @@ func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Re mux.Handle(pattern, HandlerFunc(handler)) } -// Handle registers the handler for the given pattern -// in the DefaultServeMux. -// The documentation for ServeMux explains how patterns are matched. +// Handle registers the handler for the given pattern in [DefaultServeMux]. +// The documentation for [ServeMux] explains how patterns are matched. func Handle(pattern string, handler Handler) { DefaultServeMux.Handle(pattern, handler) } -// HandleFunc registers the handler function for the given pattern -// in the DefaultServeMux. -// The documentation for ServeMux explains how patterns are matched. +// HandleFunc registers the handler function for the given pattern in [DefaultServeMux]. +// The documentation for [ServeMux] explains how patterns are matched. func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { DefaultServeMux.HandleFunc(pattern, handler) } @@ -2583,7 +2580,7 @@ func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { // creating a new service goroutine for each. The service goroutines // read requests and then call handler to reply to them. // -// The handler is typically nil, in which case the DefaultServeMux is used. +// The handler is typically nil, in which case [DefaultServeMux] is used. // // HTTP/2 support is only enabled if the Listener returns *tls.Conn // connections and they were configured with "h2" in the TLS @@ -2599,7 +2596,7 @@ func Serve(l net.Listener, handler Handler) error { // creating a new service goroutine for each. The service goroutines // read requests and then call handler to reply to them. // -// The handler is typically nil, in which case the DefaultServeMux is used. +// The handler is typically nil, in which case [DefaultServeMux] is used. // // Additionally, files containing a certificate and matching private key // for the server must be provided. If the certificate is signed by a @@ -3231,7 +3228,7 @@ func logf(r *Request, format string, args ...any) { // Serve with handler to handle requests on incoming connections. // Accepted connections are configured to enable TCP keep-alives. // -// The handler is typically nil, in which case the DefaultServeMux is used. +// The handler is typically nil, in which case [DefaultServeMux] is used. // // ListenAndServe always returns a non-nil error. func ListenAndServe(addr string, handler Handler) error { @@ -3239,7 +3236,7 @@ func ListenAndServe(addr string, handler Handler) error { return server.ListenAndServe() } -// ListenAndServeTLS acts identically to ListenAndServe, except that it +// ListenAndServeTLS acts identically to [ListenAndServe], except that it // expects HTTPS connections. Additionally, files containing a certificate and // matching private key for the server must be provided. If the certificate // is signed by a certificate authority, the certFile should be the concatenation From deb5e094cc54121c316c0194a9efc5b842398e8c Mon Sep 17 00:00:00 2001 From: Mauri de Souza Meneguzzo Date: Mon, 31 Jul 2023 20:58:45 +0000 Subject: [PATCH 07/93] net/http: add ServeFileFS, FileServerFS, NewFileTransportFS These new apis are analogous to ServeFile, FileServer and NewFileTransport respectively. The main difference is that these functions operate on an fs.FS. Fixes #51971 Change-Id: Ie56b245b795eeb7edf613657578592306945469b GitHub-Last-Rev: 26e75c0368f155a2299fbdcb72f47036b71a5e06 GitHub-Pull-Request: golang/go#61641 Reviewed-on: https://go-review.googlesource.com/c/go/+/513956 Run-TryBot: Damien Neil Reviewed-by: David Chase TryBot-Result: Gopher Robot Reviewed-by: Damien Neil --- filetransport.go | 19 ++++++++++++++++ filetransport_test.go | 42 ++++++++++++++++++++++++++++++++++++ fs.go | 50 ++++++++++++++++++++++++++++++++++++++++--- fs_test.go | 49 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 3 deletions(-) diff --git a/filetransport.go b/filetransport.go index 94684b07..2a9e9b02 100644 --- a/filetransport.go +++ b/filetransport.go @@ -7,6 +7,7 @@ package http import ( "fmt" "io" + "io/fs" ) // fileTransport implements RoundTripper for the 'file' protocol. @@ -31,6 +32,24 @@ func NewFileTransport(fs FileSystem) RoundTripper { return fileTransport{fileHandler{fs}} } +// NewFileTransportFS returns a new RoundTripper, serving the provided +// file system fsys. The returned RoundTripper ignores the URL host in its +// incoming requests, as well as most other properties of the +// request. +// +// The typical use case for NewFileTransportFS is to register the "file" +// protocol with a Transport, as in: +// +// fsys := os.DirFS("/") +// t := &http.Transport{} +// t.RegisterProtocol("file", http.NewFileTransportFS(fsys)) +// c := &http.Client{Transport: t} +// res, err := c.Get("file:///etc/passwd") +// ... +func NewFileTransportFS(fsys fs.FS) RoundTripper { + return NewFileTransport(FS(fsys)) +} + func (t fileTransport) RoundTrip(req *Request) (resp *Response, err error) { // We start ServeHTTP in a goroutine, which may take a long // time if the file is large. The newPopulateResponseWriter diff --git a/filetransport_test.go b/filetransport_test.go index 77fc8eec..b3e3301e 100644 --- a/filetransport_test.go +++ b/filetransport_test.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "testing" + "testing/fstest" ) func checker(t *testing.T) func(string, error) { @@ -62,3 +63,44 @@ func TestFileTransport(t *testing.T) { } res.Body.Close() } + +func TestFileTransportFS(t *testing.T) { + check := checker(t) + + fsys := fstest.MapFS{ + "index.html": {Data: []byte("index.html says hello")}, + } + + tr := &Transport{} + tr.RegisterProtocol("file", NewFileTransportFS(fsys)) + c := &Client{Transport: tr} + + for fname, mfile := range fsys { + urlstr := "file:///" + fname + res, err := c.Get(urlstr) + check("Get "+urlstr, err) + if res.StatusCode != 200 { + t.Errorf("for %s, StatusCode = %d, want 200", urlstr, res.StatusCode) + } + if res.ContentLength != -1 { + t.Errorf("for %s, ContentLength = %d, want -1", urlstr, res.ContentLength) + } + if res.Body == nil { + t.Fatalf("for %s, nil Body", urlstr) + } + slurp, err := io.ReadAll(res.Body) + res.Body.Close() + check("ReadAll "+urlstr, err) + if string(slurp) != string(mfile.Data) { + t.Errorf("for %s, got content %q, want %q", urlstr, string(slurp), "Bar") + } + } + + const badURL = "file://../no-exist.txt" + res, err := c.Get(badURL) + check("Get "+badURL, err) + if res.StatusCode != 404 { + t.Errorf("for %s, StatusCode = %d, want 404", badURL, res.StatusCode) + } + res.Body.Close() +} diff --git a/fs.go b/fs.go index 41e0b43a..c605fe3a 100644 --- a/fs.go +++ b/fs.go @@ -741,6 +741,40 @@ func ServeFile(w ResponseWriter, r *Request, name string) { serveFile(w, r, Dir(dir), file, false) } +// ServeFileFS replies to the request with the contents +// of the named file or directory from the file system fsys. +// +// If the provided file or directory name is a relative path, it is +// interpreted relative to the current directory and may ascend to +// parent directories. If the provided name is constructed from user +// input, it should be sanitized before calling ServeFile. +// +// As a precaution, ServeFile will reject requests where r.URL.Path +// contains a ".." path element; this protects against callers who +// might unsafely use filepath.Join on r.URL.Path without sanitizing +// it and then use that filepath.Join result as the name argument. +// +// As another special case, ServeFile redirects any request where r.URL.Path +// ends in "/index.html" to the same path, without the final +// "index.html". To avoid such redirects either modify the path or +// use ServeContent. +// +// Outside of those two special cases, ServeFile does not use +// r.URL.Path for selecting the file or directory to serve; only the +// file or directory provided in the name argument is used. +func ServeFileFS(w ResponseWriter, r *Request, fsys fs.FS, name string) { + if containsDotDot(r.URL.Path) { + // Too many programs use r.URL.Path to construct the argument to + // serveFile. Reject the request under the assumption that happened + // here and ".." may not be wanted. + // Note that name might not contain "..", for example if code (still + // incorrectly) used filepath.Join(myDir, r.URL.Path). + Error(w, "invalid URL path", StatusBadRequest) + return + } + serveFile(w, r, FS(fsys), name, false) +} + func containsDotDot(v string) bool { if !strings.Contains(v, "..") { return false @@ -850,13 +884,23 @@ func FS(fsys fs.FS) FileSystem { // // http.Handle("/", http.FileServer(http.Dir("/tmp"))) // -// To use an fs.FS implementation, use http.FS to convert it: -// -// http.Handle("/", http.FileServer(http.FS(fsys))) +// To use an fs.FS implementation, use http.FileServerFS instead. func FileServer(root FileSystem) Handler { return &fileHandler{root} } +// FileServerFS returns a handler that serves HTTP requests +// with the contents of the file system fsys. +// +// As a special case, the returned file server redirects any request +// ending in "/index.html" to the same path, without the final +// "index.html". +// +// http.Handle("/", http.FileServerFS(fsys)) +func FileServerFS(root fs.FS) Handler { + return FileServer(FS(root)) +} + func (f *fileHandler) ServeHTTP(w ResponseWriter, r *Request) { upath := r.URL.Path if !strings.HasPrefix(upath, "/") { diff --git a/fs_test.go b/fs_test.go index 3fb9e012..bb96d2ca 100644 --- a/fs_test.go +++ b/fs_test.go @@ -26,6 +26,7 @@ import ( "runtime" "strings" "testing" + "testing/fstest" "time" ) @@ -1559,3 +1560,51 @@ func testFileServerMethods(t *testing.T, mode testMode) { } } } + +func TestFileServerFS(t *testing.T) { + filename := "index.html" + contents := []byte("index.html says hello") + fsys := fstest.MapFS{ + filename: {Data: contents}, + } + ts := newClientServerTest(t, http1Mode, FileServerFS(fsys)).ts + defer ts.Close() + + res, err := ts.Client().Get(ts.URL + "/" + filename) + if err != nil { + t.Fatal(err) + } + b, err := io.ReadAll(res.Body) + if err != nil { + t.Fatal("reading Body:", err) + } + if s := string(b); s != string(contents) { + t.Errorf("for path %q got %q, want %q", filename, s, contents) + } + res.Body.Close() +} + +func TestServeFileFS(t *testing.T) { + filename := "index.html" + contents := []byte("index.html says hello") + fsys := fstest.MapFS{ + filename: {Data: contents}, + } + ts := newClientServerTest(t, http1Mode, HandlerFunc(func(w ResponseWriter, r *Request) { + ServeFileFS(w, r, fsys, filename) + })).ts + defer ts.Close() + + res, err := ts.Client().Get(ts.URL + "/" + filename) + if err != nil { + t.Fatal(err) + } + b, err := io.ReadAll(res.Body) + if err != nil { + t.Fatal("reading Body:", err) + } + if s := string(b); s != string(contents) { + t.Errorf("for path %q got %q, want %q", filename, s, contents) + } + res.Body.Close() +} From 141a4eae574eb76148ba78957fa4e822ff18baa2 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 19 Jul 2023 10:30:46 -0700 Subject: [PATCH 08/93] net/http: permit requests with invalid Host headers Historically, the Transport has silently truncated invalid Host headers at the first '/' or ' ' character. CL 506996 changed this behavior to reject invalid Host headers entirely. Unfortunately, Docker appears to rely on the previous behavior. When sending a HTTP/1 request with an invalid Host, send an empty Host header. This is safer than truncation: If you care about the Host, then you should get the one you set; if you don't care, then an empty Host should be fine. Continue to fully validate Host headers sent to a proxy, since proxies generally can't productively forward requests without a Host. For #60374 Fixes #61431 Change-Id: If170c7dd860aa20eb58fe32990fc93af832742b6 Reviewed-on: https://go-review.googlesource.com/c/go/+/511155 TryBot-Result: Gopher Robot Reviewed-by: Roland Shoemaker Run-TryBot: Damien Neil --- request.go | 23 ++++++++++++++++++++++- request_test.go | 17 ++++++++++++----- 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/request.go b/request.go index a2e8373d..d1fbd5df 100644 --- a/request.go +++ b/request.go @@ -591,8 +591,29 @@ func (r *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, waitF if err != nil { return err } + // Validate that the Host header is a valid header in general, + // but don't validate the host itself. This is sufficient to avoid + // header or request smuggling via the Host field. + // The server can (and will, if it's a net/http server) reject + // the request if it doesn't consider the host valid. if !httpguts.ValidHostHeader(host) { - return errors.New("http: invalid Host header") + // Historically, we would truncate the Host header after '/' or ' '. + // Some users have relied on this truncation to convert a network + // address such as Unix domain socket path into a valid, ignored + // Host header (see https://go.dev/issue/61431). + // + // We don't preserve the truncation, because sending an altered + // header field opens a smuggling vector. Instead, zero out the + // Host header entirely if it isn't valid. (An empty Host is valid; + // see RFC 9112 Section 3.2.) + // + // Return an error if we're sending to a proxy, since the proxy + // probably can't do anything useful with an empty Host header. + if !usingProxy { + host = "" + } else { + return errors.New("http: invalid Host header") + } } // According to RFC 6874, an HTTP client, proxy, or other diff --git a/request_test.go b/request_test.go index 0892bc25..a32b583c 100644 --- a/request_test.go +++ b/request_test.go @@ -767,16 +767,23 @@ func TestRequestWriteBufferedWriter(t *testing.T) { } } -func TestRequestBadHost(t *testing.T) { +func TestRequestBadHostHeader(t *testing.T) { got := []string{} req, err := NewRequest("GET", "http://foo/after", nil) if err != nil { t.Fatal(err) } - req.Host = "foo.com with spaces" - req.URL.Host = "foo.com with spaces" - if err := req.Write(logWrites{t, &got}); err == nil { - t.Errorf("Writing request with invalid Host: succeded, want error") + req.Host = "foo.com\nnewline" + req.URL.Host = "foo.com\nnewline" + req.Write(logWrites{t, &got}) + want := []string{ + "GET /after HTTP/1.1\r\n", + "Host: \r\n", + "User-Agent: " + DefaultUserAgent + "\r\n", + "\r\n", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Writes = %q\n Want = %q", got, want) } } From 84f051dc0c83a8218493cfc0905244a53b722437 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Mon, 7 Aug 2023 15:57:54 -0700 Subject: [PATCH 09/93] net/http: sanitize User-Agent header in request writer Apply the same transformations to the User-Agent header value that we do to other headers. Avoids header and request smuggling in Request.Write and Request.WriteProxy. RoundTrip already validates values in Request.Header, and didn't allow bad User-Agent values to make it as far as the request writer. Fixes #61824 Change-Id: I360a915c7e08d014e0532bd5af196a5b59c89395 Reviewed-on: https://go-review.googlesource.com/c/go/+/516836 Reviewed-by: Jonathan Amsterdam Run-TryBot: Damien Neil TryBot-Result: Gopher Robot --- request.go | 2 ++ request_test.go | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/request.go b/request.go index d1fbd5df..0fb73c12 100644 --- a/request.go +++ b/request.go @@ -669,6 +669,8 @@ func (r *Request) write(w io.Writer, usingProxy bool, extraHeaders Header, waitF userAgent = r.Header.Get("User-Agent") } if userAgent != "" { + userAgent = headerNewlineToSpace.Replace(userAgent) + userAgent = textproto.TrimString(userAgent) _, err = fmt.Fprintf(w, "User-Agent: %s\r\n", userAgent) if err != nil { return err diff --git a/request_test.go b/request_test.go index a32b583c..57111648 100644 --- a/request_test.go +++ b/request_test.go @@ -787,6 +787,25 @@ func TestRequestBadHostHeader(t *testing.T) { } } +func TestRequestBadUserAgent(t *testing.T) { + got := []string{} + req, err := NewRequest("GET", "http://foo/after", nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("User-Agent", "evil\r\nX-Evil: evil") + req.Write(logWrites{t, &got}) + want := []string{ + "GET /after HTTP/1.1\r\n", + "Host: foo\r\n", + "User-Agent: evil X-Evil: evil\r\n", + "\r\n", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("Writes = %q\n Want = %q", got, want) + } +} + func TestStarRequest(t *testing.T) { req, err := ReadRequest(bufio.NewReader(strings.NewReader("M-SEARCH * HTTP/1.1\r\n\r\n"))) if err != nil { From 69be2a39e1988957d3762d05cc1d74755da26275 Mon Sep 17 00:00:00 2001 From: Michael Anthony Knyszek Date: Wed, 9 Aug 2023 19:38:27 +0000 Subject: [PATCH 10/93] all: update vendored dependencies Generated by: go install golang.org/x/tools/cmd/bundle@latest go install golang.org/x/build/cmd/updatestd@latest updatestd -goroot=$GOROOT -branch=master For #36905. Change-Id: I11c3376452b0b03eb91a87619b70d74e6ce897bd Reviewed-on: https://go-review.googlesource.com/c/go/+/517875 Reviewed-by: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov Run-TryBot: Michael Knyszek TryBot-Result: Gopher Robot --- h2_bundle.go | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/h2_bundle.go b/h2_bundle.go index dc3e099c..c95fbc47 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -33,6 +33,7 @@ import ( "io/fs" "log" "math" + "math/bits" mathrand "math/rand" "net" "net/http/httptrace" @@ -8702,7 +8703,28 @@ func (cs *http2clientStream) frameScratchBufferLen(maxFrameSize int) int { return int(n) // doesn't truncate; max is 512K } -var http2bufPool sync.Pool // of *[]byte +// Seven bufPools manage different frame sizes. This helps to avoid scenarios where long-running +// streaming requests using small frame sizes occupy large buffers initially allocated for prior +// requests needing big buffers. The size ranges are as follows: +// {0 KB, 16 KB], {16 KB, 32 KB], {32 KB, 64 KB], {64 KB, 128 KB], {128 KB, 256 KB], +// {256 KB, 512 KB], {512 KB, infinity} +// In practice, the maximum scratch buffer size should not exceed 512 KB due to +// frameScratchBufferLen(maxFrameSize), thus the "infinity pool" should never be used. +// It exists mainly as a safety measure, for potential future increases in max buffer size. +var http2bufPools [7]sync.Pool // of *[]byte + +func http2bufPoolIndex(size int) int { + if size <= 16384 { + return 0 + } + size -= 1 + bits := bits.Len(uint(size)) + index := bits - 14 + if index >= len(http2bufPools) { + return len(http2bufPools) - 1 + } + return index +} func (cs *http2clientStream) writeRequestBody(req *Request) (err error) { cc := cs.cc @@ -8720,12 +8742,13 @@ func (cs *http2clientStream) writeRequestBody(req *Request) (err error) { // Scratch buffer for reading into & writing from. scratchLen := cs.frameScratchBufferLen(maxFrameSize) var buf []byte - if bp, ok := http2bufPool.Get().(*[]byte); ok && len(*bp) >= scratchLen { - defer http2bufPool.Put(bp) + index := http2bufPoolIndex(scratchLen) + if bp, ok := http2bufPools[index].Get().(*[]byte); ok && len(*bp) >= scratchLen { + defer http2bufPools[index].Put(bp) buf = *bp } else { buf = make([]byte, scratchLen) - defer http2bufPool.Put(&buf) + defer http2bufPools[index].Put(&buf) } var sawEOF bool From 12502c942391fc304df4efbd204d3d80fad1871b Mon Sep 17 00:00:00 2001 From: Mauri de Souza Meneguzzo Date: Thu, 10 Aug 2023 20:56:27 +0000 Subject: [PATCH 11/93] net/http: disallow empty Content-Length header The Content-Length must be a valid numeric value, empty values should not be accepted. See: https://www.rfc-editor.org/rfc/rfc9110.html#name-content-length Fixes #61679 Change-Id: Icbcd933087fe5e50199b62ff34c58bf92a09d3d4 GitHub-Last-Rev: 932e46b55b54d5f2050453bcaa50e9476c8559fd GitHub-Pull-Request: golang/go#61865 Reviewed-on: https://go-review.googlesource.com/c/go/+/517336 Reviewed-by: Damien Neil Auto-Submit: Bryan Mills Reviewed-by: Bryan Mills Run-TryBot: Damien Neil TryBot-Result: Gopher Robot --- response_test.go | 3 ++- transfer.go | 38 ++++++++++++++++++++++++-------------- transfer_test.go | 6 +++++- 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/response_test.go b/response_test.go index 19fb48f2..ddd31808 100644 --- a/response_test.go +++ b/response_test.go @@ -883,6 +883,7 @@ func TestReadResponseErrors(t *testing.T) { } errMultiCL := "message cannot contain multiple Content-Length headers" + errEmptyCL := "invalid empty Content-Length" tests := []testCase{ {"", "", io.ErrUnexpectedEOF}, @@ -918,7 +919,7 @@ func TestReadResponseErrors(t *testing.T) { contentLength("200 OK", "Content-Length: 7\r\nContent-Length: 7\r\n\r\nGophers\r\n", nil), contentLength("201 OK", "Content-Length: 0\r\nContent-Length: 7\r\n\r\nGophers\r\n", errMultiCL), contentLength("300 OK", "Content-Length: 0\r\nContent-Length: 0 \r\n\r\nGophers\r\n", nil), - contentLength("200 OK", "Content-Length:\r\nContent-Length:\r\n\r\nGophers\r\n", nil), + contentLength("200 OK", "Content-Length:\r\nContent-Length:\r\n\r\nGophers\r\n", errEmptyCL), contentLength("206 OK", "Content-Length:\r\nContent-Length: 0 \r\nConnection: close\r\n\r\nGophers\r\n", errMultiCL), // multiple content-length headers for 204 and 304 should still be checked diff --git a/transfer.go b/transfer.go index d6f26a70..b2499817 100644 --- a/transfer.go +++ b/transfer.go @@ -9,6 +9,7 @@ import ( "bytes" "errors" "fmt" + "internal/godebug" "io" "net/http/httptrace" "net/http/internal" @@ -527,7 +528,7 @@ func readTransfer(msg any, r *bufio.Reader) (err error) { return err } if isResponse && t.RequestMethod == "HEAD" { - if n, err := parseContentLength(t.Header.get("Content-Length")); err != nil { + if n, err := parseContentLength(t.Header["Content-Length"]); err != nil { return err } else { t.ContentLength = n @@ -707,18 +708,15 @@ func fixLength(isResponse bool, status int, requestMethod string, header Header, return -1, nil } - // Logic based on Content-Length - var cl string - if len(contentLens) == 1 { - cl = textproto.TrimString(contentLens[0]) - } - if cl != "" { - n, err := parseContentLength(cl) + if len(contentLens) > 0 { + // Logic based on Content-Length + n, err := parseContentLength(contentLens) if err != nil { return -1, err } return n, nil } + header.Del("Content-Length") if isRequest { @@ -1038,19 +1036,31 @@ func (bl bodyLocked) Read(p []byte) (n int, err error) { return bl.b.readLocked(p) } -// parseContentLength trims whitespace from s and returns -1 if no value -// is set, or the value if it's >= 0. -func parseContentLength(cl string) (int64, error) { - cl = textproto.TrimString(cl) - if cl == "" { +var laxContentLength = godebug.New("httplaxcontentlength") + +// parseContentLength checks that the header is valid and then trims +// whitespace. It returns -1 if no value is set otherwise the value +// if it's >= 0. +func parseContentLength(clHeaders []string) (int64, error) { + if len(clHeaders) == 0 { return -1, nil } + cl := textproto.TrimString(clHeaders[0]) + + // The Content-Length must be a valid numeric value. + // See: https://datatracker.ietf.org/doc/html/rfc2616/#section-14.13 + if cl == "" { + if laxContentLength.Value() == "1" { + laxContentLength.IncNonDefault() + return -1, nil + } + return 0, badStringError("invalid empty Content-Length", cl) + } n, err := strconv.ParseUint(cl, 10, 63) if err != nil { return 0, badStringError("bad Content-Length", cl) } return int64(n), nil - } // finishAsyncByteRead finishes reading the 1-byte sniff diff --git a/transfer_test.go b/transfer_test.go index 5e0df896..20cc7b5d 100644 --- a/transfer_test.go +++ b/transfer_test.go @@ -332,6 +332,10 @@ func TestParseContentLength(t *testing.T) { cl string wantErr error }{ + { + cl: "", + wantErr: badStringError("invalid empty Content-Length", ""), + }, { cl: "3", wantErr: nil, @@ -356,7 +360,7 @@ func TestParseContentLength(t *testing.T) { } for _, tt := range tests { - if _, gotErr := parseContentLength(tt.cl); !reflect.DeepEqual(gotErr, tt.wantErr) { + if _, gotErr := parseContentLength([]string{tt.cl}); !reflect.DeepEqual(gotErr, tt.wantErr) { t.Errorf("%q:\n\tgot=%v\n\twant=%v", tt.cl, gotErr, tt.wantErr) } } From d7220b133d0eddfd38d2007aaba1d997a7aa5512 Mon Sep 17 00:00:00 2001 From: Alexander Yastrebov Date: Wed, 2 Aug 2023 22:15:59 +0000 Subject: [PATCH 12/93] net/http: use cancelKey to cancel request Follows up on CL 245357 and adds missing returns in waitCondition (CL 477196) Fixes #51354 Change-Id: I7950ff889ad72c4927a969c35fedc0186e863bd6 GitHub-Last-Rev: 52ce05bc83ef88c7104df9254bc1add0dda83ae0 GitHub-Pull-Request: golang/go#61724 Reviewed-on: https://go-review.googlesource.com/c/go/+/515435 Reviewed-by: Damien Neil Reviewed-by: Bryan Mills Run-TryBot: Damien Neil Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot Auto-Submit: Damien Neil --- transport.go | 2 +- transport_test.go | 60 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/transport.go b/transport.go index c07352b0..d30eb795 100644 --- a/transport.go +++ b/transport.go @@ -2248,7 +2248,7 @@ func (pc *persistConn) readLoop() { } case <-rc.req.Cancel: alive = false - pc.t.CancelRequest(rc.req) + pc.t.cancelRequest(rc.cancelKey, errRequestCanceled) case <-rc.req.Context().Done(): alive = false pc.t.cancelRequest(rc.cancelKey, rc.req.Context().Err()) diff --git a/transport_test.go b/transport_test.go index 028fecc9..bcc26aa5 100644 --- a/transport_test.go +++ b/transport_test.go @@ -2440,6 +2440,7 @@ func testTransportCancelRequest(t *testing.T, mode testMode) { if d > 0 { t.Logf("pending requests = %d after %v (want 0)", n, d) } + return false } return true }) @@ -2599,6 +2600,65 @@ func testCancelRequestWithChannel(t *testing.T, mode testMode) { if d > 0 { t.Logf("pending requests = %d after %v (want 0)", n, d) } + return false + } + return true + }) +} + +// Issue 51354 +func TestCancelRequestWithBodyWithChannel(t *testing.T) { + run(t, testCancelRequestWithBodyWithChannel, []testMode{http1Mode}) +} +func testCancelRequestWithBodyWithChannel(t *testing.T, mode testMode) { + if testing.Short() { + t.Skip("skipping test in -short mode") + } + + const msg = "Hello" + unblockc := make(chan struct{}) + ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + io.WriteString(w, msg) + w.(Flusher).Flush() // send headers and some body + <-unblockc + })).ts + defer close(unblockc) + + c := ts.Client() + tr := c.Transport.(*Transport) + + req, _ := NewRequest("POST", ts.URL, strings.NewReader("withbody")) + cancel := make(chan struct{}) + req.Cancel = cancel + + res, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + body := make([]byte, len(msg)) + n, _ := io.ReadFull(res.Body, body) + if n != len(body) || !bytes.Equal(body, []byte(msg)) { + t.Errorf("Body = %q; want %q", body[:n], msg) + } + close(cancel) + + tail, err := io.ReadAll(res.Body) + res.Body.Close() + if err != ExportErrRequestCanceled { + t.Errorf("Body.Read error = %v; want errRequestCanceled", err) + } else if len(tail) > 0 { + t.Errorf("Spurious bytes from Body.Read: %q", tail) + } + + // Verify no outstanding requests after readLoop/writeLoop + // goroutines shut down. + waitCondition(t, 10*time.Millisecond, func(d time.Duration) bool { + n := tr.NumPendingRequestsForTesting() + if n > 0 { + if d > 0 { + t.Logf("pending requests = %d after %v (want 0)", n, d) + } + return false } return true }) From 8dfaa07c0c1b1004bd87060ff3e9951c1f14ff97 Mon Sep 17 00:00:00 2001 From: Alexander Yastrebov Date: Tue, 25 Jul 2023 10:53:58 +0000 Subject: [PATCH 13/93] net/http: clear reference to the request context after transport getConn Clears wannConn ctx and prevents pending dialConnFor after connection delivered or canceled. Updates #50798 Change-Id: I9a681ac0f222be56571fa768700220f6b5ee0888 GitHub-Last-Rev: fd6c83ab072c62d224ed8220c4c286b6e90bc151 GitHub-Pull-Request: golang/go#61524 Reviewed-on: https://go-review.googlesource.com/c/go/+/512196 Auto-Submit: Matthew Dempsky Run-TryBot: Ian Lance Taylor Reviewed-by: Matthew Dempsky Reviewed-by: Damien Neil TryBot-Result: Gopher Robot --- transport.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/transport.go b/transport.go index d30eb795..35dfe908 100644 --- a/transport.go +++ b/transport.go @@ -1205,7 +1205,6 @@ func (t *Transport) dial(ctx context.Context, network, addr string) (net.Conn, e type wantConn struct { cm connectMethod key connectMethodKey // cm.key() - ctx context.Context // context for dial ready chan struct{} // closed when pc, err pair is delivered // hooks for testing to know when dials are done @@ -1214,7 +1213,8 @@ type wantConn struct { beforeDial func() afterDial func() - mu sync.Mutex // protects pc, err, close(ready) + mu sync.Mutex // protects ctx, pc, err, close(ready) + ctx context.Context // context for dial, cleared after delivered or canceled pc *persistConn err error } @@ -1229,6 +1229,13 @@ func (w *wantConn) waiting() bool { } } +// getCtxForDial returns context for dial or nil if connection was delivered or canceled. +func (w *wantConn) getCtxForDial() context.Context { + w.mu.Lock() + defer w.mu.Unlock() + return w.ctx +} + // tryDeliver attempts to deliver pc, err to w and reports whether it succeeded. func (w *wantConn) tryDeliver(pc *persistConn, err error) bool { w.mu.Lock() @@ -1238,6 +1245,7 @@ func (w *wantConn) tryDeliver(pc *persistConn, err error) bool { return false } + w.ctx = nil w.pc = pc w.err = err if w.pc == nil && w.err == nil { @@ -1255,6 +1263,7 @@ func (w *wantConn) cancel(t *Transport, err error) { close(w.ready) // catch misbehavior in future delivery } pc := w.pc + w.ctx = nil w.pc = nil w.err = err w.mu.Unlock() @@ -1463,8 +1472,12 @@ func (t *Transport) queueForDial(w *wantConn) { // If the dial is canceled or unsuccessful, dialConnFor decrements t.connCount[w.cm.key()]. func (t *Transport) dialConnFor(w *wantConn) { defer w.afterDial() + ctx := w.getCtxForDial() + if ctx == nil { + return + } - pc, err := t.dialConn(w.ctx, w.cm) + pc, err := t.dialConn(ctx, w.cm) delivered := w.tryDeliver(pc, err) if err == nil && (!delivered || pc.alt != nil) { // pconn was not passed to w, From 31f798d18636e6b9b47d053083d3000ee20d250f Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Tue, 22 Aug 2023 11:01:29 -0400 Subject: [PATCH 14/93] net/http: use testenv.Command instead of exec.Command in tests On Unix platforms, testenv.Command sends SIGQUIT to stuck commands before the test times out. For subprocesses that are written in Go, that causes the runtime to dump running goroutines, and in other languages it triggers similar behavior (such as a core dump). If the subprocess is stuck due to a bug (such as #57999), that may help to diagnose it. For #57999. Change-Id: Ia2e9d14718a26001e030e162c69892497a8ebb21 Reviewed-on: https://go-review.googlesource.com/c/go/+/521816 Reviewed-by: Damien Neil Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot Auto-Submit: Bryan Mills --- cgi/host_test.go | 5 +++-- fs_test.go | 5 +++-- http_test.go | 7 +++---- serve_test.go | 3 +-- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cgi/host_test.go b/cgi/host_test.go index 860e9b3e..707af71d 100644 --- a/cgi/host_test.go +++ b/cgi/host_test.go @@ -9,6 +9,7 @@ package cgi import ( "bufio" "fmt" + "internal/testenv" "io" "net" "net/http" @@ -93,7 +94,7 @@ var cgiTested, cgiWorks bool func check(t *testing.T) { if !cgiTested { cgiTested = true - cgiWorks = exec.Command("./testdata/test.cgi").Run() == nil + cgiWorks = testenv.Command(t, "./testdata/test.cgi").Run() == nil } if !cgiWorks { // No Perl on Windows, needed by test.cgi @@ -462,7 +463,7 @@ func findPerl(t *testing.T) string { } perl, _ = filepath.Abs(perl) - cmd := exec.Command(perl, "-e", "print 123") + cmd := testenv.Command(t, perl, "-e", "print 123") cmd.Env = []string{"PATH=/garbage"} out, err := cmd.Output() if err != nil || string(out) != "123" { diff --git a/fs_test.go b/fs_test.go index bb96d2ca..2e157736 100644 --- a/fs_test.go +++ b/fs_test.go @@ -9,6 +9,7 @@ import ( "bytes" "errors" "fmt" + "internal/testenv" "io" "io/fs" "mime" @@ -1266,7 +1267,7 @@ func TestLinuxSendfile(t *testing.T) { defer ln.Close() // Attempt to run strace, and skip on failure - this test requires SYS_PTRACE. - if err := exec.Command("strace", "-f", "-q", os.Args[0], "-test.run=^$").Run(); err != nil { + if err := testenv.Command(t, "strace", "-f", "-q", os.Args[0], "-test.run=^$").Run(); err != nil { t.Skipf("skipping; failed to run strace: %v", err) } @@ -1279,7 +1280,7 @@ func TestLinuxSendfile(t *testing.T) { defer os.Remove(filepath) var buf strings.Builder - child := exec.Command("strace", "-f", "-q", os.Args[0], "-test.run=TestLinuxSendfileChild") + child := testenv.Command(t, "strace", "-f", "-q", os.Args[0], "-test.run=TestLinuxSendfileChild") child.ExtraFiles = append(child.ExtraFiles, lnf) child.Env = append([]string{"GO_WANT_HELPER_PROCESS=1"}, os.Environ()...) child.Stdout = &buf diff --git a/http_test.go b/http_test.go index 91bb1b26..2e7e024e 100644 --- a/http_test.go +++ b/http_test.go @@ -12,7 +12,6 @@ import ( "io/fs" "net/url" "os" - "os/exec" "reflect" "regexp" "strings" @@ -55,7 +54,7 @@ func TestForeachHeaderElement(t *testing.T) { func TestCmdGoNoHTTPServer(t *testing.T) { t.Parallel() goBin := testenv.GoToolPath(t) - out, err := exec.Command(goBin, "tool", "nm", goBin).CombinedOutput() + out, err := testenv.Command(t, goBin, "tool", "nm", goBin).CombinedOutput() if err != nil { t.Fatalf("go tool nm: %v: %s", err, out) } @@ -89,7 +88,7 @@ func TestOmitHTTP2(t *testing.T) { } t.Parallel() goTool := testenv.GoToolPath(t) - out, err := exec.Command(goTool, "test", "-short", "-tags=nethttpomithttp2", "net/http").CombinedOutput() + out, err := testenv.Command(t, goTool, "test", "-short", "-tags=nethttpomithttp2", "net/http").CombinedOutput() if err != nil { t.Fatalf("go test -short failed: %v, %s", err, out) } @@ -101,7 +100,7 @@ func TestOmitHTTP2(t *testing.T) { func TestOmitHTTP2Vet(t *testing.T) { t.Parallel() goTool := testenv.GoToolPath(t) - out, err := exec.Command(goTool, "vet", "-tags=nethttpomithttp2", "net/http").CombinedOutput() + out, err := testenv.Command(t, goTool, "vet", "-tags=nethttpomithttp2", "net/http").CombinedOutput() if err != nil { t.Fatalf("go vet failed: %v, %s", err, out) } diff --git a/serve_test.go b/serve_test.go index bb380cf4..1f215bd8 100644 --- a/serve_test.go +++ b/serve_test.go @@ -30,7 +30,6 @@ import ( "net/http/internal/testcert" "net/url" "os" - "os/exec" "path/filepath" "reflect" "regexp" @@ -5005,7 +5004,7 @@ func BenchmarkServer(b *testing.B) { defer ts.Close() b.StartTimer() - cmd := exec.Command(os.Args[0], "-test.run=XXXX", "-test.bench=BenchmarkServer$") + cmd := testenv.Command(b, os.Args[0], "-test.run=XXXX", "-test.bench=BenchmarkServer$") cmd.Env = append([]string{ fmt.Sprintf("TEST_BENCH_CLIENT_N=%d", b.N), fmt.Sprintf("TEST_BENCH_SERVER_URL=%s", ts.URL), From 2efe866be04e2cd4cf667af6f5f1a4e2432a092d Mon Sep 17 00:00:00 2001 From: Alexander Yastrebov Date: Tue, 22 Aug 2023 21:14:14 +0000 Subject: [PATCH 15/93] net/http: fix request canceler leak on connection close Due to a race condition persistConn could be closed without removing request canceler. Note that without the fix test occasionally passes and to demonstrate the issue it has to be run multiple times, e.g. using -count=10. Fixes #61708 Change-Id: I9029d7d65cf602dd29ee1b2a87a77a73e99d9c92 GitHub-Last-Rev: 6b31f9826da71dad4ee8c0491efba995a8f51440 GitHub-Pull-Request: golang/go#61745 Reviewed-on: https://go-review.googlesource.com/c/go/+/515796 Reviewed-by: Bryan Mills Reviewed-by: Damien Neil Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot --- transport.go | 1 + transport_test.go | 52 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/transport.go b/transport.go index 35dfe908..c2376aa6 100644 --- a/transport.go +++ b/transport.go @@ -2267,6 +2267,7 @@ func (pc *persistConn) readLoop() { pc.t.cancelRequest(rc.cancelKey, rc.req.Context().Err()) case <-pc.closech: alive = false + pc.t.setReqCanceler(rc.cancelKey, nil) } testHookReadLoopBeforeNextRead() diff --git a/transport_test.go b/transport_test.go index bcc26aa5..4ff26ff3 100644 --- a/transport_test.go +++ b/transport_test.go @@ -6810,3 +6810,55 @@ func testRequestSanitization(t *testing.T, mode testMode) { resp.Body.Close() } } + +// Issue 61708 +func TestTransportAndServerSharedBodyReqCancelerCleanupOnConnectionClose(t *testing.T) { + run(t, testTransportAndServerSharedBodyReqCancelerCleanupOnConnectionClose, []testMode{http1Mode}) +} +func testTransportAndServerSharedBodyReqCancelerCleanupOnConnectionClose(t *testing.T, mode testMode) { + const bodySize = 1 << 20 + + backend := newClientServerTest(t, mode, HandlerFunc(func(rw ResponseWriter, req *Request) { + io.Copy(rw, req.Body) + })) + t.Logf("Backend address: %s", backend.ts.Listener.Addr().String()) + + var proxy *clientServerTest + proxy = newClientServerTest(t, mode, HandlerFunc(func(rw ResponseWriter, req *Request) { + breq, _ := NewRequest("POST", backend.ts.URL, req.Body) + + bresp, err := backend.c.Do(breq) + if err != nil { + t.Fatalf("Unexpected proxy outbound request error: %v", err) + } + defer bresp.Body.Close() + + _, err = io.Copy(rw, bresp.Body) + if err == nil { + t.Fatalf("Expected proxy copy error") + } + t.Logf("Proxy copy error: %v", err) + })) + t.Logf("Proxy address: %s", proxy.ts.Listener.Addr().String()) + + req, _ := NewRequest("POST", proxy.ts.URL, io.LimitReader(neverEnding('a'), bodySize)) + res, err := proxy.c.Do(req) + if err != nil { + t.Fatalf("Original request: %v", err) + } + // Close body without reading to trigger proxy copy error + res.Body.Close() + + // Verify no outstanding requests after readLoop/writeLoop + // goroutines shut down. + waitCondition(t, 10*time.Millisecond, func(d time.Duration) bool { + n := backend.tr.NumPendingRequestsForTesting() + if n > 0 { + if d > 0 { + t.Logf("pending requests = %d after %v (want 0)", n, d) + } + return false + } + return true + }) +} From 187b0289e9e0f7c6c725a1ed0b0307156d263093 Mon Sep 17 00:00:00 2001 From: Alexander Yastrebov Date: Wed, 23 Aug 2023 15:39:26 +0000 Subject: [PATCH 16/93] net/http: revert fix request canceler leak on connection close This reverts CL 515796 due to a flaking test. Updates #61708 Fixes #62224 Change-Id: I53911a07677d08c3196daaaa2708269593baf472 GitHub-Last-Rev: 3544648ecc3783dcb10d54fc2b266797c02f9a75 GitHub-Pull-Request: golang/go#62233 Reviewed-on: https://go-review.googlesource.com/c/go/+/522097 Run-TryBot: Bryan Mills Reviewed-by: Bryan Mills Reviewed-by: Dmitri Shuralyov TryBot-Result: Gopher Robot Auto-Submit: Bryan Mills Reviewed-by: Than McIntosh --- transport.go | 1 - transport_test.go | 52 ----------------------------------------------- 2 files changed, 53 deletions(-) diff --git a/transport.go b/transport.go index c2376aa6..35dfe908 100644 --- a/transport.go +++ b/transport.go @@ -2267,7 +2267,6 @@ func (pc *persistConn) readLoop() { pc.t.cancelRequest(rc.cancelKey, rc.req.Context().Err()) case <-pc.closech: alive = false - pc.t.setReqCanceler(rc.cancelKey, nil) } testHookReadLoopBeforeNextRead() diff --git a/transport_test.go b/transport_test.go index 4ff26ff3..bcc26aa5 100644 --- a/transport_test.go +++ b/transport_test.go @@ -6810,55 +6810,3 @@ func testRequestSanitization(t *testing.T, mode testMode) { resp.Body.Close() } } - -// Issue 61708 -func TestTransportAndServerSharedBodyReqCancelerCleanupOnConnectionClose(t *testing.T) { - run(t, testTransportAndServerSharedBodyReqCancelerCleanupOnConnectionClose, []testMode{http1Mode}) -} -func testTransportAndServerSharedBodyReqCancelerCleanupOnConnectionClose(t *testing.T, mode testMode) { - const bodySize = 1 << 20 - - backend := newClientServerTest(t, mode, HandlerFunc(func(rw ResponseWriter, req *Request) { - io.Copy(rw, req.Body) - })) - t.Logf("Backend address: %s", backend.ts.Listener.Addr().String()) - - var proxy *clientServerTest - proxy = newClientServerTest(t, mode, HandlerFunc(func(rw ResponseWriter, req *Request) { - breq, _ := NewRequest("POST", backend.ts.URL, req.Body) - - bresp, err := backend.c.Do(breq) - if err != nil { - t.Fatalf("Unexpected proxy outbound request error: %v", err) - } - defer bresp.Body.Close() - - _, err = io.Copy(rw, bresp.Body) - if err == nil { - t.Fatalf("Expected proxy copy error") - } - t.Logf("Proxy copy error: %v", err) - })) - t.Logf("Proxy address: %s", proxy.ts.Listener.Addr().String()) - - req, _ := NewRequest("POST", proxy.ts.URL, io.LimitReader(neverEnding('a'), bodySize)) - res, err := proxy.c.Do(req) - if err != nil { - t.Fatalf("Original request: %v", err) - } - // Close body without reading to trigger proxy copy error - res.Body.Close() - - // Verify no outstanding requests after readLoop/writeLoop - // goroutines shut down. - waitCondition(t, 10*time.Millisecond, func(d time.Duration) bool { - n := backend.tr.NumPendingRequestsForTesting() - if n > 0 { - if d > 0 { - t.Logf("pending requests = %d after %v (want 0)", n, d) - } - return false - } - return true - }) -} From 2f21b7caa1d9d5007fddd3afabd6215745558059 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Thu, 24 Aug 2023 10:12:38 -0700 Subject: [PATCH 17/93] net/http: remove arbitrary timeout from TestTransportGCRequest This test expects a *Request to be garbage collected within five seconds. Some slow builders take longer. Drop the arbitrary timeout. Fixes #56809 Change-Id: I4b5bdce09002a5b52b7b5d0b33e7876d48740bc3 Reviewed-on: https://go-review.googlesource.com/c/go/+/522615 Reviewed-by: Bryan Mills Run-TryBot: Damien Neil TryBot-Result: Gopher Robot --- clientserver_test.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/clientserver_test.go b/clientserver_test.go index 58321532..32948f3a 100644 --- a/clientserver_test.go +++ b/clientserver_test.go @@ -1172,16 +1172,12 @@ func testTransportGCRequest(t *testing.T, mode testMode, body bool) { t.Fatal(err) } })() - timeout := time.NewTimer(5 * time.Second) - defer timeout.Stop() for { select { case <-didGC: return - case <-time.After(100 * time.Millisecond): + case <-time.After(1 * time.Millisecond): runtime.GC() - case <-timeout.C: - t.Fatal("never saw GC of request") } } } From 286b3c151f3b3d0b47cb20d03f0887838f7e9c34 Mon Sep 17 00:00:00 2001 From: Ian Lance Taylor Date: Wed, 23 Aug 2023 15:57:48 -0700 Subject: [PATCH 18/93] net/http: use reflect.TypeFor for known types For #60088 Change-Id: I9e4044d9c2694fe86aab1f5220622c8d952b1a90 Reviewed-on: https://go-review.googlesource.com/c/go/+/522338 Reviewed-by: Ian Lance Taylor Reviewed-by: Damien Neil Run-TryBot: Ian Lance Taylor TryBot-Result: Gopher Robot Run-TryBot: Ian Lance Taylor Auto-Submit: Ian Lance Taylor Auto-Submit: Ian Lance Taylor --- transfer_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transfer_test.go b/transfer_test.go index 20cc7b5d..3f9ebdea 100644 --- a/transfer_test.go +++ b/transfer_test.go @@ -112,8 +112,8 @@ func (w *mockTransferWriter) Write(p []byte) (int, error) { } func TestTransferWriterWriteBodyReaderTypes(t *testing.T) { - fileType := reflect.TypeOf(&os.File{}) - bufferType := reflect.TypeOf(&bytes.Buffer{}) + fileType := reflect.TypeFor[*os.File]() + bufferType := reflect.TypeFor[*bytes.Buffer]() nBytes := int64(1 << 10) newFileFunc := func() (r io.Reader, done func(), err error) { From 052b7a7328b16324db61b28e8214df3bcfd554a8 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Thu, 24 Aug 2023 10:58:51 -0700 Subject: [PATCH 19/93] net/http: deflake TestRequestBodyLimit This test can return with a Transport still processing an in-flight request, resulting in a test failure due to the leaked Transport. Avoid this by waiting for the Transport to close the request body before returning. Fixes #60264 Change-Id: I8d8b54f633c2e28da2b1bf1bc01ce09dd77769de Reviewed-on: https://go-review.googlesource.com/c/go/+/522695 Reviewed-by: Bryan Mills Auto-Submit: Damien Neil TryBot-Result: Gopher Robot Run-TryBot: Damien Neil --- serve_test.go | 49 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/serve_test.go b/serve_test.go index 1f215bd8..e71c5365 100644 --- a/serve_test.go +++ b/serve_test.go @@ -2967,15 +2967,36 @@ func (b neverEnding) Read(p []byte) (n int, err error) { return len(p), nil } -type countReader struct { - r io.Reader - n *int64 +type bodyLimitReader struct { + mu sync.Mutex + count int + limit int + closed chan struct{} } -func (cr countReader) Read(p []byte) (n int, err error) { - n, err = cr.r.Read(p) - atomic.AddInt64(cr.n, int64(n)) - return +func (r *bodyLimitReader) Read(p []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + select { + case <-r.closed: + return 0, errors.New("closed") + default: + } + if r.count > r.limit { + return 0, errors.New("at limit") + } + r.count += len(p) + for i := range p { + p[i] = 'a' + } + return len(p), nil +} + +func (r *bodyLimitReader) Close() error { + r.mu.Lock() + defer r.mu.Unlock() + close(r.closed) + return nil } func TestRequestBodyLimit(t *testing.T) { run(t, testRequestBodyLimit) } @@ -2999,8 +3020,11 @@ func testRequestBodyLimit(t *testing.T, mode testMode) { } })) - nWritten := new(int64) - req, _ := NewRequest("POST", cst.ts.URL, io.LimitReader(countReader{neverEnding('a'), nWritten}, limit*200)) + body := &bodyLimitReader{ + closed: make(chan struct{}), + limit: limit * 200, + } + req, _ := NewRequest("POST", cst.ts.URL, body) // Send the POST, but don't care it succeeds or not. The // remote side is going to reply and then close the TCP @@ -3015,10 +3039,13 @@ func testRequestBodyLimit(t *testing.T, mode testMode) { if err == nil { resp.Body.Close() } + // Wait for the Transport to finish writing the request body. + // It will close the body when done. + <-body.closed - if atomic.LoadInt64(nWritten) > limit*100 { + if body.count > limit*100 { t.Errorf("handler restricted the request body to %d bytes, but client managed to write %d", - limit, nWritten) + limit, body.count) } } From 472d9b6802c74c5da2c421c0a74c8de345573cf4 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Thu, 24 Aug 2023 10:58:51 -0700 Subject: [PATCH 20/93] net/http: document when request bodies are closed in more places It isn't obvious that request bodies can be closed asynchronously, and it's easy to overlook the documentation of this fact in RoundTripper, which is a fairly low-level interface. Change-Id: I3b825c505418af7e1d3f6ed58f3704e55cf16901 Reviewed-on: https://go-review.googlesource.com/c/go/+/523036 TryBot-Result: Gopher Robot Run-TryBot: Damien Neil Reviewed-by: Bryan Mills Auto-Submit: Damien Neil --- client.go | 3 ++- request.go | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/client.go b/client.go index 2cab53a5..5478690e 100644 --- a/client.go +++ b/client.go @@ -566,7 +566,8 @@ func urlErrorOp(method string) string { // connection to the server for a subsequent "keep-alive" request. // // The request Body, if non-nil, will be closed by the underlying -// Transport, even on errors. +// Transport, even on errors. The Body may be closed asynchronously after +// Do returns. // // On error, any Response can be ignored. A non-nil Response with a // non-nil error only occurs when CheckRedirect fails, and even then diff --git a/request.go b/request.go index 0fb73c12..12039c9a 100644 --- a/request.go +++ b/request.go @@ -845,8 +845,9 @@ func NewRequest(method, url string, body io.Reader) (*Request, error) { // optional body. // // If the provided body is also an io.Closer, the returned -// Request.Body is set to body and will be closed by the Client -// methods Do, Post, and PostForm, and Transport.RoundTrip. +// Request.Body is set to body and will be closed (possibly +// asynchronously) by the Client methods Do, Post, and PostForm, +// and Transport.RoundTrip. // // NewRequestWithContext returns a Request suitable for use with // Client.Do or Transport.RoundTrip. To create a request for use with From 3c38887193c7c86addb31b37ef4f5bc8d2e8c049 Mon Sep 17 00:00:00 2001 From: Matthew Dempsky Date: Thu, 24 Aug 2023 23:14:43 -0700 Subject: [PATCH 21/93] net/http/cgi: workaround for closure inlining issue This is a temporary workaround for issue #62277, to get the longtest builders passing again. As mentioned on the issue, the underlying issue was present even before CL 522318; it just now affects inlined closures in initialization expressions too, not just explicit init functions. This CL can and should be reverted once that issue is fixed properly. Change-Id: I612a501e131d1b5eea648aafeb1a3a3fe8fe8c83 Reviewed-on: https://go-review.googlesource.com/c/go/+/522935 Reviewed-by: Cuong Manh Le Reviewed-by: Bryan Mills TryBot-Result: Gopher Robot Run-TryBot: Matthew Dempsky Auto-Submit: Matthew Dempsky --- cgi/host.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cgi/host.go b/cgi/host.go index 073952a7..a3fba4b4 100644 --- a/cgi/host.go +++ b/cgi/host.go @@ -35,7 +35,10 @@ import ( var trailingPort = regexp.MustCompile(`:([0-9]+)$`) -var osDefaultInheritEnv = func() []string { +var osDefaultInheritEnv = getOSDefaultInheritEnv() + +// TODO(mdempsky): Revert CL 522935 after #62277 is fixed. +func getOSDefaultInheritEnv() []string { switch runtime.GOOS { case "darwin", "ios": return []string{"DYLD_LIBRARY_PATH"} @@ -51,7 +54,7 @@ var osDefaultInheritEnv = func() []string { return []string{"SystemRoot", "COMSPEC", "PATHEXT", "WINDIR"} } return nil -}() +} // Handler runs an executable in a subprocess with a CGI environment. type Handler struct { From 9dc6fbc74707b77ff6ac2c9fff21275d3a9ab81e Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Sat, 26 Aug 2023 08:08:30 +0700 Subject: [PATCH 22/93] Revert "net/http/cgi: workaround for closure inlining issue" This reverts CL 522935. Issue #62277 is fixed, the workaround can be dropped. Updates #62277 Change-Id: I7c69e35248942b4d4fcdd81121051cca9b098980 Reviewed-on: https://go-review.googlesource.com/c/go/+/523175 Run-TryBot: Cuong Manh Le TryBot-Result: Gopher Robot Auto-Submit: Cuong Manh Le Reviewed-by: Matthew Dempsky Reviewed-by: Dmitri Shuralyov --- cgi/host.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cgi/host.go b/cgi/host.go index a3fba4b4..073952a7 100644 --- a/cgi/host.go +++ b/cgi/host.go @@ -35,10 +35,7 @@ import ( var trailingPort = regexp.MustCompile(`:([0-9]+)$`) -var osDefaultInheritEnv = getOSDefaultInheritEnv() - -// TODO(mdempsky): Revert CL 522935 after #62277 is fixed. -func getOSDefaultInheritEnv() []string { +var osDefaultInheritEnv = func() []string { switch runtime.GOOS { case "darwin", "ios": return []string{"DYLD_LIBRARY_PATH"} @@ -54,7 +51,7 @@ func getOSDefaultInheritEnv() []string { return []string{"SystemRoot", "COMSPEC", "PATHEXT", "WINDIR"} } return nil -} +}() // Handler runs an executable in a subprocess with a CGI environment. type Handler struct { From b6432a75e1b0e6ff16c0e80c6cf3c2c7a4ef325d Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Mon, 24 Jul 2023 11:24:40 -0400 Subject: [PATCH 23/93] net/http: document setting of Proxy-Authorization header Add a test for setting a proxy username/password in the HTTP_PROXY environment variable as well. Fixes #61505 Change-Id: I31c3fa94c7bc463133321e9af9289fd47da75b46 Reviewed-on: https://go-review.googlesource.com/c/go/+/512555 Reviewed-by: Brad Fitzpatrick Reviewed-by: Bryan Mills Run-TryBot: Damien Neil TryBot-Result: Gopher Robot --- transport.go | 4 ++++ transport_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/transport.go b/transport.go index 35dfe908..ac7477ea 100644 --- a/transport.go +++ b/transport.go @@ -117,6 +117,10 @@ type Transport struct { // "https", and "socks5" are supported. If the scheme is empty, // "http" is assumed. // + // If the proxy URL contains a userinfo subcomponent, + // the proxy request will pass the username and password + // in a Proxy-Authorization header. + // // If Proxy is nil or returns a nil *URL, no proxy is used. Proxy func(*Request) (*url.URL, error) diff --git a/transport_test.go b/transport_test.go index bcc26aa5..9f086172 100644 --- a/transport_test.go +++ b/transport_test.go @@ -6810,3 +6810,36 @@ func testRequestSanitization(t *testing.T, mode testMode) { resp.Body.Close() } } + +func TestProxyAuthHeader(t *testing.T) { + // Not parallel: Sets an environment variable. + run(t, testProxyAuthHeader, []testMode{http1Mode}, testNotParallel) +} +func testProxyAuthHeader(t *testing.T, mode testMode) { + const username = "u" + const password = "@/?!" + cst := newClientServerTest(t, mode, HandlerFunc(func(rw ResponseWriter, req *Request) { + // Copy the Proxy-Authorization header to a new Request, + // since Request.BasicAuth only parses the Authorization header. + var r2 Request + r2.Header = Header{ + "Authorization": req.Header["Proxy-Authorization"], + } + gotuser, gotpass, ok := r2.BasicAuth() + if !ok || gotuser != username || gotpass != password { + t.Errorf("req.BasicAuth() = %q, %q, %v; want %q, %q, true", gotuser, gotpass, ok, username, password) + } + })) + u, err := url.Parse(cst.ts.URL) + if err != nil { + t.Fatal(err) + } + u.User = url.UserPassword(username, password) + t.Setenv("HTTP_PROXY", u.String()) + cst.tr.Proxy = ProxyURL(u) + resp, err := cst.c.Get("http://_/") + if err != nil { + t.Fatal(err) + } + resp.Body.Close() +} From 891480f53b0a8668c8017f06cc044eaefa19c5fe Mon Sep 17 00:00:00 2001 From: haruyama480 Date: Fri, 25 Aug 2023 15:14:35 +0900 Subject: [PATCH 24/93] net/http: revert "support streaming POST content in wasm" CL 458395 added support for streaming POST content in Wasm. Unfortunately, this breaks requests to servers that only support HTTP/1.1. Revert the change until a suitable fallback or opt-in strategy can be decided. Fixes #61889 Change-Id: If53a77e1890132063b39abde867d34515d4ac2af Reviewed-on: https://go-review.googlesource.com/c/go/+/522955 Run-TryBot: Johan Brandhorst-Satzkorn Reviewed-by: Damien Neil TryBot-Result: Gopher Robot Reviewed-by: Bryan Mills Reviewed-by: Johan Brandhorst-Satzkorn --- roundtrip_js.go | 108 ++++++++---------------------------------------- 1 file changed, 18 insertions(+), 90 deletions(-) diff --git a/roundtrip_js.go b/roundtrip_js.go index dd9efe51..9f9f0cb6 100644 --- a/roundtrip_js.go +++ b/roundtrip_js.go @@ -12,7 +12,6 @@ import ( "io" "strconv" "strings" - "sync" "syscall/js" ) @@ -56,38 +55,6 @@ var jsFetchMissing = js.Global().Get("fetch").IsUndefined() var jsFetchDisabled = js.Global().Get("process").Type() == js.TypeObject && strings.HasPrefix(js.Global().Get("process").Get("argv0").String(), "node") -// Determine whether the JS runtime supports streaming request bodies. -// Courtesy: https://developer.chrome.com/articles/fetch-streaming-requests/#feature-detection -var supportsPostRequestStreams = sync.OnceValue(func() bool { - requestOpt := js.Global().Get("Object").New() - requestBody := js.Global().Get("ReadableStream").New() - - requestOpt.Set("method", "POST") - requestOpt.Set("body", requestBody) - - // There is quite a dance required to define a getter if you do not have the { get property() { ... } } - // syntax available. However, it is possible: - // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Functions/get#defining_a_getter_on_existing_objects_using_defineproperty - duplexCalled := false - duplexGetterObj := js.Global().Get("Object").New() - duplexGetterFunc := js.FuncOf(func(this js.Value, args []js.Value) any { - duplexCalled = true - return "half" - }) - defer duplexGetterFunc.Release() - duplexGetterObj.Set("get", duplexGetterFunc) - js.Global().Get("Object").Call("defineProperty", requestOpt, "duplex", duplexGetterObj) - - // Slight difference here between the aforementioned example: Non-browser-based runtimes - // do not have a non-empty API Base URL (https://html.spec.whatwg.org/multipage/webappapis.html#api-base-url) - // so we have to supply a valid URL here. - requestObject := js.Global().Get("Request").New("https://www.example.org", requestOpt) - - hasContentTypeHeader := requestObject.Get("headers").Call("has", "Content-Type").Bool() - - return duplexCalled && !hasContentTypeHeader -}) - // RoundTrip implements the RoundTripper interface using the WHATWG Fetch API. func (t *Transport) RoundTrip(req *Request) (*Response, error) { // The Transport has a documented contract that states that if the DialContext or @@ -137,63 +104,24 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { opt.Set("headers", headers) if req.Body != nil { - if !supportsPostRequestStreams() { - body, err := io.ReadAll(req.Body) - if err != nil { - req.Body.Close() // RoundTrip must always close the body, including on errors. - return nil, err - } - req.Body.Close() - if len(body) != 0 { - buf := uint8Array.New(len(body)) - js.CopyBytesToJS(buf, body) - opt.Set("body", buf) - } - } else { - readableStreamCtorArg := js.Global().Get("Object").New() - readableStreamCtorArg.Set("type", "bytes") - readableStreamCtorArg.Set("autoAllocateChunkSize", t.writeBufferSize()) - - readableStreamPull := js.FuncOf(func(this js.Value, args []js.Value) any { - controller := args[0] - byobRequest := controller.Get("byobRequest") - if byobRequest.IsNull() { - controller.Call("close") - } - - byobRequestView := byobRequest.Get("view") - - bodyBuf := make([]byte, byobRequestView.Get("byteLength").Int()) - readBytes, readErr := io.ReadFull(req.Body, bodyBuf) - if readBytes > 0 { - buf := uint8Array.New(byobRequestView.Get("buffer")) - js.CopyBytesToJS(buf, bodyBuf) - byobRequest.Call("respond", readBytes) - } - - if readErr == io.EOF || readErr == io.ErrUnexpectedEOF { - controller.Call("close") - } else if readErr != nil { - readErrCauseObject := js.Global().Get("Object").New() - readErrCauseObject.Set("cause", readErr.Error()) - readErr := js.Global().Get("Error").New("io.ReadFull failed while streaming POST body", readErrCauseObject) - controller.Call("error", readErr) - } - // Note: This a return from the pull callback of the controller and *not* RoundTrip(). - return nil - }) - defer func() { - readableStreamPull.Release() - req.Body.Close() - }() - readableStreamCtorArg.Set("pull", readableStreamPull) - - opt.Set("body", js.Global().Get("ReadableStream").New(readableStreamCtorArg)) - // There is a requirement from the WHATWG fetch standard that the duplex property of - // the object given as the options argument to the fetch call be set to 'half' - // when the body property of the same options object is a ReadableStream: - // https://fetch.spec.whatwg.org/#dom-requestinit-duplex - opt.Set("duplex", "half") + // TODO(johanbrandhorst): Stream request body when possible. + // See https://bugs.chromium.org/p/chromium/issues/detail?id=688906 for Blink issue. + // See https://bugzilla.mozilla.org/show_bug.cgi?id=1387483 for Firefox issue. + // See https://github.com/web-platform-tests/wpt/issues/7693 for WHATWG tests issue. + // See https://developer.mozilla.org/en-US/docs/Web/API/Streams_API for more details on the Streams API + // and browser support. + // NOTE(haruyama480): Ensure HTTP/1 fallback exists. + // See https://go.dev/issue/61889 for discussion. + body, err := io.ReadAll(req.Body) + if err != nil { + req.Body.Close() // RoundTrip must always close the body, including on errors. + return nil, err + } + req.Body.Close() + if len(body) != 0 { + buf := uint8Array.New(len(body)) + js.CopyBytesToJS(buf, body) + opt.Set("body", buf) } } From dbf1c1842fe2cf752f9d8da20846e92b2dd4324e Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Sun, 3 Sep 2023 14:23:02 -0400 Subject: [PATCH 25/93] all: use ^TestName$ regular pattern for invoking a single test Use ^ and $ in the -run flag regular expression value when the intention is to invoke a single named test. This removes the reliance on there not being another similarly named test to achieve the intended result. In particular, package syscall has tests named TestUnshareMountNameSpace and TestUnshareMountNameSpaceChroot that both trigger themselves setting GO_WANT_HELPER_PROCESS=1 to run alternate code in a helper process. As a consequence of overlap in their test names, the former was inadvertently triggering one too many helpers. Spotted while reviewing CL 525196. Apply the same change in other places to make it easier for code readers to see that said tests aren't running extraneous tests. The unlikely cases of -run=TestSomething intentionally being used to run all tests that have the TestSomething substring in the name can be better written as -run=^.*TestSomething.*$ or with a comment so it is clear it wasn't an oversight. Change-Id: Iba208aba3998acdbf8c6708e5d23ab88938bfc1e Reviewed-on: https://go-review.googlesource.com/c/go/+/524948 Reviewed-by: Tobias Klauser Auto-Submit: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov Reviewed-by: Ian Lance Taylor Reviewed-by: Kirill Kolyshkin LUCI-TryBot-Result: Go LUCI --- cgi/integration_test.go | 12 ++++++------ fs_test.go | 2 +- httptest/server.go | 2 +- serve_test.go | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cgi/integration_test.go b/cgi/integration_test.go index ef2eaf74..4890ae07 100644 --- a/cgi/integration_test.go +++ b/cgi/integration_test.go @@ -31,7 +31,7 @@ func TestHostingOurselves(t *testing.T) { h := &Handler{ Path: os.Args[0], Root: "/test.go", - Args: []string{"-test.run=TestBeChildCGIProcess"}, + Args: []string{"-test.run=^TestBeChildCGIProcess$"}, } expectedMap := map[string]string{ "test": "Hello CGI-in-CGI", @@ -98,7 +98,7 @@ func TestKillChildAfterCopyError(t *testing.T) { h := &Handler{ Path: os.Args[0], Root: "/test.go", - Args: []string{"-test.run=TestBeChildCGIProcess"}, + Args: []string{"-test.run=^TestBeChildCGIProcess$"}, } req, _ := http.NewRequest("GET", "http://example.com/test.cgi?write-forever=1", nil) rec := httptest.NewRecorder() @@ -120,7 +120,7 @@ func TestChildOnlyHeaders(t *testing.T) { h := &Handler{ Path: os.Args[0], Root: "/test.go", - Args: []string{"-test.run=TestBeChildCGIProcess"}, + Args: []string{"-test.run=^TestBeChildCGIProcess$"}, } expectedMap := map[string]string{ "_body": "", @@ -139,7 +139,7 @@ func TestNilRequestBody(t *testing.T) { h := &Handler{ Path: os.Args[0], Root: "/test.go", - Args: []string{"-test.run=TestBeChildCGIProcess"}, + Args: []string{"-test.run=^TestBeChildCGIProcess$"}, } expectedMap := map[string]string{ "nil-request-body": "false", @@ -154,7 +154,7 @@ func TestChildContentType(t *testing.T) { h := &Handler{ Path: os.Args[0], Root: "/test.go", - Args: []string{"-test.run=TestBeChildCGIProcess"}, + Args: []string{"-test.run=^TestBeChildCGIProcess$"}, } var tests = []struct { name string @@ -202,7 +202,7 @@ func want500Test(t *testing.T, path string) { h := &Handler{ Path: os.Args[0], Root: "/test.go", - Args: []string{"-test.run=TestBeChildCGIProcess"}, + Args: []string{"-test.run=^TestBeChildCGIProcess$"}, } expectedMap := map[string]string{ "_body": "", diff --git a/fs_test.go b/fs_test.go index 2e157736..cfabaae3 100644 --- a/fs_test.go +++ b/fs_test.go @@ -1280,7 +1280,7 @@ func TestLinuxSendfile(t *testing.T) { defer os.Remove(filepath) var buf strings.Builder - child := testenv.Command(t, "strace", "-f", "-q", os.Args[0], "-test.run=TestLinuxSendfileChild") + child := testenv.Command(t, "strace", "-f", "-q", os.Args[0], "-test.run=^TestLinuxSendfileChild$") child.ExtraFiles = append(child.ExtraFiles, lnf) child.Env = append([]string{"GO_WANT_HELPER_PROCESS=1"}, os.Environ()...) child.Stdout = &buf diff --git a/httptest/server.go b/httptest/server.go index f254a494..79749a03 100644 --- a/httptest/server.go +++ b/httptest/server.go @@ -77,7 +77,7 @@ func newLocalListener() net.Listener { // When debugging a particular http server-based test, // this flag lets you run // -// go test -run=BrokenTest -httptest.serve=127.0.0.1:8000 +// go test -run='^BrokenTest$' -httptest.serve=127.0.0.1:8000 // // to start the broken server so you can interact with it manually. // We only register this flag if it looks like the caller knows about it diff --git a/serve_test.go b/serve_test.go index e71c5365..2473a880 100644 --- a/serve_test.go +++ b/serve_test.go @@ -4992,7 +4992,7 @@ func benchmarkClientServerParallel(b *testing.B, parallelism int, mode testMode) // For use like: // // $ go test -c -// $ ./http.test -test.run=XX -test.bench=BenchmarkServer -test.benchtime=15s -test.cpuprofile=http.prof +// $ ./http.test -test.run=XX -test.bench='^BenchmarkServer$' -test.benchtime=15s -test.cpuprofile=http.prof // $ go tool pprof http.test http.prof // (pprof) web func BenchmarkServer(b *testing.B) { @@ -5031,7 +5031,7 @@ func BenchmarkServer(b *testing.B) { defer ts.Close() b.StartTimer() - cmd := testenv.Command(b, os.Args[0], "-test.run=XXXX", "-test.bench=BenchmarkServer$") + cmd := testenv.Command(b, os.Args[0], "-test.run=XXXX", "-test.bench=^BenchmarkServer$") cmd.Env = append([]string{ fmt.Sprintf("TEST_BENCH_CLIENT_N=%d", b.N), fmt.Sprintf("TEST_BENCH_SERVER_URL=%s", ts.URL), @@ -5086,7 +5086,7 @@ func BenchmarkClient(b *testing.B) { // Start server process. ctx, cancel := context.WithCancel(context.Background()) - cmd := testenv.CommandContext(b, ctx, os.Args[0], "-test.run=XXXX", "-test.bench=BenchmarkClient$") + cmd := testenv.CommandContext(b, ctx, os.Args[0], "-test.run=XXXX", "-test.bench=^BenchmarkClient$") cmd.Env = append(cmd.Environ(), "TEST_BENCH_SERVER=yes") cmd.Stderr = os.Stderr stdout, err := cmd.StdoutPipe() From 22c22d19b26b44bd349da769aebcb469ed1673de Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Sun, 3 Sep 2023 13:48:01 -0400 Subject: [PATCH 26/93] all: use ^$ instead of XXXX, NoSuchTestExists to match no tests It's shorter and can't accidentally match unlikely test names. Change-Id: I96dd9da018cad1acf604f266819470278f54c128 Reviewed-on: https://go-review.googlesource.com/c/go/+/524949 Auto-Submit: Dmitri Shuralyov Reviewed-by: Ian Lance Taylor Reviewed-by: Dmitri Shuralyov Reviewed-by: Tobias Klauser LUCI-TryBot-Result: Go LUCI --- serve_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/serve_test.go b/serve_test.go index 2473a880..f26a6b31 100644 --- a/serve_test.go +++ b/serve_test.go @@ -4992,7 +4992,7 @@ func benchmarkClientServerParallel(b *testing.B, parallelism int, mode testMode) // For use like: // // $ go test -c -// $ ./http.test -test.run=XX -test.bench='^BenchmarkServer$' -test.benchtime=15s -test.cpuprofile=http.prof +// $ ./http.test -test.run='^$' -test.bench='^BenchmarkServer$' -test.benchtime=15s -test.cpuprofile=http.prof // $ go tool pprof http.test http.prof // (pprof) web func BenchmarkServer(b *testing.B) { @@ -5031,7 +5031,7 @@ func BenchmarkServer(b *testing.B) { defer ts.Close() b.StartTimer() - cmd := testenv.Command(b, os.Args[0], "-test.run=XXXX", "-test.bench=^BenchmarkServer$") + cmd := testenv.Command(b, os.Args[0], "-test.run=^$", "-test.bench=^BenchmarkServer$") cmd.Env = append([]string{ fmt.Sprintf("TEST_BENCH_CLIENT_N=%d", b.N), fmt.Sprintf("TEST_BENCH_SERVER_URL=%s", ts.URL), @@ -5086,7 +5086,7 @@ func BenchmarkClient(b *testing.B) { // Start server process. ctx, cancel := context.WithCancel(context.Background()) - cmd := testenv.CommandContext(b, ctx, os.Args[0], "-test.run=XXXX", "-test.bench=^BenchmarkClient$") + cmd := testenv.CommandContext(b, ctx, os.Args[0], "-test.run=^$", "-test.bench=^BenchmarkClient$") cmd.Env = append(cmd.Environ(), "TEST_BENCH_SERVER=yes") cmd.Stderr = os.Stderr stdout, err := cmd.StdoutPipe() From f97543873dbf9492a917693e96c6f40c534fe5f4 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Thu, 7 Sep 2023 20:14:47 -0400 Subject: [PATCH 27/93] net/http: extended routing patterns This is the first of several CLs implementing the proposal for enhanced ServeMux routing, https://go.dev/issue/61410. Define a type to represent extended routing patterns and a function to parse a string into one. Updates #61410. Change-Id: I779689acf1f14b20d12c9264251f7dc002b68c49 Reviewed-on: https://go-review.googlesource.com/c/go/+/526815 Run-TryBot: Jonathan Amsterdam Reviewed-by: Eli Bendersky TryBot-Result: Gopher Robot --- pattern.go | 187 ++++++++++++++++++++++++++++++++++++++++++++++++ pattern_test.go | 167 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 354 insertions(+) create mode 100644 pattern.go create mode 100644 pattern_test.go diff --git a/pattern.go b/pattern.go new file mode 100644 index 00000000..a04fd901 --- /dev/null +++ b/pattern.go @@ -0,0 +1,187 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Patterns for ServeMux routing. + +package http + +import ( + "errors" + "fmt" + "strings" + "unicode" +) + +// A pattern is something that can be matched against an HTTP request. +// It has an optional method, an optional host, and a path. +type pattern struct { + str string // original string + method string + host string + // The representation of a path differs from the surface syntax, which + // simplifies most algorithms. + // + // Paths ending in '/' are represented with an anonymous "..." wildcard. + // For example, the path "a/" is represented as a literal segment "a" followed + // by a segment with multi==true. + // + // Paths ending in "{$}" are represented with the literal segment "/". + // For example, the path "a/{$}" is represented as a literal segment "a" followed + // by a literal segment "/". + segments []segment + loc string // source location of registering call, for helpful messages +} + +// A segment is a pattern piece that matches one or more path segments, or +// a trailing slash. +// +// If wild is false, it matches a literal segment, or, if s == "/", a trailing slash. +// Examples: +// +// "a" => segment{s: "a"} +// "/{$}" => segment{s: "/"} +// +// If wild is true and multi is false, it matches a single path segment. +// Example: +// +// "{x}" => segment{s: "x", wild: true} +// +// If both wild and multi are true, it matches all remaining path segments. +// Example: +// +// "{rest...}" => segment{s: "rest", wild: true, multi: true} +type segment struct { + s string // literal or wildcard name or "/" for "/{$}". + wild bool + multi bool // "..." wildcard +} + +// parsePattern parses a string into a Pattern. +// The string's syntax is +// +// [METHOD] [HOST]/[PATH] +// +// where: +// - METHOD is an HTTP method +// - HOST is a hostname +// - PATH consists of slash-separated segments, where each segment is either +// a literal or a wildcard of the form "{name}", "{name...}", or "{$}". +// +// METHOD, HOST and PATH are all optional; that is, the string can be "/". +// If METHOD is present, it must be followed by a single space. +// Wildcard names must be valid Go identifiers. +// The "{$}" and "{name...}" wildcard must occur at the end of PATH. +// PATH may end with a '/'. +// Wildcard names in a path must be distinct. +func parsePattern(s string) (*pattern, error) { + if len(s) == 0 { + return nil, errors.New("empty pattern") + } + // TODO(jba): record the rune offset in s to provide more information in errors. + method, rest, found := strings.Cut(s, " ") + if !found { + rest = method + method = "" + } + if method != "" && !validMethod(method) { + return nil, fmt.Errorf("net/http: invalid method %q", method) + } + p := &pattern{str: s, method: method} + + i := strings.IndexByte(rest, '/') + if i < 0 { + return nil, errors.New("host/path missing /") + } + p.host = rest[:i] + rest = rest[i:] + if strings.IndexByte(p.host, '{') >= 0 { + return nil, errors.New("host contains '{' (missing initial '/'?)") + } + // At this point, rest is the path. + + // An unclean path with a method that is not CONNECT can never match, + // because paths are cleaned before matching. + if method != "" && method != "CONNECT" && rest != cleanPath(rest) { + return nil, errors.New("non-CONNECT pattern with unclean path can never match") + } + + seenNames := map[string]bool{} // remember wildcard names to catch dups + for len(rest) > 0 { + // Invariant: rest[0] == '/'. + rest = rest[1:] + if len(rest) == 0 { + // Trailing slash. + p.segments = append(p.segments, segment{wild: true, multi: true}) + break + } + i := strings.IndexByte(rest, '/') + if i < 0 { + i = len(rest) + } + var seg string + seg, rest = rest[:i], rest[i:] + if i := strings.IndexByte(seg, '{'); i < 0 { + // Literal. + p.segments = append(p.segments, segment{s: seg}) + } else { + // Wildcard. + if i != 0 { + return nil, errors.New("bad wildcard segment (must start with '{')") + } + if seg[len(seg)-1] != '}' { + return nil, errors.New("bad wildcard segment (must end with '}')") + } + name := seg[1 : len(seg)-1] + if name == "$" { + if len(rest) != 0 { + return nil, errors.New("{$} not at end") + } + p.segments = append(p.segments, segment{s: "/"}) + break + } + name, multi := strings.CutSuffix(name, "...") + if multi && len(rest) != 0 { + return nil, errors.New("{...} wildcard not at end") + } + if name == "" { + return nil, errors.New("empty wildcard") + } + if !isValidWildcardName(name) { + return nil, fmt.Errorf("bad wildcard name %q", name) + } + if seenNames[name] { + return nil, fmt.Errorf("duplicate wildcard name %q", name) + } + seenNames[name] = true + p.segments = append(p.segments, segment{s: name, wild: true, multi: multi}) + } + } + return p, nil +} + +func isValidHTTPToken(s string) bool { + if s == "" { + return false + } + // See https://www.rfc-editor.org/rfc/rfc9110#section-5.6.2. + for _, r := range s { + if !unicode.IsLetter(r) && !unicode.IsDigit(r) && !strings.ContainsRune("!#$%&'*+.^_`|~-", r) { + return false + } + } + return true +} + +func isValidWildcardName(s string) bool { + if s == "" { + return false + } + // Valid Go identifier. + for i, c := range s { + if !unicode.IsLetter(c) && c != '_' && (i == 0 || !unicode.IsDigit(c)) { + return false + } + } + return true +} diff --git a/pattern_test.go b/pattern_test.go new file mode 100644 index 00000000..759e1267 --- /dev/null +++ b/pattern_test.go @@ -0,0 +1,167 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "slices" + "strings" + "testing" +) + +func TestParsePattern(t *testing.T) { + lit := func(name string) segment { + return segment{s: name} + } + + wild := func(name string) segment { + return segment{s: name, wild: true} + } + + multi := func(name string) segment { + s := wild(name) + s.multi = true + return s + } + + for _, test := range []struct { + in string + want pattern + }{ + {"/", pattern{segments: []segment{multi("")}}}, + {"/a", pattern{segments: []segment{lit("a")}}}, + { + "/a/", + pattern{segments: []segment{lit("a"), multi("")}}, + }, + {"/path/to/something", pattern{segments: []segment{ + lit("path"), lit("to"), lit("something"), + }}}, + { + "/{w1}/lit/{w2}", + pattern{ + segments: []segment{wild("w1"), lit("lit"), wild("w2")}, + }, + }, + { + "/{w1}/lit/{w2}/", + pattern{ + segments: []segment{wild("w1"), lit("lit"), wild("w2"), multi("")}, + }, + }, + { + "example.com/", + pattern{host: "example.com", segments: []segment{multi("")}}, + }, + { + "GET /", + pattern{method: "GET", segments: []segment{multi("")}}, + }, + { + "POST example.com/foo/{w}", + pattern{ + method: "POST", + host: "example.com", + segments: []segment{lit("foo"), wild("w")}, + }, + }, + { + "/{$}", + pattern{segments: []segment{lit("/")}}, + }, + { + "DELETE example.com/a/{foo12}/{$}", + pattern{method: "DELETE", host: "example.com", segments: []segment{lit("a"), wild("foo12"), lit("/")}}, + }, + { + "/foo/{$}", + pattern{segments: []segment{lit("foo"), lit("/")}}, + }, + { + "/{a}/foo/{rest...}", + pattern{segments: []segment{wild("a"), lit("foo"), multi("rest")}}, + }, + { + "//", + pattern{segments: []segment{lit(""), multi("")}}, + }, + { + "/foo///./../bar", + pattern{segments: []segment{lit("foo"), lit(""), lit(""), lit("."), lit(".."), lit("bar")}}, + }, + { + "a.com/foo//", + pattern{host: "a.com", segments: []segment{lit("foo"), lit(""), multi("")}}, + }, + } { + got := mustParsePattern(t, test.in) + if !got.equal(&test.want) { + t.Errorf("%q:\ngot %#v\nwant %#v", test.in, got, &test.want) + } + } +} + +func TestParsePatternError(t *testing.T) { + for _, test := range []struct { + in string + contains string + }{ + {"", "empty pattern"}, + {"A=B /", "invalid method"}, + {" ", "missing /"}, + {"/{w}x", "bad wildcard segment"}, + {"/x{w}", "bad wildcard segment"}, + {"/{wx", "bad wildcard segment"}, + {"/{a$}", "bad wildcard name"}, + {"/{}", "empty wildcard"}, + {"/{...}", "empty wildcard"}, + {"/{$...}", "bad wildcard"}, + {"/{$}/", "{$} not at end"}, + {"/{$}/x", "{$} not at end"}, + {"/{a...}/", "not at end"}, + {"/{a...}/x", "not at end"}, + {"{a}/b", "missing initial '/'"}, + {"/a/{x}/b/{x...}", "duplicate wildcard name"}, + {"GET //", "unclean path"}, + } { + _, err := parsePattern(test.in) + if err == nil || !strings.Contains(err.Error(), test.contains) { + t.Errorf("%q:\ngot %v, want error containing %q", test.in, err, test.contains) + } + } +} + +func (p1 *pattern) equal(p2 *pattern) bool { + return p1.method == p2.method && p1.host == p2.host && + slices.Equal(p1.segments, p2.segments) +} + +func TestIsValidHTTPToken(t *testing.T) { + for _, test := range []struct { + in string + want bool + }{ + {"", false}, + {"GET", true}, + {"get", true}, + {"white space", false}, + {"#!~", true}, + {"a-b1_2", true}, + {"notok)", false}, + } { + got := isValidHTTPToken(test.in) + if g, w := got, test.want; g != w { + t.Errorf("%q: got %t, want %t", test.in, g, w) + } + } +} + +func mustParsePattern(t *testing.T, s string) *pattern { + t.Helper() + p, err := parsePattern(s) + if err != nil { + t.Fatal(err) + } + return p +} From 073eae93214581a238ad1804415d3cd85c5c6e14 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Fri, 8 Sep 2023 08:58:29 -0400 Subject: [PATCH 28/93] net/http: pattern.conflictsWith Add the conflictsWith method, which determines whether two patterns conflict with each other. Updates #61410. Change-Id: Id4f9a471dc9d0420d927a68d2864128a096b74f4 Reviewed-on: https://go-review.googlesource.com/c/go/+/526616 Run-TryBot: Jonathan Amsterdam Reviewed-by: Eli Bendersky Reviewed-by: Damien Neil TryBot-Result: Gopher Robot --- pattern.go | 188 ++++++++++++++++++++++++++++++++++++++++ pattern_test.go | 224 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 412 insertions(+) diff --git a/pattern.go b/pattern.go index a04fd901..3fd20b71 100644 --- a/pattern.go +++ b/pattern.go @@ -33,6 +33,12 @@ type pattern struct { loc string // source location of registering call, for helpful messages } +func (p *pattern) String() string { return p.str } + +func (p *pattern) lastSegment() segment { + return p.segments[len(p.segments)-1] +} + // A segment is a pattern piece that matches one or more path segments, or // a trailing slash. // @@ -185,3 +191,185 @@ func isValidWildcardName(s string) bool { } return true } + +// relationship is a relationship between two patterns, p1 and p2. +type relationship string + +const ( + equivalent relationship = "equivalent" // both match the same requests + moreGeneral relationship = "moreGeneral" // p1 matches everything p2 does & more + moreSpecific relationship = "moreSpecific" // p2 matches everything p1 does & more + disjoint relationship = "disjoint" // there is no request that both match + overlaps relationship = "overlaps" // there is a request that both match, but neither is more specific +) + +// conflictsWith reports whether p1 conflicts with p2, that is, whether +// there is a request that both match but where neither is higher precedence +// than the other. +// +// Precedence is defined by two rules: +// 1. Patterns with a host win over patterns without a host. +// 2. Patterns whose method and path is more specific win. One pattern is more +// specific than another if the second matches all the (method, path) pairs +// of the first and more. +// +// If rule 1 doesn't apply, then two patterns conflict if their relationship +// is either equivalence (they match the same set of requests) or overlap +// (they both match some requests, but neither is more specific than the other). +func (p1 *pattern) conflictsWith(p2 *pattern) bool { + if p1.host != p2.host { + // Either one host is empty and the other isn't, in which case the + // one with the host wins by rule 1, or neither host is empty + // and they differ, so they won't match the same paths. + return false + } + rel := p1.comparePathsAndMethods(p2) + return rel == equivalent || rel == overlaps +} + +func (p1 *pattern) comparePathsAndMethods(p2 *pattern) relationship { + mrel := p1.compareMethods(p2) + // Optimization: avoid a call to comparePaths. + if mrel == disjoint { + return disjoint + } + prel := p1.comparePaths(p2) + return combineRelationships(mrel, prel) +} + +// compareMethods determines the relationship between the method +// part of patterns p1 and p2. +// +// A method can either be empty, "GET", or something else. +// The empty string matches any method, so it is the most general. +// "GET" matches both GET and HEAD. +// Anything else matches only itself. +func (p1 *pattern) compareMethods(p2 *pattern) relationship { + if p1.method == p2.method { + return equivalent + } + if p1.method == "" { + // p1 matches any method, but p2 does not, so p1 is more general. + return moreGeneral + } + if p2.method == "" { + return moreSpecific + } + if p1.method == "GET" && p2.method == "HEAD" { + // p1 matches GET and HEAD; p2 matches only HEAD. + return moreGeneral + } + if p2.method == "GET" && p1.method == "HEAD" { + return moreSpecific + } + return disjoint +} + +// comparePaths determines the relationship between the path +// part of two patterns. +func (p1 *pattern) comparePaths(p2 *pattern) relationship { + // Optimization: if a path pattern doesn't end in a multi ("...") wildcard, then it + // can only match paths with the same number of segments. + if len(p1.segments) != len(p2.segments) && !p1.lastSegment().multi && !p2.lastSegment().multi { + return disjoint + } + var segs1, segs2 []segment + // Look at corresponding segments in the two path patterns. + rel := equivalent + for segs1, segs2 = p1.segments, p2.segments; len(segs1) > 0 && len(segs2) > 0; segs1, segs2 = segs1[1:], segs2[1:] { + rel = combineRelationships(rel, compareSegments(segs1[0], segs2[0])) + if rel == disjoint || rel == overlaps { + return rel + } + } + // We've reached the end of the corresponding segments of the patterns. + // If they have the same number of segments, then we've already determined + // their relationship. + if len(segs1) == 0 && len(segs2) == 0 { + return rel + } + // Otherwise, the only way they could fail to be disjoint is if the shorter + // pattern ends in a multi and is more general. + if len(segs1) < len(segs2) && p1.lastSegment().multi && rel == moreGeneral { + return moreGeneral + } + if len(segs2) < len(segs1) && p2.lastSegment().multi && rel == moreSpecific { + return moreSpecific + } + return disjoint +} + +// compareSegments determines the relationship between two segments. +func compareSegments(s1, s2 segment) relationship { + if s1.multi && s2.multi { + return equivalent + } + if s1.multi { + return moreGeneral + } + if s2.multi { + return moreSpecific + } + if s1.wild && s2.wild { + return equivalent + } + if s1.wild { + if s2.s == "/" { + // A single wildcard doesn't match a trailing slash. + return disjoint + } + return moreGeneral + } + if s2.wild { + if s1.s == "/" { + return disjoint + } + return moreSpecific + } + // Both literals. + if s1.s == s2.s { + return equivalent + } + return disjoint +} + +// combineRelationships determines the overall relationship of two patterns +// given the relationships of a partition of the patterns into two parts. +// +// For example, if p1 is more general than p2 in one way but equivalent +// in the other, then it is more general overall. +// +// Or if p1 is more general in one way and more specific in the other, then +// they overlap. +func combineRelationships(r1, r2 relationship) relationship { + switch r1 { + case equivalent: + return r2 + case disjoint, overlaps: + return r1 + case moreGeneral, moreSpecific: + switch r2 { + case equivalent: + return r1 + case inverseRelationship(r1): + return overlaps + default: + return r2 + } + default: + panic(fmt.Sprintf("unknown relationship %q", r1)) + } +} + +// If p1 has relationship `r` to p2, then +// p2 has inverseRelationship(r) to p1. +func inverseRelationship(r relationship) relationship { + switch r { + case moreSpecific: + return moreGeneral + case moreGeneral: + return moreSpecific + default: + return r + } +} diff --git a/pattern_test.go b/pattern_test.go index 759e1267..cd27cd8d 100644 --- a/pattern_test.go +++ b/pattern_test.go @@ -165,3 +165,227 @@ func mustParsePattern(t *testing.T, s string) *pattern { } return p } + +func TestCompareMethods(t *testing.T) { + for _, test := range []struct { + p1, p2 string + want relationship + }{ + {"/", "/", equivalent}, + {"GET /", "GET /", equivalent}, + {"HEAD /", "HEAD /", equivalent}, + {"POST /", "POST /", equivalent}, + {"GET /", "POST /", disjoint}, + {"GET /", "/", moreSpecific}, + {"HEAD /", "/", moreSpecific}, + {"GET /", "HEAD /", moreGeneral}, + } { + pat1 := mustParsePattern(t, test.p1) + pat2 := mustParsePattern(t, test.p2) + got := pat1.compareMethods(pat2) + if got != test.want { + t.Errorf("%s vs %s: got %s, want %s", test.p1, test.p2, got, test.want) + } + got2 := pat2.compareMethods(pat1) + want2 := inverseRelationship(test.want) + if got2 != want2 { + t.Errorf("%s vs %s: got %s, want %s", test.p2, test.p1, got2, want2) + } + } +} + +func TestComparePaths(t *testing.T) { + for _, test := range []struct { + p1, p2 string + want relationship + }{ + // A non-final pattern segment can have one of two values: literal or + // single wildcard. A final pattern segment can have one of 5: empty + // (trailing slash), literal, dollar, single wildcard, or multi + // wildcard. Trailing slash and multi wildcard are the same. + + // A literal should be more specific than anything it overlaps, except itself. + {"/a", "/a", equivalent}, + {"/a", "/b", disjoint}, + {"/a", "/", moreSpecific}, + {"/a", "/{$}", disjoint}, + {"/a", "/{x}", moreSpecific}, + {"/a", "/{x...}", moreSpecific}, + + // Adding a segment doesn't change that. + {"/b/a", "/b/a", equivalent}, + {"/b/a", "/b/b", disjoint}, + {"/b/a", "/b/", moreSpecific}, + {"/b/a", "/b/{$}", disjoint}, + {"/b/a", "/b/{x}", moreSpecific}, + {"/b/a", "/b/{x...}", moreSpecific}, + {"/{z}/a", "/{z}/a", equivalent}, + {"/{z}/a", "/{z}/b", disjoint}, + {"/{z}/a", "/{z}/", moreSpecific}, + {"/{z}/a", "/{z}/{$}", disjoint}, + {"/{z}/a", "/{z}/{x}", moreSpecific}, + {"/{z}/a", "/{z}/{x...}", moreSpecific}, + + // Single wildcard on left. + {"/{z}", "/a", moreGeneral}, + {"/{z}", "/a/b", disjoint}, + {"/{z}", "/{$}", disjoint}, + {"/{z}", "/{x}", equivalent}, + {"/{z}", "/", moreSpecific}, + {"/{z}", "/{x...}", moreSpecific}, + {"/b/{z}", "/b/a", moreGeneral}, + {"/b/{z}", "/b/a/b", disjoint}, + {"/b/{z}", "/b/{$}", disjoint}, + {"/b/{z}", "/b/{x}", equivalent}, + {"/b/{z}", "/b/", moreSpecific}, + {"/b/{z}", "/b/{x...}", moreSpecific}, + + // Trailing slash on left. + {"/", "/a", moreGeneral}, + {"/", "/a/b", moreGeneral}, + {"/", "/{$}", moreGeneral}, + {"/", "/{x}", moreGeneral}, + {"/", "/", equivalent}, + {"/", "/{x...}", equivalent}, + + {"/b/", "/b/a", moreGeneral}, + {"/b/", "/b/a/b", moreGeneral}, + {"/b/", "/b/{$}", moreGeneral}, + {"/b/", "/b/{x}", moreGeneral}, + {"/b/", "/b/", equivalent}, + {"/b/", "/b/{x...}", equivalent}, + + {"/{z}/", "/{z}/a", moreGeneral}, + {"/{z}/", "/{z}/a/b", moreGeneral}, + {"/{z}/", "/{z}/{$}", moreGeneral}, + {"/{z}/", "/{z}/{x}", moreGeneral}, + {"/{z}/", "/{z}/", equivalent}, + {"/{z}/", "/a/", moreGeneral}, + {"/{z}/", "/{z}/{x...}", equivalent}, + {"/{z}/", "/a/{x...}", moreGeneral}, + {"/a/{z}/", "/{z}/a/", overlaps}, + {"/a/{z}/b/", "/{x}/c/{y...}", overlaps}, + + // Multi wildcard on left. + {"/{m...}", "/a", moreGeneral}, + {"/{m...}", "/a/b", moreGeneral}, + {"/{m...}", "/{$}", moreGeneral}, + {"/{m...}", "/{x}", moreGeneral}, + {"/{m...}", "/", equivalent}, + {"/{m...}", "/{x...}", equivalent}, + + {"/b/{m...}", "/b/a", moreGeneral}, + {"/b/{m...}", "/b/a/b", moreGeneral}, + {"/b/{m...}", "/b/{$}", moreGeneral}, + {"/b/{m...}", "/b/{x}", moreGeneral}, + {"/b/{m...}", "/b/", equivalent}, + {"/b/{m...}", "/b/{x...}", equivalent}, + {"/b/{m...}", "/a/{x...}", disjoint}, + + {"/{z}/{m...}", "/{z}/a", moreGeneral}, + {"/{z}/{m...}", "/{z}/a/b", moreGeneral}, + {"/{z}/{m...}", "/{z}/{$}", moreGeneral}, + {"/{z}/{m...}", "/{z}/{x}", moreGeneral}, + {"/{z}/{m...}", "/{w}/", equivalent}, + {"/{z}/{m...}", "/a/", moreGeneral}, + {"/{z}/{m...}", "/{z}/{x...}", equivalent}, + {"/{z}/{m...}", "/a/{x...}", moreGeneral}, + {"/a/{m...}", "/a/b/{y...}", moreGeneral}, + {"/a/{m...}", "/a/{x}/{y...}", moreGeneral}, + {"/a/{z}/{m...}", "/a/b/{y...}", moreGeneral}, + {"/a/{z}/{m...}", "/{z}/a/", overlaps}, + {"/a/{z}/{m...}", "/{z}/b/{y...}", overlaps}, + {"/a/{z}/b/{m...}", "/{x}/c/{y...}", overlaps}, + + // Dollar on left. + {"/{$}", "/a", disjoint}, + {"/{$}", "/a/b", disjoint}, + {"/{$}", "/{$}", equivalent}, + {"/{$}", "/{x}", disjoint}, + {"/{$}", "/", moreSpecific}, + {"/{$}", "/{x...}", moreSpecific}, + + {"/b/{$}", "/b", disjoint}, + {"/b/{$}", "/b/a", disjoint}, + {"/b/{$}", "/b/a/b", disjoint}, + {"/b/{$}", "/b/{$}", equivalent}, + {"/b/{$}", "/b/{x}", disjoint}, + {"/b/{$}", "/b/", moreSpecific}, + {"/b/{$}", "/b/{x...}", moreSpecific}, + {"/b/{$}", "/b/c/{x...}", disjoint}, + {"/b/{x}/a/{$}", "/{x}/c/{y...}", overlaps}, + + {"/{z}/{$}", "/{z}/a", disjoint}, + {"/{z}/{$}", "/{z}/a/b", disjoint}, + {"/{z}/{$}", "/{z}/{$}", equivalent}, + {"/{z}/{$}", "/{z}/{x}", disjoint}, + {"/{z}/{$}", "/{z}/", moreSpecific}, + {"/{z}/{$}", "/a/", overlaps}, + {"/{z}/{$}", "/a/{x...}", overlaps}, + {"/{z}/{$}", "/{z}/{x...}", moreSpecific}, + {"/a/{z}/{$}", "/{z}/a/", overlaps}, + } { + pat1 := mustParsePattern(t, test.p1) + pat2 := mustParsePattern(t, test.p2) + if g := pat1.comparePaths(pat1); g != equivalent { + t.Errorf("%s does not match itself; got %s", pat1, g) + } + if g := pat2.comparePaths(pat2); g != equivalent { + t.Errorf("%s does not match itself; got %s", pat2, g) + } + got := pat1.comparePaths(pat2) + if got != test.want { + t.Errorf("%s vs %s: got %s, want %s", test.p1, test.p2, got, test.want) + t.Logf("pat1: %+v\n", pat1.segments) + t.Logf("pat2: %+v\n", pat2.segments) + } + want2 := inverseRelationship(test.want) + got2 := pat2.comparePaths(pat1) + if got2 != want2 { + t.Errorf("%s vs %s: got %s, want %s", test.p2, test.p1, got2, want2) + } + } +} + +func TestConflictsWith(t *testing.T) { + for _, test := range []struct { + p1, p2 string + want bool + }{ + {"/a", "/a", true}, + {"/a", "/ab", false}, + {"/a/b/cd", "/a/b/cd", true}, + {"/a/b/cd", "/a/b/c", false}, + {"/a/b/c", "/a/c/c", false}, + {"/{x}", "/{y}", true}, + {"/{x}", "/a", false}, // more specific + {"/{x}/{y}", "/{x}/a", false}, + {"/{x}/{y}", "/{x}/a/b", false}, + {"/{x}", "/a/{y}", false}, + {"/{x}/{y}", "/{x}/a/", false}, + {"/{x}", "/a/{y...}", false}, // more specific + {"/{x}/a/{y}", "/{x}/a/{y...}", false}, // more specific + {"/{x}/{y}", "/{x}/a/{$}", false}, // more specific + {"/{x}/{y}/{$}", "/{x}/a/{$}", false}, + {"/a/{x}", "/{x}/b", true}, + {"/", "GET /", false}, + {"/", "GET /foo", false}, + {"GET /", "GET /foo", false}, + {"GET /", "/foo", true}, + {"GET /foo", "HEAD /", true}, + } { + pat1 := mustParsePattern(t, test.p1) + pat2 := mustParsePattern(t, test.p2) + got := pat1.conflictsWith(pat2) + if got != test.want { + t.Errorf("%q.ConflictsWith(%q) = %t, want %t", + test.p1, test.p2, got, test.want) + } + // conflictsWith should be commutative. + got = pat2.conflictsWith(pat1) + if got != test.want { + t.Errorf("%q.ConflictsWith(%q) = %t, want %t", + test.p2, test.p1, got, test.want) + } + } +} From bb0da64f9f9d2c4c3b8529b4e073f1313bc69454 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Mon, 11 Sep 2023 10:59:48 -0400 Subject: [PATCH 29/93] net/http: mapping data structure Our goal for the new ServeMux patterns is to match the routing performance of the existing ServeMux patterns. To achieve that we needed to optimize lookup for small maps. This CL introduces a simple data structure called a mapping that optimizes lookup by using a slice for small collections of key-value pairs, switching to a map when the collection gets large. Mappings are a core part of the routing algorithm, which uses a decision tree to match path elements. The children of a tree node are held in a mapping. Change-Id: I923b3ad1376ace2c3e3421aa9b802ad12d47c871 Reviewed-on: https://go-review.googlesource.com/c/go/+/526617 Run-TryBot: Jonathan Amsterdam TryBot-Result: Gopher Robot Reviewed-by: Eli Bendersky Reviewed-by: Damien Neil --- mapping.go | 78 ++++++++++++++++++++++++ mapping_test.go | 154 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 232 insertions(+) create mode 100644 mapping.go create mode 100644 mapping_test.go diff --git a/mapping.go b/mapping.go new file mode 100644 index 00000000..87e6d5ae --- /dev/null +++ b/mapping.go @@ -0,0 +1,78 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +// A mapping is a collection of key-value pairs where the keys are unique. +// A zero mapping is empty and ready to use. +// A mapping tries to pick a representation that makes [mapping.find] most efficient. +type mapping[K comparable, V any] struct { + s []entry[K, V] // for few pairs + m map[K]V // for many pairs +} + +type entry[K comparable, V any] struct { + key K + value V +} + +// maxSlice is the maximum number of pairs for which a slice is used. +// It is a variable for benchmarking. +var maxSlice int = 8 + +// add adds a key-value pair to the mapping. +func (h *mapping[K, V]) add(k K, v V) { + if h.m == nil && len(h.s) < maxSlice { + h.s = append(h.s, entry[K, V]{k, v}) + } else { + if h.m == nil { + h.m = map[K]V{} + for _, e := range h.s { + h.m[e.key] = e.value + } + h.s = nil + } + h.m[k] = v + } +} + +// find returns the value corresponding to the given key. +// The second return value is false if there is no value +// with that key. +func (h *mapping[K, V]) find(k K) (v V, found bool) { + if h == nil { + return v, false + } + if h.m != nil { + v, found = h.m[k] + return v, found + } + for _, e := range h.s { + if e.key == k { + return e.value, true + } + } + return v, false +} + +// eachPair calls f for each pair in the mapping. +// If f returns false, pairs returns immediately. +func (h *mapping[K, V]) eachPair(f func(k K, v V) bool) { + if h == nil { + return + } + if h.m != nil { + for k, v := range h.m { + if !f(k, v) { + return + } + } + } else { + for _, e := range h.s { + if !f(e.key, e.value) { + return + } + } + } +} diff --git a/mapping_test.go b/mapping_test.go new file mode 100644 index 00000000..0aed9d9e --- /dev/null +++ b/mapping_test.go @@ -0,0 +1,154 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "cmp" + "fmt" + "slices" + "strconv" + "testing" +) + +func TestMapping(t *testing.T) { + var m mapping[int, string] + for i := 0; i < maxSlice; i++ { + m.add(i, strconv.Itoa(i)) + } + if m.m != nil { + t.Fatal("m.m != nil") + } + for i := 0; i < maxSlice; i++ { + g, _ := m.find(i) + w := strconv.Itoa(i) + if g != w { + t.Fatalf("%d: got %s, want %s", i, g, w) + } + } + m.add(4, "4") + if m.s != nil { + t.Fatal("m.s != nil") + } + if m.m == nil { + t.Fatal("m.m == nil") + } + g, _ := m.find(4) + if w := "4"; g != w { + t.Fatalf("got %s, want %s", g, w) + } +} + +func TestMappingEachPair(t *testing.T) { + var m mapping[int, string] + var want []entry[int, string] + for i := 0; i < maxSlice*2; i++ { + v := strconv.Itoa(i) + m.add(i, v) + want = append(want, entry[int, string]{i, v}) + + } + + var got []entry[int, string] + m.eachPair(func(k int, v string) bool { + got = append(got, entry[int, string]{k, v}) + return true + }) + slices.SortFunc(got, func(e1, e2 entry[int, string]) int { + return cmp.Compare(e1.key, e2.key) + }) + if !slices.Equal(got, want) { + t.Errorf("got %v, want %v", got, want) + } +} + +func BenchmarkFindChild(b *testing.B) { + key := "articles" + children := []string{ + "*", + "cmd.html", + "code.html", + "contrib.html", + "contribute.html", + "debugging_with_gdb.html", + "docs.html", + "effective_go.html", + "files.log", + "gccgo_contribute.html", + "gccgo_install.html", + "go-logo-black.png", + "go-logo-blue.png", + "go-logo-white.png", + "go1.1.html", + "go1.2.html", + "go1.html", + "go1compat.html", + "go_faq.html", + "go_mem.html", + "go_spec.html", + "help.html", + "ie.css", + "install-source.html", + "install.html", + "logo-153x55.png", + "Makefile", + "root.html", + "share.png", + "sieve.gif", + "tos.html", + "articles", + } + if len(children) != 32 { + panic("bad len") + } + for _, n := range []int{2, 4, 8, 16, 32} { + list := children[:n] + b.Run(fmt.Sprintf("n=%d", n), func(b *testing.B) { + + b.Run("rep=linear", func(b *testing.B) { + var entries []entry[string, any] + for _, c := range list { + entries = append(entries, entry[string, any]{c, nil}) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + findChildLinear(key, entries) + } + }) + b.Run("rep=map", func(b *testing.B) { + m := map[string]any{} + for _, c := range list { + m[c] = nil + } + var x any + b.ResetTimer() + for i := 0; i < b.N; i++ { + x = m[key] + } + _ = x + }) + b.Run(fmt.Sprintf("rep=hybrid%d", maxSlice), func(b *testing.B) { + var h mapping[string, any] + for _, c := range list { + h.add(c, nil) + } + var x any + b.ResetTimer() + for i := 0; i < b.N; i++ { + x, _ = h.find(key) + } + _ = x + }) + }) + } +} + +func findChildLinear(key string, entries []entry[string, any]) any { + for _, e := range entries { + if key == e.key { + return e.value + } + } + return nil +} From 0d0a3ef9f2dbfa99ac5650ab0146f15776340795 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Mon, 11 Sep 2023 12:09:40 -0400 Subject: [PATCH 30/93] net/http: routing tree This CL implements a decision tree for efficient routing. The tree holds all the registered patterns. To match a request, we walk the tree looking for a match. Change-Id: I7ed1cdf585fc95b73ef5ca2f942f278100a90583 Reviewed-on: https://go-review.googlesource.com/c/go/+/527315 TryBot-Result: Gopher Robot Reviewed-by: Damien Neil Run-TryBot: Jonathan Amsterdam --- routing_tree.go | 222 ++++++++++++++++++++++++++++++++++++++++ routing_tree_test.go | 234 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 456 insertions(+) create mode 100644 routing_tree.go create mode 100644 routing_tree_test.go diff --git a/routing_tree.go b/routing_tree.go new file mode 100644 index 00000000..e225b5fd --- /dev/null +++ b/routing_tree.go @@ -0,0 +1,222 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file implements a decision tree for fast matching of requests to +// patterns. +// +// The root of the tree branches on the host of the request. +// The next level branches on the method. +// The remaining levels branch on consecutive segments of the path. +// +// The "more specific wins" precedence rule can result in backtracking. +// For example, given the patterns +// /a/b/z +// /a/{x}/c +// we will first try to match the path "/a/b/c" with /a/b/z, and +// when that fails we will try against /a/{x}/c. + +package http + +import ( + "net/url" + "strings" +) + +// A routingNode is a node in the decision tree. +// The same struct is used for leaf and interior nodes. +type routingNode struct { + // A leaf node holds a single pattern and the Handler it was registered + // with. + pattern *pattern + handler Handler + + // An interior node maps parts of the incoming request to child nodes. + // special children keys: + // "/" trailing slash (resulting from {$}) + // "" single wildcard + // "*" multi wildcard + children mapping[string, *routingNode] + emptyChild *routingNode // optimization: child with key "" +} + +// addPattern adds a pattern and its associated Handler to the tree +// at root. +func (root *routingNode) addPattern(p *pattern, h Handler) { + // First level of tree is host. + n := root.addChild(p.host) + // Second level of tree is method. + n = n.addChild(p.method) + // Remaining levels are path. + n.addSegments(p.segments, p, h) +} + +// addSegments adds the given segments to the tree rooted at n. +// If there are no segments, then n is a leaf node that holds +// the given pattern and handler. +func (n *routingNode) addSegments(segs []segment, p *pattern, h Handler) { + if len(segs) == 0 { + n.set(p, h) + return + } + seg := segs[0] + if seg.multi { + if len(segs) != 1 { + panic("multi wildcard not last") + } + n.addChild("*").set(p, h) + } else if seg.wild { + n.addChild("").addSegments(segs[1:], p, h) + } else { + n.addChild(seg.s).addSegments(segs[1:], p, h) + } +} + +// set sets the pattern and handler for n, which +// must be a leaf node. +func (n *routingNode) set(p *pattern, h Handler) { + if n.pattern != nil || n.handler != nil { + panic("non-nil leaf fields") + } + n.pattern = p + n.handler = h +} + +// addChild adds a child node with the given key to n +// if one does not exist, and returns the child. +func (n *routingNode) addChild(key string) *routingNode { + if key == "" { + if n.emptyChild == nil { + n.emptyChild = &routingNode{} + } + return n.emptyChild + } + if c := n.findChild(key); c != nil { + return c + } + c := &routingNode{} + n.children.add(key, c) + return c +} + +// findChild returns the child of n with the given key, or nil +// if there is no child with that key. +func (n *routingNode) findChild(key string) *routingNode { + if key == "" { + return n.emptyChild + } + r, _ := n.children.find(key) + return r +} + +// match returns the leaf node under root that matches the arguments, and a list +// of values for pattern wildcards in the order that the wildcards appear. +// For example, if the request path is "/a/b/c" and the pattern is "/{x}/b/{y}", +// then the second return value will be []string{"a", "c"}. +func (root *routingNode) match(host, method, path string) (*routingNode, []string) { + if host != "" { + // There is a host. If there is a pattern that specifies that host and it + // matches, we are done. If the pattern doesn't match, fall through to + // try patterns with no host. + if l, m := root.findChild(host).matchMethodAndPath(method, path); l != nil { + return l, m + } + } + return root.emptyChild.matchMethodAndPath(method, path) +} + +// matchMethodAndPath matches the method and path. +// Its return values are the same as [routingNode.match]. +// The receiver should be a child of the root. +func (n *routingNode) matchMethodAndPath(method, path string) (*routingNode, []string) { + if n == nil { + return nil, nil + } + if l, m := n.findChild(method).matchPath(path, nil); l != nil { + // Exact match of method name. + return l, m + } + if method == "HEAD" { + // GET matches HEAD too. + if l, m := n.findChild("GET").matchPath(path, nil); l != nil { + return l, m + } + } + // No exact match; try patterns with no method. + return n.emptyChild.matchPath(path, nil) +} + +// matchPath matches a path. +// Its return values are the same as [routingNode.match]. +// matchPath calls itself recursively. The matches argument holds the wildcard matches +// found so far. +func (n *routingNode) matchPath(path string, matches []string) (*routingNode, []string) { + if n == nil { + return nil, nil + } + // If path is empty, then we are done. + // If n is a leaf node, we found a match; return it. + // If n is an interior node (which means it has a nil pattern), + // then we failed to match. + if path == "" { + if n.pattern == nil { + return nil, nil + } + return n, matches + } + // Get the first segment of path. + seg, rest := firstSegment(path) + // First try matching against patterns that have a literal for this position. + // We know by construction that such patterns are more specific than those + // with a wildcard at this position (they are either more specific, equivalent, + // or overlap, and we ruled out the first two when the patterns were registered). + if n, m := n.findChild(seg).matchPath(rest, matches); n != nil { + return n, m + } + // If matching a literal fails, try again with patterns that have a single + // wildcard (represented by an empty string in the child mapping). + // Again, by construction, patterns with a single wildcard must be more specific than + // those with a multi wildcard. + // We skip this step if the segment is a trailing slash, because single wildcards + // don't match trailing slashes. + if seg != "/" { + if n, m := n.emptyChild.matchPath(rest, append(matches, matchValue(seg))); n != nil { + return n, m + } + } + // Lastly, match the pattern (there can be at most one) that has a multi + // wildcard in this position to the rest of the path. + if c := n.findChild("*"); c != nil { + // Don't record a match for a nameless wildcard (which arises from a + // trailing slash in the pattern). + if c.pattern.lastSegment().s != "" { + matches = append(matches, matchValue(path[1:])) // remove initial slash + } + return c, matches + } + return nil, nil +} + +func matchValue(path string) string { + m, err := url.PathUnescape(path) + if err != nil { + // Path is not properly escaped, so use the original. + return path + } + return m +} + +// firstSegment splits path into its first segment, and the rest. +// The path must begin with "/". +// If path consists of only a slash, firstSegment returns ("/", ""). +func firstSegment(path string) (seg, rest string) { + if path == "/" { + return "/", "" + } + path = path[1:] // drop initial slash + i := strings.IndexByte(path, '/') + if i < 0 { + return path, "" + } + return path[:i], path[i:] +} diff --git a/routing_tree_test.go b/routing_tree_test.go new file mode 100644 index 00000000..42d7b995 --- /dev/null +++ b/routing_tree_test.go @@ -0,0 +1,234 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "fmt" + "io" + "sort" + "strings" + "testing" + + "slices" +) + +func TestRoutingFirstSegment(t *testing.T) { + for _, test := range []struct { + in string + want []string + }{ + {"/a/b/c", []string{"a", "b", "c"}}, + {"/a/b/", []string{"a", "b", "/"}}, + {"/", []string{"/"}}, + } { + var got []string + rest := test.in + for len(rest) > 0 { + var seg string + seg, rest = firstSegment(rest) + got = append(got, seg) + } + if !slices.Equal(got, test.want) { + t.Errorf("%q: got %v, want %v", test.in, got, test.want) + } + } +} + +// TODO: test host and method +var testTree *routingNode + +func getTestTree() *routingNode { + if testTree == nil { + testTree = buildTree("/a", "/a/b", "/a/{x}", + "/g/h/i", "/g/{x}/j", + "/a/b/{x...}", "/a/b/{y}", "/a/b/{$}") + } + return testTree +} + +func buildTree(pats ...string) *routingNode { + root := &routingNode{} + for _, p := range pats { + pat, err := parsePattern(p) + if err != nil { + panic(err) + } + root.addPattern(pat, nil) + } + return root +} + +func TestRoutingAddPattern(t *testing.T) { + want := `"": + "": + "a": + "/a" + "": + "/a/{x}" + "b": + "/a/b" + "": + "/a/b/{y}" + "*": + "/a/b/{x...}" + "/": + "/a/b/{$}" + "g": + "": + "j": + "/g/{x}/j" + "h": + "i": + "/g/h/i" +` + + var b strings.Builder + getTestTree().print(&b, 0) + got := b.String() + if got != want { + t.Errorf("got\n%s\nwant\n%s", got, want) + } +} + +type testCase struct { + method, host, path string + wantPat string // "" for nil (no match) + wantMatches []string +} + +func TestRoutingNodeMatch(t *testing.T) { + + test := func(tree *routingNode, tests []testCase) { + t.Helper() + for _, test := range tests { + gotNode, gotMatches := tree.match(test.host, test.method, test.path) + got := "" + if gotNode != nil { + got = gotNode.pattern.String() + } + if got != test.wantPat { + t.Errorf("%s, %s, %s: got %q, want %q", test.host, test.method, test.path, got, test.wantPat) + } + if !slices.Equal(gotMatches, test.wantMatches) { + t.Errorf("%s, %s, %s: got matches %v, want %v", test.host, test.method, test.path, gotMatches, test.wantMatches) + } + } + } + + test(getTestTree(), []testCase{ + {"GET", "", "/a", "/a", nil}, + {"Get", "", "/b", "", nil}, + {"Get", "", "/a/b", "/a/b", nil}, + {"Get", "", "/a/c", "/a/{x}", []string{"c"}}, + {"Get", "", "/a/b/", "/a/b/{$}", nil}, + {"Get", "", "/a/b/c", "/a/b/{y}", []string{"c"}}, + {"Get", "", "/a/b/c/d", "/a/b/{x...}", []string{"c/d"}}, + {"Get", "", "/g/h/i", "/g/h/i", nil}, + {"Get", "", "/g/h/j", "/g/{x}/j", []string{"h"}}, + }) + + tree := buildTree( + "/item/", + "POST /item/{user}", + "GET /item/{user}", + "/item/{user}", + "/item/{user}/{id}", + "/item/{user}/new", + "/item/{$}", + "POST alt.com/item/{user}", + "GET /headwins", + "HEAD /headwins", + "/path/{p...}") + + test(tree, []testCase{ + {"GET", "", "/item/jba", + "GET /item/{user}", []string{"jba"}}, + {"POST", "", "/item/jba", + "POST /item/{user}", []string{"jba"}}, + {"HEAD", "", "/item/jba", + "GET /item/{user}", []string{"jba"}}, + {"get", "", "/item/jba", + "/item/{user}", []string{"jba"}}, // method matches are case-sensitive + {"POST", "", "/item/jba/17", + "/item/{user}/{id}", []string{"jba", "17"}}, + {"GET", "", "/item/jba/new", + "/item/{user}/new", []string{"jba"}}, + {"GET", "", "/item/", + "/item/{$}", []string{}}, + {"GET", "", "/item/jba/17/line2", + "/item/", nil}, + {"POST", "alt.com", "/item/jba", + "POST alt.com/item/{user}", []string{"jba"}}, + {"GET", "alt.com", "/item/jba", + "GET /item/{user}", []string{"jba"}}, + {"GET", "", "/item", + "", nil}, // does not match + {"GET", "", "/headwins", + "GET /headwins", nil}, + {"HEAD", "", "/headwins", // HEAD is more specific than GET + "HEAD /headwins", nil}, + {"GET", "", "/path/to/file", + "/path/{p...}", []string{"to/file"}}, + }) + + // A pattern ending in {$} should only match URLS with a trailing slash. + pat1 := "/a/b/{$}" + test(buildTree(pat1), []testCase{ + {"GET", "", "/a/b", "", nil}, + {"GET", "", "/a/b/", pat1, nil}, + {"GET", "", "/a/b/c", "", nil}, + {"GET", "", "/a/b/c/d", "", nil}, + }) + + // A pattern ending in a single wildcard should not match a trailing slash URL. + pat2 := "/a/b/{w}" + test(buildTree(pat2), []testCase{ + {"GET", "", "/a/b", "", nil}, + {"GET", "", "/a/b/", "", nil}, + {"GET", "", "/a/b/c", pat2, []string{"c"}}, + {"GET", "", "/a/b/c/d", "", nil}, + }) + + // A pattern ending in a multi wildcard should match both URLs. + pat3 := "/a/b/{w...}" + test(buildTree(pat3), []testCase{ + {"GET", "", "/a/b", "", nil}, + {"GET", "", "/a/b/", pat3, []string{""}}, + {"GET", "", "/a/b/c", pat3, []string{"c"}}, + {"GET", "", "/a/b/c/d", pat3, []string{"c/d"}}, + }) + + // All three of the above should work together. + test(buildTree(pat1, pat2, pat3), []testCase{ + {"GET", "", "/a/b", "", nil}, + {"GET", "", "/a/b/", pat1, nil}, + {"GET", "", "/a/b/c", pat2, []string{"c"}}, + {"GET", "", "/a/b/c/d", pat3, []string{"c/d"}}, + }) +} + +func (n *routingNode) print(w io.Writer, level int) { + indent := strings.Repeat(" ", level) + if n.pattern != nil { + fmt.Fprintf(w, "%s%q\n", indent, n.pattern) + } + if n.emptyChild != nil { + fmt.Fprintf(w, "%s%q:\n", indent, "") + n.emptyChild.print(w, level+1) + } + + var keys []string + n.children.eachPair(func(k string, _ *routingNode) bool { + keys = append(keys, k) + return true + }) + sort.Strings(keys) + + for _, k := range keys { + fmt.Fprintf(w, "%s%q:\n", indent, k) + n, _ := n.children.find(k) + n.print(w, level+1) + } +} From 6097cd85e2ba959d7f16df546bbc34a17fa940a8 Mon Sep 17 00:00:00 2001 From: Abhinav Gupta Date: Wed, 13 Sep 2023 15:55:06 +0000 Subject: [PATCH 31/93] net/http: use new Go Doc list syntax This tweaks the documentation for http.Client to use the list syntax introduced in Go 1.19. Change-Id: I1f7e0256c13f57e04fc76e5e2362608c8f9f524d GitHub-Last-Rev: 11d384f9adb25605d44dbb7aaeec88fbb3b457ed GitHub-Pull-Request: golang/go#62574 Reviewed-on: https://go-review.googlesource.com/c/go/+/527335 LUCI-TryBot-Result: Go LUCI Reviewed-by: Ian Lance Taylor Reviewed-by: Heschi Kreinick Auto-Submit: Ian Lance Taylor --- client.go | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/client.go b/client.go index 5478690e..5fd86a1e 100644 --- a/client.go +++ b/client.go @@ -41,20 +41,19 @@ import ( // When following redirects, the Client will forward all headers set on the // initial Request except: // -// • when forwarding sensitive headers like "Authorization", -// "WWW-Authenticate", and "Cookie" to untrusted targets. -// These headers will be ignored when following a redirect to a domain -// that is not a subdomain match or exact match of the initial domain. -// For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com" -// will forward the sensitive headers, but a redirect to "bar.com" will not. -// -// • when forwarding the "Cookie" header with a non-nil cookie Jar. -// Since each redirect may mutate the state of the cookie jar, -// a redirect may possibly alter a cookie set in the initial request. -// When forwarding the "Cookie" header, any mutated cookies will be omitted, -// with the expectation that the Jar will insert those mutated cookies -// with the updated values (assuming the origin matches). -// If Jar is nil, the initial cookies are forwarded without change. +// - when forwarding sensitive headers like "Authorization", +// "WWW-Authenticate", and "Cookie" to untrusted targets. +// These headers will be ignored when following a redirect to a domain +// that is not a subdomain match or exact match of the initial domain. +// For example, a redirect from "foo.com" to either "foo.com" or "sub.foo.com" +// will forward the sensitive headers, but a redirect to "bar.com" will not. +// - when forwarding the "Cookie" header with a non-nil cookie Jar. +// Since each redirect may mutate the state of the cookie jar, +// a redirect may possibly alter a cookie set in the initial request. +// When forwarding the "Cookie" header, any mutated cookies will be omitted, +// with the expectation that the Jar will insert those mutated cookies +// with the updated values (assuming the origin matches). +// If Jar is nil, the initial cookies are forwarded without change. type Client struct { // Transport specifies the mechanism by which individual // HTTP requests are made. From dc55a0ebd17dda70f18439f875b9ab5dcc273280 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Wed, 13 Sep 2023 12:02:38 -0400 Subject: [PATCH 32/93] net/http: ServeMux handles extended patterns Modify ServeMux to handle patterns with methods and wildcards. Remove the map and list of patterns. Instead patterns are registered and matched using a routing tree. We also reorganize the code around "trailing-slash redirection," the feature whereby a trailing slash is added to a path if it doesn't match an existing one. The existing code checked the map of paths twice, but searching the tree twice would be needlessly expensive. The rewrite searches the tree once, and then again only if a trailing-slash redirection is possible. There are a few omitted features in this CL, indicated with TODOs. Change-Id: Ifaef59f6c8c7b7131dc4a5d0f101cc22887bdc74 Reviewed-on: https://go-review.googlesource.com/c/go/+/528039 Run-TryBot: Jonathan Amsterdam TryBot-Result: Gopher Robot Reviewed-by: Damien Neil --- server.go | 333 +++++++++++++++++++++++++++---------------------- server_test.go | 86 ++++++++++++- 2 files changed, 268 insertions(+), 151 deletions(-) diff --git a/server.go b/server.go index 0d75b877..26df2384 100644 --- a/server.go +++ b/server.go @@ -23,7 +23,6 @@ import ( urlpkg "net/url" "path" "runtime" - "sort" "strconv" "strings" "sync" @@ -2281,6 +2280,9 @@ func RedirectHandler(url string, code int) Handler { return &redirectHandler{url, code} } +// TODO(jba): rewrite the following doc for enhanced patterns (proposal +// https://go.dev/issue/61410). + // ServeMux is an HTTP request multiplexer. // It matches the URL of each incoming request against a list of registered // patterns and calls the handler for the pattern that @@ -2317,19 +2319,15 @@ func RedirectHandler(url string, code int) Handler { // header, stripping the port number and redirecting any request containing . or // .. elements or repeated slashes to an equivalent, cleaner URL. type ServeMux struct { - mu sync.RWMutex - m map[string]muxEntry - es []muxEntry // slice of entries sorted from longest to shortest. - hosts bool // whether any patterns contain hostnames -} - -type muxEntry struct { - h Handler - pattern string + mu sync.RWMutex + tree routingNode + patterns []*pattern } // NewServeMux allocates and returns a new ServeMux. -func NewServeMux() *ServeMux { return new(ServeMux) } +func NewServeMux() *ServeMux { + return &ServeMux{} +} // DefaultServeMux is the default ServeMux used by Serve. var DefaultServeMux = &defaultServeMux @@ -2371,66 +2369,6 @@ func stripHostPort(h string) string { return host } -// Find a handler on a handler map given a path string. -// Most-specific (longest) pattern wins. -func (mux *ServeMux) match(path string) (h Handler, pattern string) { - // Check for exact match first. - v, ok := mux.m[path] - if ok { - return v.h, v.pattern - } - - // Check for longest valid match. mux.es contains all patterns - // that end in / sorted from longest to shortest. - for _, e := range mux.es { - if strings.HasPrefix(path, e.pattern) { - return e.h, e.pattern - } - } - return nil, "" -} - -// redirectToPathSlash determines if the given path needs appending "/" to it. -// This occurs when a handler for path + "/" was already registered, but -// not for path itself. If the path needs appending to, it creates a new -// URL, setting the path to u.Path + "/" and returning true to indicate so. -func (mux *ServeMux) redirectToPathSlash(host, path string, u *url.URL) (*url.URL, bool) { - mux.mu.RLock() - shouldRedirect := mux.shouldRedirectRLocked(host, path) - mux.mu.RUnlock() - if !shouldRedirect { - return u, false - } - path = path + "/" - u = &url.URL{Path: path, RawQuery: u.RawQuery} - return u, true -} - -// shouldRedirectRLocked reports whether the given path and host should be redirected to -// path+"/". This should happen if a handler is registered for path+"/" but -// not path -- see comments at ServeMux. -func (mux *ServeMux) shouldRedirectRLocked(host, path string) bool { - p := []string{path, host + path} - - for _, c := range p { - if _, exist := mux.m[c]; exist { - return false - } - } - - n := len(path) - if n == 0 { - return false - } - for _, c := range p { - if _, exist := mux.m[c+"/"]; exist { - return path[n-1] != '/' - } - } - - return false -} - // Handler returns the handler to use for the given request, // consulting r.Method, r.Host, and r.URL.Path. It always returns // a non-nil handler. If the path is not in its canonical form, the @@ -2442,61 +2380,144 @@ func (mux *ServeMux) shouldRedirectRLocked(host, path string) bool { // // Handler also returns the registered pattern that matches the // request or, in the case of internally-generated redirects, -// the pattern that will match after following the redirect. +// the path that will match after following the redirect. // // If there is no registered handler that applies to the request, // Handler returns a “page not found” handler and an empty pattern. func (mux *ServeMux) Handler(r *Request) (h Handler, pattern string) { + return mux.findHandler(r) +} + +// findHandler finds a handler for a request. +// If there is a matching handler, it returns it and the pattern that matched. +// Otherwise it returns a Redirect or NotFound handler with the path that would match +// after the redirect. +func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string) { + var n *routingNode + // TODO(jba): use escaped path. This is an independent change that is also part + // of proposal https://go.dev/issue/61410. + path := r.URL.Path // CONNECT requests are not canonicalized. if r.Method == "CONNECT" { // If r.URL.Path is /tree and its handler is not registered, // the /tree -> /tree/ redirect applies to CONNECT requests // but the path canonicalization does not. - if u, ok := mux.redirectToPathSlash(r.URL.Host, r.URL.Path, r.URL); ok { + _, _, u := mux.handler(r.URL.Host, r.Method, path, r.URL) + if u != nil { return RedirectHandler(u.String(), StatusMovedPermanently), u.Path } - - return mux.handler(r.Host, r.URL.Path) - } - - // All other requests have any port stripped and path cleaned - // before passing to mux.handler. - host := stripHostPort(r.Host) - path := cleanPath(r.URL.Path) - - // If the given path is /tree and its handler is not registered, - // redirect for /tree/. - if u, ok := mux.redirectToPathSlash(host, path, r.URL); ok { - return RedirectHandler(u.String(), StatusMovedPermanently), u.Path + // Redo the match, this time with r.Host instead of r.URL.Host. + // Pass a nil URL to skip the trailing-slash redirect logic. + n, _, _ = mux.handler(r.Host, r.Method, path, nil) + } else { + // All other requests have any port stripped and path cleaned + // before passing to mux.handler. + host := stripHostPort(r.Host) + path = cleanPath(path) + + // If the given path is /tree and its handler is not registered, + // redirect for /tree/. + var u *url.URL + n, _, u = mux.handler(host, r.Method, path, r.URL) + if u != nil { + return RedirectHandler(u.String(), StatusMovedPermanently), u.Path + } + if path != r.URL.Path { + // Redirect to cleaned path. + patStr := "" + if n != nil { + patStr = n.pattern.String() + } + u := &url.URL{Path: path, RawQuery: r.URL.RawQuery} + return RedirectHandler(u.String(), StatusMovedPermanently), patStr + } } - - if path != r.URL.Path { - _, pattern = mux.handler(host, path) - u := &url.URL{Path: path, RawQuery: r.URL.RawQuery} - return RedirectHandler(u.String(), StatusMovedPermanently), pattern + if n == nil { + // TODO(jba): support 405 (MethodNotAllowed) by checking for patterns with different methods. + return NotFoundHandler(), "" } - - return mux.handler(host, r.URL.Path) + return n.handler, n.pattern.String() } -// handler is the main implementation of Handler. +// handler looks up a node in the tree that matches the host, method and path. // The path is known to be in canonical form, except for CONNECT methods. -func (mux *ServeMux) handler(host, path string) (h Handler, pattern string) { + +// If the url argument is non-nil, handler also deals with trailing-slash +// redirection: when a path doesn't match exactly, the match is tried again + +// after appending "/" to the path. If that second match succeeds, the last +// return value is the URL to redirect to. +// +// TODO(jba): give this a better name. For now we're keeping the name of the closest +// corresponding function in the original code. +func (mux *ServeMux) handler(host, method, path string, u *url.URL) (_ *routingNode, matches []string, redirectTo *url.URL) { mux.mu.RLock() defer mux.mu.RUnlock() - // Host-specific pattern takes precedence over generic ones - if mux.hosts { - h, pattern = mux.match(host + path) + n, matches := mux.tree.match(host, method, path) + // If we have an exact match, or we were asked not to try trailing-slash redirection, + // then we're done. + if !exactMatch(n, path) && u != nil { + // If there is an exact match with a trailing slash, then redirect. + path += "/" + n2, _ := mux.tree.match(host, method, path) + if exactMatch(n2, path) { + return nil, nil, &url.URL{Path: path, RawQuery: u.RawQuery} + } + } + return n, matches, nil +} + +// exactMatch reports whether the node's pattern exactly matches the path. +// As a special case, if the node is nil, exactMatch return false. +// +// Before wildcards were introduced, it was clear that an exact match meant +// that the pattern and path were the same string. The only other possibility +// was that a trailing-slash pattern, like "/", matched a path longer than +// it, like "/a". +// +// With wildcards, we define an inexact match as any one where a multi wildcard +// matches a non-empty string. All other matches are exact. +// For example, these are all exact matches: +// +// pattern path +// /a /a +// /{x} /a +// /a/{$} /a/ +// /a/ /a/ +// +// The last case has a multi wildcard (implicitly), but the match is exact because +// the wildcard matches the empty string. +// +// Examples of matches that are not exact: +// +// pattern path +// / /a +// /a/{x...} /a/b +func exactMatch(n *routingNode, path string) bool { + if n == nil { + return false } - if h == nil { - h, pattern = mux.match(path) + // We can't directly implement the definition (empty match for multi + // wildcard) because we don't record a match for anonymous multis. + + // If there is no multi, the match is exact. + if !n.pattern.lastSegment().multi { + return true } - if h == nil { - h, pattern = NotFoundHandler(), "" + + // If the path doesn't end in a trailing slash, then the multi match + // is non-empty. + if len(path) > 0 && path[len(path)-1] != '/' { + return false } - return + // Only patterns ending in {$} or a multi wildcard can + // match a path with a trailing slash. + // For the match to be exact, the number of pattern + // segments should be the same as the number of slashes in the path. + // E.g. "/a/b/{$}" and "/a/b/{...}" exactly match "/a/b/", but "/a/" does not. + return len(n.pattern.segments) == strings.Count(path, "/") } // ServeHTTP dispatches the request to the handler whose @@ -2509,71 +2530,83 @@ func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { w.WriteHeader(StatusBadRequest) return } - h, _ := mux.Handler(r) + h, _ := mux.findHandler(r) + // TODO(jba); save matches in Request. h.ServeHTTP(w, r) } +// The four functions below all call register so that callerLocation +// always refers to user code. + // Handle registers the handler for the given pattern. // If a handler already exists for pattern, Handle panics. func (mux *ServeMux) Handle(pattern string, handler Handler) { - mux.mu.Lock() - defer mux.mu.Unlock() - - if pattern == "" { - panic("http: invalid pattern") - } - if handler == nil { - panic("http: nil handler") - } - if _, exist := mux.m[pattern]; exist { - panic("http: multiple registrations for " + pattern) - } - - if mux.m == nil { - mux.m = make(map[string]muxEntry) - } - e := muxEntry{h: handler, pattern: pattern} - mux.m[pattern] = e - if pattern[len(pattern)-1] == '/' { - mux.es = appendSorted(mux.es, e) - } - - if pattern[0] != '/' { - mux.hosts = true - } -} - -func appendSorted(es []muxEntry, e muxEntry) []muxEntry { - n := len(es) - i := sort.Search(n, func(i int) bool { - return len(es[i].pattern) < len(e.pattern) - }) - if i == n { - return append(es, e) - } - // we now know that i points at where we want to insert - es = append(es, muxEntry{}) // try to grow the slice in place, any entry works. - copy(es[i+1:], es[i:]) // Move shorter entries down - es[i] = e - return es + mux.register(pattern, handler) } // HandleFunc registers the handler function for the given pattern. func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { - if handler == nil { - panic("http: nil handler") - } - mux.Handle(pattern, HandlerFunc(handler)) + mux.register(pattern, HandlerFunc(handler)) } // Handle registers the handler for the given pattern in [DefaultServeMux]. // The documentation for [ServeMux] explains how patterns are matched. -func Handle(pattern string, handler Handler) { DefaultServeMux.Handle(pattern, handler) } +func Handle(pattern string, handler Handler) { + DefaultServeMux.register(pattern, handler) +} // HandleFunc registers the handler function for the given pattern in [DefaultServeMux]. // The documentation for [ServeMux] explains how patterns are matched. func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { - DefaultServeMux.HandleFunc(pattern, handler) + DefaultServeMux.register(pattern, HandlerFunc(handler)) +} + +func (mux *ServeMux) register(pattern string, handler Handler) { + if err := mux.registerErr(pattern, handler); err != nil { + panic(err) + } +} + +func (mux *ServeMux) registerErr(pattern string, handler Handler) error { + if pattern == "" { + return errors.New("http: invalid pattern") + } + if handler == nil { + return errors.New("http: nil handler") + } + if f, ok := handler.(HandlerFunc); ok && f == nil { + return errors.New("http: nil handler") + } + + pat, err := parsePattern(pattern) + if err != nil { + return err + } + + // Get the caller's location, for better conflict error messages. + // Skip register and whatever calls it. + _, file, line, ok := runtime.Caller(3) + if !ok { + pat.loc = "unknown location" + } else { + pat.loc = fmt.Sprintf("%s:%d", file, line) + } + + mux.mu.Lock() + defer mux.mu.Unlock() + // Check for conflict. + // This makes a quadratic number of calls to conflictsWith: we check + // each pattern against every other pattern. + // TODO(jba): add indexing to speed this up. + for _, pat2 := range mux.patterns { + if pat.conflictsWith(pat2) { + return fmt.Errorf("pattern %q (registered at %s) conflicts with pattern %q (registered at %s)", + pat, pat.loc, pat2, pat2.loc) + } + } + mux.tree.addPattern(pat, handler) + mux.patterns = append(mux.patterns, pat) + return nil } // Serve accepts incoming HTTP connections on the listener l, diff --git a/server_test.go b/server_test.go index d17c5c1e..0e7bdb2f 100644 --- a/server_test.go +++ b/server_test.go @@ -8,6 +8,7 @@ package http import ( "fmt" + "net/url" "testing" "time" ) @@ -64,6 +65,85 @@ func TestServerTLSHandshakeTimeout(t *testing.T) { } } +type handler struct{ i int } + +func (handler) ServeHTTP(ResponseWriter, *Request) {} + +func TestFindHandler(t *testing.T) { + mux := NewServeMux() + for _, ph := range []struct { + pat string + h Handler + }{ + {"/", &handler{1}}, + {"/foo/", &handler{2}}, + {"/foo", &handler{3}}, + {"/bar/", &handler{4}}, + {"//foo", &handler{5}}, + } { + mux.Handle(ph.pat, ph.h) + } + + for _, test := range []struct { + method string + path string + wantHandler string + }{ + {"GET", "/", "&http.handler{i:1}"}, + {"GET", "//", `&http.redirectHandler{url:"/", code:301}`}, + {"GET", "/foo/../bar/./..//baz", `&http.redirectHandler{url:"/baz", code:301}`}, + {"GET", "/foo", "&http.handler{i:3}"}, + {"GET", "/foo/x", "&http.handler{i:2}"}, + {"GET", "/bar/x", "&http.handler{i:4}"}, + {"GET", "/bar", `&http.redirectHandler{url:"/bar/", code:301}`}, + {"CONNECT", "/", "&http.handler{i:1}"}, + {"CONNECT", "//", "&http.handler{i:1}"}, + {"CONNECT", "//foo", "&http.handler{i:5}"}, + {"CONNECT", "/foo/../bar/./..//baz", "&http.handler{i:2}"}, + {"CONNECT", "/foo", "&http.handler{i:3}"}, + {"CONNECT", "/foo/x", "&http.handler{i:2}"}, + {"CONNECT", "/bar/x", "&http.handler{i:4}"}, + {"CONNECT", "/bar", `&http.redirectHandler{url:"/bar/", code:301}`}, + } { + var r Request + r.Method = test.method + r.Host = "example.com" + r.URL = &url.URL{Path: test.path} + gotH, _ := mux.findHandler(&r) + got := fmt.Sprintf("%#v", gotH) + if got != test.wantHandler { + t.Errorf("%s %q: got %q, want %q", test.method, test.path, got, test.wantHandler) + } + } +} + +func TestExactMatch(t *testing.T) { + for _, test := range []struct { + pattern string + path string + want bool + }{ + {"", "/a", false}, + {"/", "/a", false}, + {"/a", "/a", true}, + {"/a/{x...}", "/a/b", false}, + {"/a/{x}", "/a/b", true}, + {"/a/b/", "/a/b/", true}, + {"/a/b/{$}", "/a/b/", true}, + {"/a/", "/a/b/", false}, + } { + var n *routingNode + if test.pattern != "" { + pat := mustParsePattern(t, test.pattern) + n = &routingNode{pattern: pat} + } + got := exactMatch(n, test.path) + if got != test.want { + t.Errorf("%q, %s: got %t, want %t", test.pattern, test.path, got, test.want) + } + } +} + func BenchmarkServerMatch(b *testing.B) { fn := func(w ResponseWriter, r *Request) { fmt.Fprintf(w, "OK") @@ -90,7 +170,11 @@ func BenchmarkServerMatch(b *testing.B) { "/products/", "/products/3/image.jpg"} b.StartTimer() for i := 0; i < b.N; i++ { - if h, p := mux.match(paths[i%len(paths)]); h != nil && p == "" { + r, err := NewRequest("GET", "http://example.com/"+paths[i%len(paths)], nil) + if err != nil { + b.Fatal(err) + } + if h, p := mux.findHandler(r); h != nil && p == "" { b.Error("impossible") } } From cd9a6097d06278d211c76070eae85891c11d1a8e Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Mon, 11 Sep 2023 16:17:03 -0400 Subject: [PATCH 33/93] net/http: scale rstAvoidanceDelay to reduce test flakiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As far as I can tell, some flakiness is unavoidable in tests that race a large client request write against a server's response when the server doesn't read the full request. It does not appear to be possible to simultaneously ensure that well-behaved clients see EOF instead of ECONNRESET and also prevent misbehaving clients from consuming arbitrary server resources. (See RFC 7230 §6.6 for more detail.) Since there doesn't appear to be a way to cleanly eliminate this source of flakiness, we can instead work around it: we can allow the test to adjust the hard-coded delay if it sees a plausibly-related failure, so that the test can retry with a longer delay. As a nice side benefit, this also allows the tests to run more quickly in the typical case: since the test will retry in case of spurious failures, we can start with an aggressively short delay, and only back off to a longer one if it is really needed on the specific machine running the test. Fixes #57084. Fixes #51104. For #58398. Change-Id: Ia4050679f0777e5eeba7670307a77d93cfce856f Cq-Include-Trybots: luci.golang.try:gotip-linux-amd64-longtest-race,gotip-linux-amd64-race,gotip-windows-amd64-race Reviewed-on: https://go-review.googlesource.com/c/go/+/527196 LUCI-TryBot-Result: Go LUCI Reviewed-by: Damien Neil Auto-Submit: Bryan Mills --- export_test.go | 18 ++ serve_test.go | 492 +++++++++++++++++++++++++--------------------- server.go | 29 ++- transport_test.go | 169 +++++++++------- 4 files changed, 419 insertions(+), 289 deletions(-) diff --git a/export_test.go b/export_test.go index 5d198f3f..7e6d3d8e 100644 --- a/export_test.go +++ b/export_test.go @@ -315,3 +315,21 @@ func ResponseWriterConnForTesting(w ResponseWriter) (c net.Conn, ok bool) { } return nil, false } + +func init() { + // Set the default rstAvoidanceDelay to the minimum possible value to shake + // out tests that unexpectedly depend on it. Such tests should use + // runTimeSensitiveTest and SetRSTAvoidanceDelay to explicitly raise the delay + // if needed. + rstAvoidanceDelay = 1 * time.Nanosecond +} + +// SetRSTAvoidanceDelay sets how long we are willing to wait between calling +// CloseWrite on a connection and fully closing the connection. +func SetRSTAvoidanceDelay(t *testing.T, d time.Duration) { + prevDelay := rstAvoidanceDelay + t.Cleanup(func() { + rstAvoidanceDelay = prevDelay + }) + rstAvoidanceDelay = d +} diff --git a/serve_test.go b/serve_test.go index f26a6b31..9fe99a37 100644 --- a/serve_test.go +++ b/serve_test.go @@ -646,19 +646,15 @@ func benchmarkServeMux(b *testing.B, runHandler bool) { func TestServerTimeouts(t *testing.T) { run(t, testServerTimeouts, []testMode{http1Mode}) } func testServerTimeouts(t *testing.T, mode testMode) { - // Try three times, with increasing timeouts. - tries := []time.Duration{250 * time.Millisecond, 500 * time.Millisecond, 1 * time.Second} - for i, timeout := range tries { - err := testServerTimeoutsWithTimeout(t, timeout, mode) - if err == nil { - return - } - t.Logf("failed at %v: %v", timeout, err) - if i != len(tries)-1 { - t.Logf("retrying at %v ...", tries[i+1]) - } - } - t.Fatal("all attempts failed") + runTimeSensitiveTest(t, []time.Duration{ + 10 * time.Millisecond, + 50 * time.Millisecond, + 100 * time.Millisecond, + 500 * time.Millisecond, + 1 * time.Second, + }, func(t *testing.T, timeout time.Duration) error { + return testServerTimeoutsWithTimeout(t, timeout, mode) + }) } func testServerTimeoutsWithTimeout(t *testing.T, timeout time.Duration, mode testMode) error { @@ -3101,47 +3097,68 @@ func TestServerBufferedChunking(t *testing.T) { // closing the TCP connection, causing the client to get a RST. // See https://golang.org/issue/3595 func TestServerGracefulClose(t *testing.T) { - run(t, testServerGracefulClose, []testMode{http1Mode}) + // Not parallel: modifies the global rstAvoidanceDelay. + run(t, testServerGracefulClose, []testMode{http1Mode}, testNotParallel) } func testServerGracefulClose(t *testing.T, mode testMode) { - ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - Error(w, "bye", StatusUnauthorized) - })).ts + runTimeSensitiveTest(t, []time.Duration{ + 1 * time.Millisecond, + 5 * time.Millisecond, + 10 * time.Millisecond, + 50 * time.Millisecond, + 100 * time.Millisecond, + 500 * time.Millisecond, + time.Second, + 5 * time.Second, + }, func(t *testing.T, timeout time.Duration) error { + SetRSTAvoidanceDelay(t, timeout) + t.Logf("set RST avoidance delay to %v", timeout) - conn, err := net.Dial("tcp", ts.Listener.Addr().String()) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - const bodySize = 5 << 20 - req := []byte(fmt.Sprintf("POST / HTTP/1.1\r\nHost: foo.com\r\nContent-Length: %d\r\n\r\n", bodySize)) - for i := 0; i < bodySize; i++ { - req = append(req, 'x') - } - writeErr := make(chan error) - go func() { - _, err := conn.Write(req) - writeErr <- err - }() - br := bufio.NewReader(conn) - lineNum := 0 - for { - line, err := br.ReadString('\n') - if err == io.EOF { - break + const bodySize = 5 << 20 + req := []byte(fmt.Sprintf("POST / HTTP/1.1\r\nHost: foo.com\r\nContent-Length: %d\r\n\r\n", bodySize)) + for i := 0; i < bodySize; i++ { + req = append(req, 'x') } + + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + Error(w, "bye", StatusUnauthorized) + })) + // We need to close cst explicitly here so that in-flight server + // requests don't race with the call to SetRSTAvoidanceDelay for a retry. + defer cst.close() + ts := cst.ts + + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) if err != nil { - t.Fatalf("ReadLine: %v", err) + return err } - lineNum++ - if lineNum == 1 && !strings.Contains(line, "401 Unauthorized") { - t.Errorf("Response line = %q; want a 401", line) + defer conn.Close() + writeErr := make(chan error) + go func() { + _, err := conn.Write(req) + writeErr <- err + }() + br := bufio.NewReader(conn) + lineNum := 0 + for { + line, err := br.ReadString('\n') + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("ReadLine: %v", err) + } + lineNum++ + if lineNum == 1 && !strings.Contains(line, "401 Unauthorized") { + t.Errorf("Response line = %q; want a 401", line) + } } - } - // Wait for write to finish. This is a broken pipe on both - // Darwin and Linux, but checking this isn't the point of - // the test. - <-writeErr + // Wait for write to finish. This is a broken pipe on both + // Darwin and Linux, but checking this isn't the point of + // the test. + <-writeErr + return nil + }) } func TestCaseSensitiveMethod(t *testing.T) { run(t, testCaseSensitiveMethod) } @@ -3923,91 +3940,78 @@ func TestContentTypeOkayOn204(t *testing.T) { // and the http client), and both think they can close it on failure. // Therefore, all incoming server requests Bodies need to be thread-safe. func TestTransportAndServerSharedBodyRace(t *testing.T) { - run(t, testTransportAndServerSharedBodyRace) + run(t, testTransportAndServerSharedBodyRace, testNotParallel) } func testTransportAndServerSharedBodyRace(t *testing.T, mode testMode) { - const bodySize = 1 << 20 - - // errorf is like t.Errorf, but also writes to println. When - // this test fails, it hangs. This helps debugging and I've - // added this enough times "temporarily". It now gets added - // full time. - errorf := func(format string, args ...any) { - v := fmt.Sprintf(format, args...) - println(v) - t.Error(v) - } - - unblockBackend := make(chan bool) - backend := newClientServerTest(t, mode, HandlerFunc(func(rw ResponseWriter, req *Request) { - gone := rw.(CloseNotifier).CloseNotify() - didCopy := make(chan any) - go func() { + // The proxy server in the middle of the stack for this test potentially + // from its handler after only reading half of the body. + // That can trigger https://go.dev/issue/3595, which is otherwise + // irrelevant to this test. + runTimeSensitiveTest(t, []time.Duration{ + 1 * time.Millisecond, + 5 * time.Millisecond, + 10 * time.Millisecond, + 50 * time.Millisecond, + 100 * time.Millisecond, + 500 * time.Millisecond, + time.Second, + 5 * time.Second, + }, func(t *testing.T, timeout time.Duration) error { + SetRSTAvoidanceDelay(t, timeout) + t.Logf("set RST avoidance delay to %v", timeout) + + const bodySize = 1 << 20 + + backend := newClientServerTest(t, mode, HandlerFunc(func(rw ResponseWriter, req *Request) { n, err := io.CopyN(rw, req.Body, bodySize) - didCopy <- []any{n, err} - }() - isGone := false - Loop: - for { - select { - case <-didCopy: - break Loop - case <-gone: - isGone = true - case <-time.After(time.Second): - println("1 second passes in backend, proxygone=", isGone) + t.Logf("backend CopyN: %v, %v", n, err) + <-req.Context().Done() + })) + // We need to close explicitly here so that in-flight server + // requests don't race with the call to SetRSTAvoidanceDelay for a retry. + defer backend.close() + + var proxy *clientServerTest + proxy = newClientServerTest(t, mode, HandlerFunc(func(rw ResponseWriter, req *Request) { + req2, _ := NewRequest("POST", backend.ts.URL, req.Body) + req2.ContentLength = bodySize + cancel := make(chan struct{}) + req2.Cancel = cancel + + bresp, err := proxy.c.Do(req2) + if err != nil { + t.Errorf("Proxy outbound request: %v", err) + return } - } - <-unblockBackend - })) - defer backend.close() - - backendRespc := make(chan *Response, 1) - var proxy *clientServerTest - proxy = newClientServerTest(t, mode, HandlerFunc(func(rw ResponseWriter, req *Request) { - req2, _ := NewRequest("POST", backend.ts.URL, req.Body) - req2.ContentLength = bodySize - cancel := make(chan struct{}) - req2.Cancel = cancel + _, err = io.CopyN(io.Discard, bresp.Body, bodySize/2) + if err != nil { + t.Errorf("Proxy copy error: %v", err) + return + } + t.Cleanup(func() { bresp.Body.Close() }) + + // Try to cause a race. Canceling the client request will cause the client + // transport to close req2.Body. Returning from the server handler will + // cause the server to close req.Body. Since they are the same underlying + // ReadCloser, that will result in concurrent calls to Close (and possibly a + // Read concurrent with a Close). + if mode == http2Mode { + close(cancel) + } else { + proxy.c.Transport.(*Transport).CancelRequest(req2) + } + rw.Write([]byte("OK")) + })) + defer proxy.close() - bresp, err := proxy.c.Do(req2) - if err != nil { - errorf("Proxy outbound request: %v", err) - return - } - _, err = io.CopyN(io.Discard, bresp.Body, bodySize/2) + req, _ := NewRequest("POST", proxy.ts.URL, io.LimitReader(neverEnding('a'), bodySize)) + res, err := proxy.c.Do(req) if err != nil { - errorf("Proxy copy error: %v", err) - return - } - backendRespc <- bresp // to close later - - // Try to cause a race: Both the Transport and the proxy handler's Server - // will try to read/close req.Body (aka req2.Body) - if mode == http2Mode { - close(cancel) - } else { - proxy.c.Transport.(*Transport).CancelRequest(req2) + return fmt.Errorf("original request: %v", err) } - rw.Write([]byte("OK")) - })) - defer proxy.close() - - defer close(unblockBackend) - req, _ := NewRequest("POST", proxy.ts.URL, io.LimitReader(neverEnding('a'), bodySize)) - res, err := proxy.c.Do(req) - if err != nil { - t.Fatalf("Original request: %v", err) - } - - // Cleanup, so we don't leak goroutines. - res.Body.Close() - select { - case res := <-backendRespc: res.Body.Close() - default: - // We failed earlier. (e.g. on proxy.c.Do(req2)) - } + return nil + }) } // Test that a hanging Request.Body.Read from another goroutine can't @@ -4342,7 +4346,8 @@ func (c *closeWriteTestConn) CloseWrite() error { } func TestCloseWrite(t *testing.T) { - setParallel(t) + SetRSTAvoidanceDelay(t, 1*time.Millisecond) + var srv Server var testConn closeWriteTestConn c := ExportServerNewConn(&srv, &testConn) @@ -5382,49 +5387,73 @@ func testServerIdleTimeout(t *testing.T, mode testMode) { if testing.Short() { t.Skip("skipping in short mode") } - ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - io.Copy(io.Discard, r.Body) - io.WriteString(w, r.RemoteAddr) - }), func(ts *httptest.Server) { - ts.Config.ReadHeaderTimeout = 1 * time.Second - ts.Config.IdleTimeout = 2 * time.Second - }).ts - c := ts.Client() + runTimeSensitiveTest(t, []time.Duration{ + 10 * time.Millisecond, + 100 * time.Millisecond, + 1 * time.Second, + 10 * time.Second, + }, func(t *testing.T, readHeaderTimeout time.Duration) error { + ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + io.Copy(io.Discard, r.Body) + io.WriteString(w, r.RemoteAddr) + }), func(ts *httptest.Server) { + ts.Config.ReadHeaderTimeout = readHeaderTimeout + ts.Config.IdleTimeout = 2 * readHeaderTimeout + }).ts + t.Logf("ReadHeaderTimeout = %v", ts.Config.ReadHeaderTimeout) + t.Logf("IdleTimeout = %v", ts.Config.IdleTimeout) + c := ts.Client() - get := func() string { - res, err := c.Get(ts.URL) + get := func() (string, error) { + res, err := c.Get(ts.URL) + if err != nil { + return "", err + } + defer res.Body.Close() + slurp, err := io.ReadAll(res.Body) + if err != nil { + // If we're at this point the headers have definitely already been + // read and the server is not idle, so neither timeout applies: + // this should never fail. + t.Fatal(err) + } + return string(slurp), nil + } + + a1, err := get() if err != nil { - t.Fatal(err) + return err } - defer res.Body.Close() - slurp, err := io.ReadAll(res.Body) + a2, err := get() if err != nil { - t.Fatal(err) + return err + } + if a1 != a2 { + return fmt.Errorf("did requests on different connections") + } + time.Sleep(ts.Config.IdleTimeout * 3 / 2) + a3, err := get() + if err != nil { + return err + } + if a2 == a3 { + return fmt.Errorf("request three unexpectedly on same connection") } - return string(slurp) - } - a1, a2 := get(), get() - if a1 != a2 { - t.Fatalf("did requests on different connections") - } - time.Sleep(3 * time.Second) - a3 := get() - if a2 == a3 { - t.Fatal("request three unexpectedly on same connection") - } + // And test that ReadHeaderTimeout still works: + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) + if err != nil { + return err + } + defer conn.Close() + conn.Write([]byte("GET / HTTP/1.1\r\nHost: foo.com\r\n")) + time.Sleep(ts.Config.ReadHeaderTimeout * 2) + if _, err := io.CopyN(io.Discard, conn, 1); err == nil { + return fmt.Errorf("copy byte succeeded; want err") + } - // And test that ReadHeaderTimeout still works: - conn, err := net.Dial("tcp", ts.Listener.Addr().String()) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - conn.Write([]byte("GET / HTTP/1.1\r\nHost: foo.com\r\n")) - time.Sleep(2 * time.Second) - if _, err := io.CopyN(io.Discard, conn, 1); err == nil { - t.Fatal("copy byte succeeded; want err") - } + return nil + }) } func get(t *testing.T, c *Client, url string) string { @@ -5773,9 +5802,10 @@ func runTimeSensitiveTest(t *testing.T, durations []time.Duration, test func(t * if err == nil { return } - if i == len(durations)-1 { + if i == len(durations)-1 || t.Failed() { t.Fatalf("failed with duration %v: %v", d, err) } + t.Logf("retrying after error with duration %v: %v", d, err) } } @@ -6620,7 +6650,7 @@ func testQuerySemicolon(t *testing.T, mode testMode, query string, wantX string, } func TestMaxBytesHandler(t *testing.T) { - setParallel(t) + // Not parallel: modifies the global rstAvoidanceDelay. defer afterTest(t) for _, maxSize := range []int64{100, 1_000, 1_000_000} { @@ -6629,77 +6659,99 @@ func TestMaxBytesHandler(t *testing.T) { func(t *testing.T) { run(t, func(t *testing.T, mode testMode) { testMaxBytesHandler(t, mode, maxSize, requestSize) - }) + }, testNotParallel) }) } } } func testMaxBytesHandler(t *testing.T, mode testMode, maxSize, requestSize int64) { - var ( - handlerN int64 - handlerErr error - ) - echo := HandlerFunc(func(w ResponseWriter, r *Request) { - var buf bytes.Buffer - handlerN, handlerErr = io.Copy(&buf, r.Body) - io.Copy(w, &buf) - }) - - ts := newClientServerTest(t, mode, MaxBytesHandler(echo, maxSize)).ts - defer ts.Close() + runTimeSensitiveTest(t, []time.Duration{ + 1 * time.Millisecond, + 5 * time.Millisecond, + 10 * time.Millisecond, + 50 * time.Millisecond, + 100 * time.Millisecond, + 500 * time.Millisecond, + time.Second, + 5 * time.Second, + }, func(t *testing.T, timeout time.Duration) error { + SetRSTAvoidanceDelay(t, timeout) + t.Logf("set RST avoidance delay to %v", timeout) + + var ( + handlerN int64 + handlerErr error + ) + echo := HandlerFunc(func(w ResponseWriter, r *Request) { + var buf bytes.Buffer + handlerN, handlerErr = io.Copy(&buf, r.Body) + io.Copy(w, &buf) + }) - c := ts.Client() + cst := newClientServerTest(t, mode, MaxBytesHandler(echo, maxSize)) + // We need to close cst explicitly here so that in-flight server + // requests don't race with the call to SetRSTAvoidanceDelay for a retry. + defer cst.close() + ts := cst.ts + c := ts.Client() - body := strings.Repeat("a", int(requestSize)) - var wg sync.WaitGroup - defer wg.Wait() - getBody := func() (io.ReadCloser, error) { - wg.Add(1) - body := &wgReadCloser{ - Reader: strings.NewReader(body), - wg: &wg, + body := strings.Repeat("a", int(requestSize)) + var wg sync.WaitGroup + defer wg.Wait() + getBody := func() (io.ReadCloser, error) { + wg.Add(1) + body := &wgReadCloser{ + Reader: strings.NewReader(body), + wg: &wg, + } + return body, nil } - return body, nil - } - reqBody, _ := getBody() - req, err := NewRequest("POST", ts.URL, reqBody) - if err != nil { - reqBody.Close() - t.Fatal(err) - } - req.ContentLength = int64(len(body)) - req.GetBody = getBody - req.Header.Set("Content-Type", "text/plain") + reqBody, _ := getBody() + req, err := NewRequest("POST", ts.URL, reqBody) + if err != nil { + reqBody.Close() + t.Fatal(err) + } + req.ContentLength = int64(len(body)) + req.GetBody = getBody + req.Header.Set("Content-Type", "text/plain") - var buf strings.Builder - res, err := c.Do(req) - if err != nil { - t.Errorf("unexpected connection error: %v", err) - } else { - _, err = io.Copy(&buf, res.Body) - res.Body.Close() + var buf strings.Builder + res, err := c.Do(req) if err != nil { - t.Errorf("unexpected read error: %v", err) + return fmt.Errorf("unexpected connection error: %v", err) + } else { + _, err = io.Copy(&buf, res.Body) + res.Body.Close() + if err != nil { + return fmt.Errorf("unexpected read error: %v", err) + } } - } - if handlerN > maxSize { - t.Errorf("expected max request body %d; got %d", maxSize, handlerN) - } - if requestSize > maxSize && handlerErr == nil { - t.Error("expected error on handler side; got nil") - } - if requestSize <= maxSize { - if handlerErr != nil { - t.Errorf("%d expected nil error on handler side; got %v", requestSize, handlerErr) + // We don't expect any of the errors after this point to occur due + // to rstAvoidanceDelay being too short, so we use t.Errorf for those + // instead of returning a (retriable) error. + + if handlerN > maxSize { + t.Errorf("expected max request body %d; got %d", maxSize, handlerN) } - if handlerN != requestSize { - t.Errorf("expected request of size %d; got %d", requestSize, handlerN) + if requestSize > maxSize && handlerErr == nil { + t.Error("expected error on handler side; got nil") } - } - if buf.Len() != int(handlerN) { - t.Errorf("expected echo of size %d; got %d", handlerN, buf.Len()) - } + if requestSize <= maxSize { + if handlerErr != nil { + t.Errorf("%d expected nil error on handler side; got %v", requestSize, handlerErr) + } + if handlerN != requestSize { + t.Errorf("expected request of size %d; got %d", requestSize, handlerN) + } + } + if buf.Len() != int(handlerN) { + t.Errorf("expected echo of size %d; got %d", handlerN, buf.Len()) + } + + return nil + }) } func TestEarlyHints(t *testing.T) { diff --git a/server.go b/server.go index 26df2384..6fe917e0 100644 --- a/server.go +++ b/server.go @@ -1750,8 +1750,12 @@ func (c *conn) close() { // and processes its final data before they process the subsequent RST // from closing a connection with known unread data. // This RST seems to occur mostly on BSD systems. (And Windows?) -// This timeout is somewhat arbitrary (~latency around the planet). -const rstAvoidanceDelay = 500 * time.Millisecond +// This timeout is somewhat arbitrary (~latency around the planet), +// and may be modified by tests. +// +// TODO(bcmills): This should arguably be a server configuration parameter, +// not a hard-coded value. +var rstAvoidanceDelay = 500 * time.Millisecond type closeWriter interface { CloseWrite() error @@ -1770,6 +1774,27 @@ func (c *conn) closeWriteAndWait() { if tcp, ok := c.rwc.(closeWriter); ok { tcp.CloseWrite() } + + // When we return from closeWriteAndWait, the caller will fully close the + // connection. If client is still writing to the connection, this will cause + // the write to fail with ECONNRESET or similar. Unfortunately, many TCP + // implementations will also drop unread packets from the client's read buffer + // when a write fails, causing our final response to be truncated away too. + // + // As a result, https://www.rfc-editor.org/rfc/rfc7230#section-6.6 recommends + // that “[t]he server … continues to read from the connection until it + // receives a corresponding close by the client, or until the server is + // reasonably certain that its own TCP stack has received the client's + // acknowledgement of the packet(s) containing the server's last response.” + // + // Unfortunately, we have no straightforward way to be “reasonably certain” + // that we have received the client's ACK, and at any rate we don't want to + // allow a misbehaving client to soak up server connections indefinitely by + // withholding an ACK, nor do we want to go through the complexity or overhead + // of using low-level APIs to figure out when a TCP round-trip has completed. + // + // Instead, we declare that we are “reasonably certain” that we received the + // ACK if maxRSTAvoidanceDelay has elapsed. time.Sleep(rstAvoidanceDelay) } diff --git a/transport_test.go b/transport_test.go index 9f086172..8c09de70 100644 --- a/transport_test.go +++ b/transport_test.go @@ -2099,25 +2099,50 @@ func testIssue3644(t *testing.T, mode testMode) { // Test that a client receives a server's reply, even if the server doesn't read // the entire request body. -func TestIssue3595(t *testing.T) { run(t, testIssue3595) } +func TestIssue3595(t *testing.T) { + // Not parallel: modifies the global rstAvoidanceDelay. + run(t, testIssue3595, testNotParallel) +} func testIssue3595(t *testing.T, mode testMode) { - const deniedMsg = "sorry, denied." - ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - Error(w, deniedMsg, StatusUnauthorized) - })).ts - c := ts.Client() - res, err := c.Post(ts.URL, "application/octet-stream", neverEnding('a')) - if err != nil { - t.Errorf("Post: %v", err) - return - } - got, err := io.ReadAll(res.Body) - if err != nil { - t.Fatalf("Body ReadAll: %v", err) - } - if !strings.Contains(string(got), deniedMsg) { - t.Errorf("Known bug: response %q does not contain %q", got, deniedMsg) - } + runTimeSensitiveTest(t, []time.Duration{ + 1 * time.Millisecond, + 5 * time.Millisecond, + 10 * time.Millisecond, + 50 * time.Millisecond, + 100 * time.Millisecond, + 500 * time.Millisecond, + time.Second, + 5 * time.Second, + }, func(t *testing.T, timeout time.Duration) error { + SetRSTAvoidanceDelay(t, timeout) + t.Logf("set RST avoidance delay to %v", timeout) + + const deniedMsg = "sorry, denied." + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + Error(w, deniedMsg, StatusUnauthorized) + })) + // We need to close cst explicitly here so that in-flight server + // requests don't race with the call to SetRSTAvoidanceDelay for a retry. + defer cst.close() + ts := cst.ts + c := ts.Client() + + res, err := c.Post(ts.URL, "application/octet-stream", neverEnding('a')) + if err != nil { + return fmt.Errorf("Post: %v", err) + } + got, err := io.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("Body ReadAll: %v", err) + } + t.Logf("server response:\n%s", got) + if !strings.Contains(string(got), deniedMsg) { + // If we got an RST packet too early, we should have seen an error + // from io.ReadAll, not a silently-truncated body. + t.Errorf("Known bug: response %q does not contain %q", got, deniedMsg) + } + return nil + }) } // From https://golang.org/issue/4454 , @@ -4327,68 +4352,78 @@ func (c *wgReadCloser) Close() error { // Issue 11745. func TestTransportPrefersResponseOverWriteError(t *testing.T) { - run(t, testTransportPrefersResponseOverWriteError) + // Not parallel: modifies the global rstAvoidanceDelay. + run(t, testTransportPrefersResponseOverWriteError, testNotParallel) } func testTransportPrefersResponseOverWriteError(t *testing.T, mode testMode) { if testing.Short() { t.Skip("skipping in short mode") } - const contentLengthLimit = 1024 * 1024 // 1MB - ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { - if r.ContentLength >= contentLengthLimit { - w.WriteHeader(StatusBadRequest) - r.Body.Close() - return - } - w.WriteHeader(StatusOK) - })).ts - c := ts.Client() - fail := 0 - count := 100 + runTimeSensitiveTest(t, []time.Duration{ + 1 * time.Millisecond, + 5 * time.Millisecond, + 10 * time.Millisecond, + 50 * time.Millisecond, + 100 * time.Millisecond, + 500 * time.Millisecond, + time.Second, + 5 * time.Second, + }, func(t *testing.T, timeout time.Duration) error { + SetRSTAvoidanceDelay(t, timeout) + t.Logf("set RST avoidance delay to %v", timeout) + + const contentLengthLimit = 1024 * 1024 // 1MB + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + if r.ContentLength >= contentLengthLimit { + w.WriteHeader(StatusBadRequest) + r.Body.Close() + return + } + w.WriteHeader(StatusOK) + })) + // We need to close cst explicitly here so that in-flight server + // requests don't race with the call to SetRSTAvoidanceDelay for a retry. + defer cst.close() + ts := cst.ts + c := ts.Client() - bigBody := strings.Repeat("a", contentLengthLimit*2) - var wg sync.WaitGroup - defer wg.Wait() - getBody := func() (io.ReadCloser, error) { - wg.Add(1) - body := &wgReadCloser{ - Reader: strings.NewReader(bigBody), - wg: &wg, - } - return body, nil - } + count := 100 - for i := 0; i < count; i++ { - reqBody, _ := getBody() - req, err := NewRequest("PUT", ts.URL, reqBody) - if err != nil { - reqBody.Close() - t.Fatal(err) + bigBody := strings.Repeat("a", contentLengthLimit*2) + var wg sync.WaitGroup + defer wg.Wait() + getBody := func() (io.ReadCloser, error) { + wg.Add(1) + body := &wgReadCloser{ + Reader: strings.NewReader(bigBody), + wg: &wg, + } + return body, nil } - req.ContentLength = int64(len(bigBody)) - req.GetBody = getBody - resp, err := c.Do(req) - if err != nil { - fail++ - t.Logf("%d = %#v", i, err) - if ue, ok := err.(*url.Error); ok { - t.Logf("urlErr = %#v", ue.Err) - if ne, ok := ue.Err.(*net.OpError); ok { - t.Logf("netOpError = %#v", ne.Err) - } + for i := 0; i < count; i++ { + reqBody, _ := getBody() + req, err := NewRequest("PUT", ts.URL, reqBody) + if err != nil { + reqBody.Close() + t.Fatal(err) } - } else { - resp.Body.Close() - if resp.StatusCode != 400 { - t.Errorf("Expected status code 400, got %v", resp.Status) + req.ContentLength = int64(len(bigBody)) + req.GetBody = getBody + + resp, err := c.Do(req) + if err != nil { + return fmt.Errorf("Do %d: %v", i, err) + } else { + resp.Body.Close() + if resp.StatusCode != 400 { + t.Errorf("Expected status code 400, got %v", resp.Status) + } } } - } - if fail > 0 { - t.Errorf("Failed %v out of %v\n", fail, count) - } + return nil + }) } func TestTransportAutomaticHTTP2(t *testing.T) { From 94c8fbe79628299730549c15ba5f6406d87e00b8 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Wed, 13 Sep 2023 15:58:25 -0400 Subject: [PATCH 34/93] net/http: add test for registration errors Change-Id: Ice378e2f1c4cce180f020683d25070c5ae1edbad Reviewed-on: https://go-review.googlesource.com/c/go/+/528255 Run-TryBot: Jonathan Amsterdam Reviewed-by: Damien Neil TryBot-Result: Gopher Robot --- server.go | 2 +- server_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/server.go b/server.go index 6fe917e0..7ce078ce 100644 --- a/server.go +++ b/server.go @@ -2605,7 +2605,7 @@ func (mux *ServeMux) registerErr(pattern string, handler Handler) error { pat, err := parsePattern(pattern) if err != nil { - return err + return fmt.Errorf("parsing %q: %w", pattern, err) } // Get the caller's location, for better conflict error messages. diff --git a/server_test.go b/server_test.go index 0e7bdb2f..b0cc093d 100644 --- a/server_test.go +++ b/server_test.go @@ -9,6 +9,7 @@ package http import ( "fmt" "net/url" + "regexp" "testing" "time" ) @@ -117,6 +118,35 @@ func TestFindHandler(t *testing.T) { } } +func TestRegisterErr(t *testing.T) { + mux := NewServeMux() + h := &handler{} + mux.Handle("/a", h) + + for _, test := range []struct { + pattern string + handler Handler + wantRegexp string + }{ + {"", h, "invalid pattern"}, + {"/", nil, "nil handler"}, + {"/", HandlerFunc(nil), "nil handler"}, + {"/{x", h, `parsing "/\{x": bad wildcard segment`}, + {"/a", h, `conflicts with pattern.* \(registered at .*/server_test.go:\d+`}, + } { + t.Run(fmt.Sprintf("%s:%#v", test.pattern, test.handler), func(t *testing.T) { + err := mux.registerErr(test.pattern, test.handler) + if err == nil { + t.Fatal("got nil error") + } + re := regexp.MustCompile(test.wantRegexp) + if g := err.Error(); !re.MatchString(g) { + t.Errorf("\ngot %q\nwant string matching %q", g, test.wantRegexp) + } + }) + } +} + func TestExactMatch(t *testing.T) { for _, test := range []struct { pattern string From 90debc44926be8021fcb36688cf473f7492226ec Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Wed, 13 Sep 2023 16:00:45 -0400 Subject: [PATCH 35/93] net/http: give ServeMux.handler a better name Change-Id: I27bb7d9d5f172a84aa31304194b8a13036b9c5d1 Reviewed-on: https://go-review.googlesource.com/c/go/+/528275 Run-TryBot: Jonathan Amsterdam Reviewed-by: Damien Neil TryBot-Result: Gopher Robot --- server.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/server.go b/server.go index 7ce078ce..74362a69 100644 --- a/server.go +++ b/server.go @@ -2428,13 +2428,13 @@ func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string) { // If r.URL.Path is /tree and its handler is not registered, // the /tree -> /tree/ redirect applies to CONNECT requests // but the path canonicalization does not. - _, _, u := mux.handler(r.URL.Host, r.Method, path, r.URL) + _, _, u := mux.matchOrRedirect(r.URL.Host, r.Method, path, r.URL) if u != nil { return RedirectHandler(u.String(), StatusMovedPermanently), u.Path } // Redo the match, this time with r.Host instead of r.URL.Host. // Pass a nil URL to skip the trailing-slash redirect logic. - n, _, _ = mux.handler(r.Host, r.Method, path, nil) + n, _, _ = mux.matchOrRedirect(r.Host, r.Method, path, nil) } else { // All other requests have any port stripped and path cleaned // before passing to mux.handler. @@ -2444,7 +2444,7 @@ func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string) { // If the given path is /tree and its handler is not registered, // redirect for /tree/. var u *url.URL - n, _, u = mux.handler(host, r.Method, path, r.URL) + n, _, u = mux.matchOrRedirect(host, r.Method, path, r.URL) if u != nil { return RedirectHandler(u.String(), StatusMovedPermanently), u.Path } @@ -2465,18 +2465,14 @@ func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string) { return n.handler, n.pattern.String() } -// handler looks up a node in the tree that matches the host, method and path. +// matchOrRedirect looks up a node in the tree that matches the host, method and path. // The path is known to be in canonical form, except for CONNECT methods. // If the url argument is non-nil, handler also deals with trailing-slash // redirection: when a path doesn't match exactly, the match is tried again - // after appending "/" to the path. If that second match succeeds, the last // return value is the URL to redirect to. -// -// TODO(jba): give this a better name. For now we're keeping the name of the closest -// corresponding function in the original code. -func (mux *ServeMux) handler(host, method, path string, u *url.URL) (_ *routingNode, matches []string, redirectTo *url.URL) { +func (mux *ServeMux) matchOrRedirect(host, method, path string, u *url.URL) (_ *routingNode, matches []string, redirectTo *url.URL) { mux.mu.RLock() defer mux.mu.RUnlock() From 6790ea86f4b0a3037724e4a396e6ab6bfcf055d9 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Wed, 13 Sep 2023 16:58:24 -0400 Subject: [PATCH 36/93] net/http: implement path value methods on Request Add Request.PathValue and Request.SetPathValue, and the fields on Request required to support them. Populate those fields in ServeMux.ServeHTTP. Updates #61410. Change-Id: Ic88cb865b0d865a30d3b35ece8e0382c58ef67d1 Reviewed-on: https://go-review.googlesource.com/c/go/+/528355 Run-TryBot: Jonathan Amsterdam Reviewed-by: Damien Neil TryBot-Result: Gopher Robot --- request.go | 47 ++++++++++++++++++++++++++ request_test.go | 90 +++++++++++++++++++++++++++++++++++++++++++++++++ server.go | 24 +++++++------ server_test.go | 4 +-- 4 files changed, 152 insertions(+), 13 deletions(-) diff --git a/request.go b/request.go index 12039c9a..b66e6853 100644 --- a/request.go +++ b/request.go @@ -329,6 +329,11 @@ type Request struct { // It is unexported to prevent people from using Context wrong // and mutating the contexts held by callers of the same request. ctx context.Context + + // The following fields are for requests matched by ServeMux. + pat *pattern // the pattern that matched + matches []string // values for the matching wildcards in pat + otherValues map[string]string // for calls to SetPathValue that don't match a wildcard } // Context returns the request's context. To change the context, use @@ -1415,6 +1420,48 @@ func (r *Request) FormFile(key string) (multipart.File, *multipart.FileHeader, e return nil, nil, ErrMissingFile } +// PathValue returns the value for the named path wildcard in the ServeMux pattern +// that matched the request. +// It returns the empty string if the request was not matched against a pattern +// or there is no such wildcard in the pattern. +func (r *Request) PathValue(name string) string { + if i := r.patIndex(name); i >= 0 { + return r.matches[i] + } + return r.otherValues[name] +} + +func (r *Request) SetPathValue(name, value string) { + if i := r.patIndex(name); i >= 0 { + r.matches[i] = value + } else { + if r.otherValues == nil { + r.otherValues = map[string]string{} + } + r.otherValues[name] = value + } +} + +// patIndex returns the index of name in the list of named wildcards of the +// request's pattern, or -1 if there is no such name. +func (r *Request) patIndex(name string) int { + // The linear search seems expensive compared to a map, but just creating the map + // takes a lot of time, and most patterns will just have a couple of wildcards. + if r.pat == nil { + return -1 + } + i := 0 + for _, seg := range r.pat.segments { + if seg.wild && seg.s != "" { + if name == seg.s { + return i + } + i++ + } + } + return -1 +} + func (r *Request) expectsContinue() bool { return hasToken(r.Header.get("Expect"), "100-continue") } diff --git a/request_test.go b/request_test.go index 57111648..1aeb93fe 100644 --- a/request_test.go +++ b/request_test.go @@ -16,6 +16,7 @@ import ( "math" "mime/multipart" . "net/http" + "net/http/httptest" "net/url" "os" "reflect" @@ -1414,3 +1415,92 @@ func TestErrNotSupported(t *testing.T) { t.Error("errors.Is(ErrNotSupported, errors.ErrUnsupported) failed") } } + +func TestPathValueNoMatch(t *testing.T) { + // Check that PathValue and SetPathValue work on a Request that was never matched. + var r Request + if g, w := r.PathValue("x"), ""; g != w { + t.Errorf("got %q, want %q", g, w) + } + r.SetPathValue("x", "a") + if g, w := r.PathValue("x"), "a"; g != w { + t.Errorf("got %q, want %q", g, w) + } +} + +func TestPathValue(t *testing.T) { + for _, test := range []struct { + pattern string + url string + want map[string]string + }{ + { + "/{a}/is/{b}/{c...}", + "/now/is/the/time/for/all", + map[string]string{ + "a": "now", + "b": "the", + "c": "time/for/all", + "d": "", + }, + }, + // TODO(jba): uncomment these tests when we implement path escaping (forthcoming). + // { + // "/names/{name}/{other...}", + // "/names/" + url.PathEscape("/john") + "/address", + // map[string]string{ + // "name": "/john", + // "other": "address", + // }, + // }, + // { + // "/names/{name}/{other...}", + // "/names/" + url.PathEscape("john/doe") + "/address", + // map[string]string{ + // "name": "john/doe", + // "other": "address", + // }, + // }, + } { + mux := NewServeMux() + mux.HandleFunc(test.pattern, func(w ResponseWriter, r *Request) { + for name, want := range test.want { + got := r.PathValue(name) + if got != want { + t.Errorf("%q, %q: got %q, want %q", test.pattern, name, got, want) + } + } + }) + server := httptest.NewServer(mux) + defer server.Close() + _, err := Get(server.URL + test.url) + if err != nil { + t.Fatal(err) + } + } +} + +func TestSetPathValue(t *testing.T) { + mux := NewServeMux() + mux.HandleFunc("/a/{b}/c/{d...}", func(_ ResponseWriter, r *Request) { + kvs := map[string]string{ + "b": "X", + "d": "Y", + "a": "Z", + } + for k, v := range kvs { + r.SetPathValue(k, v) + } + for k, w := range kvs { + if g := r.PathValue(k); g != w { + t.Errorf("got %q, want %q", g, w) + } + } + }) + server := httptest.NewServer(mux) + defer server.Close() + _, err := Get(server.URL + "/a/b/c/d/e") + if err != nil { + t.Fatal(err) + } +} diff --git a/server.go b/server.go index 74362a69..a2291691 100644 --- a/server.go +++ b/server.go @@ -2410,14 +2410,15 @@ func stripHostPort(h string) string { // If there is no registered handler that applies to the request, // Handler returns a “page not found” handler and an empty pattern. func (mux *ServeMux) Handler(r *Request) (h Handler, pattern string) { - return mux.findHandler(r) + h, p, _, _ := mux.findHandler(r) + return h, p } // findHandler finds a handler for a request. // If there is a matching handler, it returns it and the pattern that matched. // Otherwise it returns a Redirect or NotFound handler with the path that would match // after the redirect. -func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string) { +func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string, _ *pattern, matches []string) { var n *routingNode // TODO(jba): use escaped path. This is an independent change that is also part // of proposal https://go.dev/issue/61410. @@ -2430,11 +2431,11 @@ func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string) { // but the path canonicalization does not. _, _, u := mux.matchOrRedirect(r.URL.Host, r.Method, path, r.URL) if u != nil { - return RedirectHandler(u.String(), StatusMovedPermanently), u.Path + return RedirectHandler(u.String(), StatusMovedPermanently), u.Path, nil, nil } // Redo the match, this time with r.Host instead of r.URL.Host. // Pass a nil URL to skip the trailing-slash redirect logic. - n, _, _ = mux.matchOrRedirect(r.Host, r.Method, path, nil) + n, matches, _ = mux.matchOrRedirect(r.Host, r.Method, path, nil) } else { // All other requests have any port stripped and path cleaned // before passing to mux.handler. @@ -2444,9 +2445,9 @@ func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string) { // If the given path is /tree and its handler is not registered, // redirect for /tree/. var u *url.URL - n, _, u = mux.matchOrRedirect(host, r.Method, path, r.URL) + n, matches, u = mux.matchOrRedirect(host, r.Method, path, r.URL) if u != nil { - return RedirectHandler(u.String(), StatusMovedPermanently), u.Path + return RedirectHandler(u.String(), StatusMovedPermanently), u.Path, nil, nil } if path != r.URL.Path { // Redirect to cleaned path. @@ -2455,14 +2456,14 @@ func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string) { patStr = n.pattern.String() } u := &url.URL{Path: path, RawQuery: r.URL.RawQuery} - return RedirectHandler(u.String(), StatusMovedPermanently), patStr + return RedirectHandler(u.String(), StatusMovedPermanently), patStr, nil, nil } } if n == nil { // TODO(jba): support 405 (MethodNotAllowed) by checking for patterns with different methods. - return NotFoundHandler(), "" + return NotFoundHandler(), "", nil, nil } - return n.handler, n.pattern.String() + return n.handler, n.pattern.String(), n.pattern, matches } // matchOrRedirect looks up a node in the tree that matches the host, method and path. @@ -2551,8 +2552,9 @@ func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { w.WriteHeader(StatusBadRequest) return } - h, _ := mux.findHandler(r) - // TODO(jba); save matches in Request. + h, _, pat, matches := mux.findHandler(r) + r.pat = pat + r.matches = matches h.ServeHTTP(w, r) } diff --git a/server_test.go b/server_test.go index b0cc093d..0c361c7d 100644 --- a/server_test.go +++ b/server_test.go @@ -110,7 +110,7 @@ func TestFindHandler(t *testing.T) { r.Method = test.method r.Host = "example.com" r.URL = &url.URL{Path: test.path} - gotH, _ := mux.findHandler(&r) + gotH, _, _, _ := mux.findHandler(&r) got := fmt.Sprintf("%#v", gotH) if got != test.wantHandler { t.Errorf("%s %q: got %q, want %q", test.method, test.path, got, test.wantHandler) @@ -204,7 +204,7 @@ func BenchmarkServerMatch(b *testing.B) { if err != nil { b.Fatal(err) } - if h, p := mux.findHandler(r); h != nil && p == "" { + if h, p, _, _ := mux.findHandler(r); h != nil && p == "" { b.Error("impossible") } } From 62023bc7d8ee7e1b259c1e09d89af9550bae73cb Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Thu, 14 Sep 2023 09:03:37 -0400 Subject: [PATCH 37/93] net/http: synchronize tests that use reqNum counters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This suppresses the race reported in #62638. I am not 100% certain how that race happens, but here is my theory. The increment of reqNum happens before the server writes the response headers, and the server necessarily writes the headers before the client receives them. However, that write/read pair occurs through I/O syscalls rather than Go synchronization primitives, so it doesn't necessarily create a “happens before” relationship as defined by the Go memory model: although we can establish a sequence of events, that sequence is not visible to the race detector, nor to the compiler. Fixes #62638. Change-Id: I90d66ec3fc32b9b8e1f9bbf0bc2eb289b964b99b Reviewed-on: https://go-review.googlesource.com/c/go/+/528475 LUCI-TryBot-Result: Go LUCI Reviewed-by: Damien Neil Auto-Submit: Bryan Mills --- serve_test.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/serve_test.go b/serve_test.go index 9fe99a37..ebf685bc 100644 --- a/serve_test.go +++ b/serve_test.go @@ -658,10 +658,9 @@ func testServerTimeouts(t *testing.T, mode testMode) { } func testServerTimeoutsWithTimeout(t *testing.T, timeout time.Duration, mode testMode) error { - reqNum := 0 + var reqNum atomic.Int32 ts := newClientServerTest(t, mode, HandlerFunc(func(res ResponseWriter, req *Request) { - reqNum++ - fmt.Fprintf(res, "req=%d", reqNum) + fmt.Fprintf(res, "req=%d", reqNum.Add(1)) }), func(ts *httptest.Server) { ts.Config.ReadTimeout = timeout ts.Config.WriteTimeout = timeout @@ -861,13 +860,15 @@ func TestWriteDeadlineEnforcedPerStream(t *testing.T) { } func testWriteDeadlineEnforcedPerStream(t *testing.T, mode testMode, timeout time.Duration) error { - reqNum := 0 + firstRequest := make(chan bool, 1) ts := newClientServerTest(t, mode, HandlerFunc(func(res ResponseWriter, req *Request) { - reqNum++ - if reqNum == 1 { - return // first request succeeds + select { + case firstRequest <- true: + // first request succeeds + default: + // second request times out + time.Sleep(timeout) } - time.Sleep(timeout) // second request times out }), func(ts *httptest.Server) { ts.Config.WriteTimeout = timeout / 2 }).ts @@ -917,13 +918,15 @@ func TestNoWriteDeadline(t *testing.T) { } func testNoWriteDeadline(t *testing.T, mode testMode, timeout time.Duration) error { - reqNum := 0 + firstRequest := make(chan bool, 1) ts := newClientServerTest(t, mode, HandlerFunc(func(res ResponseWriter, req *Request) { - reqNum++ - if reqNum == 1 { - return // first request succeeds + select { + case firstRequest <- true: + // first request succeeds + default: + // second request times out + time.Sleep(timeout) } - time.Sleep(timeout) // second request timesout })).ts c := ts.Client() From c2c93ecc1d68a44e791ce1fed3e6175d7754df47 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Fri, 15 Sep 2023 12:17:15 -0400 Subject: [PATCH 38/93] net/http: handle MethodNotAllowed If no pattern matches a request, but a pattern would have matched if the request had a different method, then serve a 405 (Method Not Allowed), and populate the "Allow" header with the methods that would have succeeded. Updates #61640. Change-Id: I0ae9eb95e62c71ff7766a03043525a97099ac1bb Reviewed-on: https://go-review.googlesource.com/c/go/+/528401 Reviewed-by: Damien Neil LUCI-TryBot-Result: Go LUCI --- request_test.go | 55 +++++++++++++++++++++++++++++++++++++++-- routing_tree.go | 27 ++++++++++++++++++++ routing_tree_test.go | 59 ++++++++++++++++++++++++++++++++++++++++++++ server.go | 42 ++++++++++++++++++++++++++++--- 4 files changed, 177 insertions(+), 6 deletions(-) diff --git a/request_test.go b/request_test.go index 1aeb93fe..18034ce1 100644 --- a/request_test.go +++ b/request_test.go @@ -15,6 +15,7 @@ import ( "io" "math" "mime/multipart" + "net/http" . "net/http" "net/http/httptest" "net/url" @@ -1473,10 +1474,11 @@ func TestPathValue(t *testing.T) { }) server := httptest.NewServer(mux) defer server.Close() - _, err := Get(server.URL + test.url) + res, err := Get(server.URL + test.url) if err != nil { t.Fatal(err) } + res.Body.Close() } } @@ -1499,8 +1501,57 @@ func TestSetPathValue(t *testing.T) { }) server := httptest.NewServer(mux) defer server.Close() - _, err := Get(server.URL + "/a/b/c/d/e") + res, err := Get(server.URL + "/a/b/c/d/e") if err != nil { t.Fatal(err) } + res.Body.Close() +} + +func TestStatus(t *testing.T) { + // The main purpose of this test is to check 405 responses and the Allow header. + h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}) + mux := NewServeMux() + mux.Handle("GET /g", h) + mux.Handle("POST /p", h) + mux.Handle("PATCH /p", h) + mux.Handle("PUT /r", h) + mux.Handle("GET /r/", h) + server := httptest.NewServer(mux) + defer server.Close() + + for _, test := range []struct { + method, path string + wantStatus int + wantAllow string + }{ + {"GET", "/g", 200, ""}, + {"HEAD", "/g", 200, ""}, + {"POST", "/g", 405, "GET, HEAD"}, + {"GET", "/x", 404, ""}, + {"GET", "/p", 405, "PATCH, POST"}, + {"GET", "/./p", 405, "PATCH, POST"}, + {"GET", "/r/", 200, ""}, + {"GET", "/r", 200, ""}, // redirected + {"HEAD", "/r/", 200, ""}, + {"HEAD", "/r", 200, ""}, // redirected + {"PUT", "/r/", 405, "GET, HEAD"}, + {"PUT", "/r", 200, ""}, + } { + req, err := http.NewRequest(test.method, server.URL+test.path, nil) + if err != nil { + t.Fatal(err) + } + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + res.Body.Close() + if g, w := res.StatusCode, test.wantStatus; g != w { + t.Errorf("%s %s: got %d, want %d", test.method, test.path, g, w) + } + if g, w := res.Header.Get("Allow"), test.wantAllow; g != w { + t.Errorf("%s %s, Allow: got %q, want %q", test.method, test.path, g, w) + } + } } diff --git a/routing_tree.go b/routing_tree.go index e225b5fd..46287174 100644 --- a/routing_tree.go +++ b/routing_tree.go @@ -220,3 +220,30 @@ func firstSegment(path string) (seg, rest string) { } return path[:i], path[i:] } + +// matchingMethods adds to methodSet all the methods that would result in a +// match if passed to routingNode.match with the given host and path. +func (root *routingNode) matchingMethods(host, path string, methodSet map[string]bool) { + if host != "" { + root.findChild(host).matchingMethodsPath(path, methodSet) + } + root.emptyChild.matchingMethodsPath(path, methodSet) + if methodSet["GET"] { + methodSet["HEAD"] = true + } +} + +func (n *routingNode) matchingMethodsPath(path string, set map[string]bool) { + if n == nil { + return + } + n.children.eachPair(func(method string, c *routingNode) bool { + if p, _ := c.matchPath(path, nil); p != nil { + set[method] = true + } + return true + }) + // Don't look at the empty child. If there were an empty + // child, it would match on any method, but we only + // call this when we fail to match on a method. +} diff --git a/routing_tree_test.go b/routing_tree_test.go index 42d7b995..149349f3 100644 --- a/routing_tree_test.go +++ b/routing_tree_test.go @@ -209,6 +209,65 @@ func TestRoutingNodeMatch(t *testing.T) { }) } +func TestMatchingMethods(t *testing.T) { + hostTree := buildTree("GET a.com/", "PUT b.com/", "POST /foo/{x}") + for _, test := range []struct { + name string + tree *routingNode + host, path string + want string + }{ + { + "post", + buildTree("POST /"), "", "/foo", + "POST", + }, + { + "get", + buildTree("GET /"), "", "/foo", + "GET,HEAD", + }, + { + "host", + hostTree, "", "/foo", + "", + }, + { + "host", + hostTree, "", "/foo/bar", + "POST", + }, + { + "host2", + hostTree, "a.com", "/foo/bar", + "GET,HEAD,POST", + }, + { + "host3", + hostTree, "b.com", "/bar", + "PUT", + }, + { + // This case shouldn't come up because we only call matchingMethods + // when there was no match, but we include it for completeness. + "empty", + buildTree("/"), "", "/", + "", + }, + } { + t.Run(test.name, func(t *testing.T) { + ms := map[string]bool{} + test.tree.matchingMethods(test.host, test.path, ms) + keys := mapKeys(ms) + sort.Strings(keys) + got := strings.Join(keys, ",") + if got != test.want { + t.Errorf("got %s, want %s", got, test.want) + } + }) + } +} + func (n *routingNode) print(w io.Writer, level int) { indent := strings.Repeat(" ", level) if n.pattern != nil { diff --git a/server.go b/server.go index a2291691..bc5bcb9a 100644 --- a/server.go +++ b/server.go @@ -23,6 +23,7 @@ import ( urlpkg "net/url" "path" "runtime" + "sort" "strconv" "strings" "sync" @@ -2423,13 +2424,13 @@ func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string, _ *patte // TODO(jba): use escaped path. This is an independent change that is also part // of proposal https://go.dev/issue/61410. path := r.URL.Path - + host := r.URL.Host // CONNECT requests are not canonicalized. if r.Method == "CONNECT" { // If r.URL.Path is /tree and its handler is not registered, // the /tree -> /tree/ redirect applies to CONNECT requests // but the path canonicalization does not. - _, _, u := mux.matchOrRedirect(r.URL.Host, r.Method, path, r.URL) + _, _, u := mux.matchOrRedirect(host, r.Method, path, r.URL) if u != nil { return RedirectHandler(u.String(), StatusMovedPermanently), u.Path, nil, nil } @@ -2439,7 +2440,7 @@ func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string, _ *patte } else { // All other requests have any port stripped and path cleaned // before passing to mux.handler. - host := stripHostPort(r.Host) + host = stripHostPort(r.Host) path = cleanPath(path) // If the given path is /tree and its handler is not registered, @@ -2460,7 +2461,16 @@ func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string, _ *patte } } if n == nil { - // TODO(jba): support 405 (MethodNotAllowed) by checking for patterns with different methods. + // We didn't find a match with the request method. To distinguish between + // Not Found and Method Not Allowed, see if there is another pattern that + // matches except for the method. + allowedMethods := mux.matchingMethods(host, path) + if len(allowedMethods) > 0 { + return HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Allow", strings.Join(allowedMethods, ", ")) + Error(w, StatusText(StatusMethodNotAllowed), StatusMethodNotAllowed) + }), "", nil, nil + } return NotFoundHandler(), "", nil, nil } return n.handler, n.pattern.String(), n.pattern, matches @@ -2542,6 +2552,30 @@ func exactMatch(n *routingNode, path string) bool { return len(n.pattern.segments) == strings.Count(path, "/") } +// matchingMethods return a sorted list of all methods that would match with the given host and path. +func (mux *ServeMux) matchingMethods(host, path string) []string { + // Hold the read lock for the entire method so that the two matches are done + // on the same set of registered patterns. + mux.mu.RLock() + defer mux.mu.RUnlock() + ms := map[string]bool{} + mux.tree.matchingMethods(host, path, ms) + // matchOrRedirect will try appending a trailing slash if there is no match. + mux.tree.matchingMethods(host, path+"/", ms) + methods := mapKeys(ms) + sort.Strings(methods) + return methods +} + +// TODO: replace with maps.Keys when it is defined. +func mapKeys[K comparable, V any](m map[K]V) []K { + var ks []K + for k := range m { + ks = append(ks, k) + } + return ks +} + // ServeHTTP dispatches the request to the handler whose // pattern most closely matches the request URL. func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { From ab839be4af00019267ca434031306a38608b826e Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 15 Sep 2023 10:37:08 -0400 Subject: [PATCH 39/93] net/http: avoid leaking goroutines when TestServerGracefulClose retries If the call to ReadString returns an error, the closure in testServerGracefulClose will return an error and retry the test with a longer timeout. If that happens, we need to wait for the conn.Write goroutine to complete so that we don't leak connections across tests. Updates #57084. Fixes #62643. Change-Id: Ia86c1bbd0a5e5d0aeccf4dfeb994c19d1fb10b00 Reviewed-on: https://go-review.googlesource.com/c/go/+/528398 Auto-Submit: Bryan Mills Reviewed-by: Than McIntosh TryBot-Result: Gopher Robot Run-TryBot: Bryan Mills Reviewed-by: Damien Neil --- serve_test.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/serve_test.go b/serve_test.go index ebf685bc..cadadf48 100644 --- a/serve_test.go +++ b/serve_test.go @@ -3135,12 +3135,19 @@ func testServerGracefulClose(t *testing.T, mode testMode) { if err != nil { return err } - defer conn.Close() writeErr := make(chan error) go func() { _, err := conn.Write(req) writeErr <- err }() + defer func() { + conn.Close() + // Wait for write to finish. This is a broken pipe on both + // Darwin and Linux, but checking this isn't the point of + // the test. + <-writeErr + }() + br := bufio.NewReader(conn) lineNum := 0 for { @@ -3156,10 +3163,6 @@ func testServerGracefulClose(t *testing.T, mode testMode) { t.Errorf("Response line = %q; want a 401", line) } } - // Wait for write to finish. This is a broken pipe on both - // Darwin and Linux, but checking this isn't the point of - // the test. - <-writeErr return nil }) } From 44b3cfbed1dcd66608d95e4894bf619087e271ab Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Mon, 18 Sep 2023 13:55:42 -0400 Subject: [PATCH 40/93] net/http: fix bugs in comparePaths and combineRelationships combineRelationships was wrong on one case: if one part of a pattern overlaps and the other is disjoint, the result is disjoint, not overlaps. For example: /a/{x}/c /{x}/b/d Here the prefix consisting of the first two segments overlaps, but the third segments are disjoint. The patterns as a whole are disjoint. comparePaths was wrong in a couple of ways: First, the loop shouldn't exit early when it sees an overlap, for the reason above: later information may change that. Once the loop was allowed to continue, we had to handle the "overlaps" case at the end. The insight there, which generalized the existing code, is that if the shorter path ends in a multi, that multi matches the remainder of the longer path and more. (It must be "and more": the longer path has at least two segments, so it couldn't match one segment while the shorter path's multi can.) That means we can treat the result as the combination moreGeneral and the relationship of the common prefix. Change-Id: I11dab2c020d820730fb38296d9d6b072bd2a5350 Reviewed-on: https://go-review.googlesource.com/c/go/+/529119 Reviewed-by: Damien Neil Run-TryBot: Jonathan Amsterdam TryBot-Result: Gopher Robot --- pattern.go | 33 ++++++++++++++++++++++++--------- pattern_test.go | 3 +++ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/pattern.go b/pattern.go index 3fd20b71..6b9e535b 100644 --- a/pattern.go +++ b/pattern.go @@ -273,12 +273,13 @@ func (p1 *pattern) comparePaths(p2 *pattern) relationship { if len(p1.segments) != len(p2.segments) && !p1.lastSegment().multi && !p2.lastSegment().multi { return disjoint } + + // Consider corresponding segments in the two path patterns. var segs1, segs2 []segment - // Look at corresponding segments in the two path patterns. rel := equivalent for segs1, segs2 = p1.segments, p2.segments; len(segs1) > 0 && len(segs2) > 0; segs1, segs2 = segs1[1:], segs2[1:] { rel = combineRelationships(rel, compareSegments(segs1[0], segs2[0])) - if rel == disjoint || rel == overlaps { + if rel == disjoint { return rel } } @@ -289,12 +290,13 @@ func (p1 *pattern) comparePaths(p2 *pattern) relationship { return rel } // Otherwise, the only way they could fail to be disjoint is if the shorter - // pattern ends in a multi and is more general. - if len(segs1) < len(segs2) && p1.lastSegment().multi && rel == moreGeneral { - return moreGeneral + // pattern ends in a multi. In that case, that multi is more general + // than the remainder of the longer pattern, so combine those two relationships. + if len(segs1) < len(segs2) && p1.lastSegment().multi { + return combineRelationships(rel, moreGeneral) } - if len(segs2) < len(segs1) && p2.lastSegment().multi && rel == moreSpecific { - return moreSpecific + if len(segs2) < len(segs1) && p2.lastSegment().multi { + return combineRelationships(rel, moreSpecific) } return disjoint } @@ -345,8 +347,13 @@ func combineRelationships(r1, r2 relationship) relationship { switch r1 { case equivalent: return r2 - case disjoint, overlaps: - return r1 + case disjoint: + return disjoint + case overlaps: + if r2 == disjoint { + return disjoint + } + return overlaps case moreGeneral, moreSpecific: switch r2 { case equivalent: @@ -373,3 +380,11 @@ func inverseRelationship(r relationship) relationship { return r } } + +// isLitOrSingle reports whether the segment is a non-dollar literal or a single wildcard. +func isLitOrSingle(seg segment) bool { + if seg.wild { + return !seg.multi + } + return seg.s != "/" +} diff --git a/pattern_test.go b/pattern_test.go index cd27cd8d..7c518979 100644 --- a/pattern_test.go +++ b/pattern_test.go @@ -296,6 +296,7 @@ func TestComparePaths(t *testing.T) { {"/a/{z}/{m...}", "/{z}/a/", overlaps}, {"/a/{z}/{m...}", "/{z}/b/{y...}", overlaps}, {"/a/{z}/b/{m...}", "/{x}/c/{y...}", overlaps}, + {"/a/{z}/a/{m...}", "/{x}/b", disjoint}, // Dollar on left. {"/{$}", "/a", disjoint}, @@ -314,6 +315,8 @@ func TestComparePaths(t *testing.T) { {"/b/{$}", "/b/{x...}", moreSpecific}, {"/b/{$}", "/b/c/{x...}", disjoint}, {"/b/{x}/a/{$}", "/{x}/c/{y...}", overlaps}, + {"/{x}/b/{$}", "/a/{x}/{y}", disjoint}, + {"/{x}/b/{$}", "/a/{x}/c", disjoint}, {"/{z}/{$}", "/{z}/a", disjoint}, {"/{z}/{$}", "/{z}/a/b", disjoint}, From fc452117ec6d140faaf7ed2028c8852e8b438a46 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Tue, 19 Sep 2023 10:37:28 -0400 Subject: [PATCH 41/93] net/http: buffer the testConn close channel in TestHandlerFinishSkipBigContentLengthRead Previously the test used an unbuffered channel, but testConn.Close sends to it with a select-with-default, so the send would be dropped if the test goroutine happened not to have parked on the receive yet. To make this kind of bug less likely in future tests, use a newTestConn helper function instead of constructing testConn channel literals in each test individually. Fixes #62622. Change-Id: I016cd0a89cf8a2a748ed57a4cdbd01a178f04dab Reviewed-on: https://go-review.googlesource.com/c/go/+/529475 LUCI-TryBot-Result: Go LUCI Auto-Submit: Bryan Mills Reviewed-by: Damien Neil --- serve_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/serve_test.go b/serve_test.go index cadadf48..8fa40e61 100644 --- a/serve_test.go +++ b/serve_test.go @@ -107,10 +107,14 @@ type testConn struct { readMu sync.Mutex // for TestHandlerBodyClose readBuf bytes.Buffer writeBuf bytes.Buffer - closec chan bool // if non-nil, send value to it on close + closec chan bool // 1-buffered; receives true when Close is called noopConn } +func newTestConn() *testConn { + return &testConn{closec: make(chan bool, 1)} +} + func (c *testConn) Read(b []byte) (int, error) { c.readMu.Lock() defer c.readMu.Unlock() @@ -4589,10 +4593,10 @@ Host: foo } // If a Handler finishes and there's an unread request body, -// verify the server try to do implicit read on it before replying. +// verify the server implicitly tries to do a read on it before replying. func TestHandlerFinishSkipBigContentLengthRead(t *testing.T) { setParallel(t) - conn := &testConn{closec: make(chan bool)} + conn := newTestConn() conn.readBuf.Write([]byte(fmt.Sprintf( "POST / HTTP/1.1\r\n" + "Host: test\r\n" + @@ -4682,7 +4686,7 @@ func TestServerValidatesHostHeader(t *testing.T) { {"GET / HTTP/3.0", "", 505}, } for _, tt := range tests { - conn := &testConn{closec: make(chan bool, 1)} + conn := newTestConn() methodTarget := "GET / " if !strings.HasPrefix(tt.proto, "HTTP/") { methodTarget = "" @@ -4780,7 +4784,7 @@ func TestServerValidatesHeaders(t *testing.T) { {"foo: foo\xfffoo\r\n", 200}, // non-ASCII high octets in value are fine } for _, tt := range tests { - conn := &testConn{closec: make(chan bool, 1)} + conn := newTestConn() io.WriteString(&conn.readBuf, "GET / HTTP/1.1\r\nHost: foo\r\n"+tt.header+"\r\n") ln := &oneConnListener{conn} @@ -5166,11 +5170,7 @@ Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3 `) res := []byte("Hello world!\n") - conn := &testConn{ - // testConn.Close will not push into the channel - // if it's full. - closec: make(chan bool, 1), - } + conn := newTestConn() handler := HandlerFunc(func(rw ResponseWriter, r *Request) { rw.Header().Set("Content-Type", "text/html; charset=utf-8") rw.Write(res) @@ -5991,7 +5991,7 @@ func TestServerValidatesMethod(t *testing.T) { {"GE(T", 400}, } for _, tt := range tests { - conn := &testConn{closec: make(chan bool, 1)} + conn := newTestConn() io.WriteString(&conn.readBuf, tt.method+" / HTTP/1.1\r\nHost: foo.example\r\n\r\n") ln := &oneConnListener{conn} From 65cb8e5447f0a1778b10a2d118ba10cc73031017 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Mon, 18 Sep 2023 15:51:04 -0400 Subject: [PATCH 42/93] net/http: index patterns for faster conflict detection Add an index so that pattern registration isn't always quadratic. If there were no index, then every pattern that was registered would have to be compared to every existing pattern for conflicts. This would make registration quadratic in the number of patterns, in every case. The index in this CL should help most of the time. If a pattern has a literal segment, it will weed out all other patterns that have a different literal in that position. The worst case will still be quadratic, but it is unlikely that a set of such patterns would arise naturally. One novel (to me) aspect of the CL is the use of fuzz testing on data that is neither a string nor a byte slice. The test uses fuzzing to generate a byte slice, then decodes the byte slice into a valid pattern (most of the time). This test actually caught a bug: see https://go.dev/cl/529119. Change-Id: Ice0be6547decb5ce75a8062e4e17227815d5d0b0 Reviewed-on: https://go-review.googlesource.com/c/go/+/529121 Run-TryBot: Jonathan Amsterdam TryBot-Result: Gopher Robot Reviewed-by: Damien Neil --- routing_index.go | 124 ++++++++++++++ routing_index_test.go | 207 +++++++++++++++++++++++ server.go | 20 ++- testdata/fuzz/FuzzIndex/48161038f0c8b2da | 2 + testdata/fuzz/FuzzIndex/716514f590ce7ab3 | 2 + 5 files changed, 346 insertions(+), 9 deletions(-) create mode 100644 routing_index.go create mode 100644 routing_index_test.go create mode 100644 testdata/fuzz/FuzzIndex/48161038f0c8b2da create mode 100644 testdata/fuzz/FuzzIndex/716514f590ce7ab3 diff --git a/routing_index.go b/routing_index.go new file mode 100644 index 00000000..9ac42c99 --- /dev/null +++ b/routing_index.go @@ -0,0 +1,124 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import "math" + +// A routingIndex optimizes conflict detection by indexing patterns. +// +// The basic idea is to rule out patterns that cannot conflict with a given +// pattern because they have a different literal in a corresponding segment. +// See the comments in [routingIndex.possiblyConflictingPatterns] for more details. +type routingIndex struct { + // map from a particular segment position and value to all registered patterns + // with that value in that position. + // For example, the key {1, "b"} would hold the patterns "/a/b" and "/a/b/c" + // but not "/a", "b/a", "/a/c" or "/a/{x}". + segments map[routingIndexKey][]*pattern + // All patterns that end in a multi wildcard (including trailing slash). + // We do not try to be clever about indexing multi patterns, because there + // are unlikely to be many of them. + multis []*pattern +} + +type routingIndexKey struct { + pos int // 0-based segment position + s string // literal, or empty for wildcard +} + +func (idx *routingIndex) addPattern(pat *pattern) { + if pat.lastSegment().multi { + idx.multis = append(idx.multis, pat) + } else { + if idx.segments == nil { + idx.segments = map[routingIndexKey][]*pattern{} + } + for pos, seg := range pat.segments { + key := routingIndexKey{pos: pos, s: ""} + if !seg.wild { + key.s = seg.s + } + idx.segments[key] = append(idx.segments[key], pat) + } + } +} + +// possiblyConflictingPatterns calls f on all patterns that might conflict with +// pat. If f returns a non-nil error, possiblyConflictingPatterns returns immediately +// with that error. +// +// To be correct, possiblyConflictingPatterns must include all patterns that +// might conflict. But it may also include patterns that cannot conflict. +// For instance, an implementation that returns all registered patterns is correct. +// We use this fact throughout, simplifying the implementation by returning more +// patterns that we might need to. +func (idx *routingIndex) possiblyConflictingPatterns(pat *pattern, f func(*pattern) error) (err error) { + // Terminology: + // dollar pattern: one ending in "{$}" + // multi pattern: one ending in a trailing slash or "{x...}" wildcard + // ordinary pattern: neither of the above + + // apply f to all the pats, stopping on error. + apply := func(pats []*pattern) error { + if err != nil { + return err + } + for _, p := range pats { + err = f(p) + if err != nil { + return err + } + } + return nil + } + + // Our simple indexing scheme doesn't try to prune multi patterns; assume + // any of them can match the argument. + if err := apply(idx.multis); err != nil { + return err + } + if pat.lastSegment().s == "/" { + // All paths that a dollar pattern matches end in a slash; no paths that + // an ordinary pattern matches do. So only other dollar or multi + // patterns can conflict with a dollar pattern. Furthermore, conflicting + // dollar patterns must have the {$} in the same position. + return apply(idx.segments[routingIndexKey{s: "/", pos: len(pat.segments) - 1}]) + } + // For ordinary and multi patterns, the only conflicts can be with a multi, + // or a pattern that has the same literal or a wildcard at some literal + // position. + // We could intersect all the possible matches at each position, but we + // do something simpler: we find the position with the fewest patterns. + var lmin, wmin []*pattern + min := math.MaxInt + hasLit := false + for i, seg := range pat.segments { + if seg.multi { + break + } + if !seg.wild { + hasLit = true + lpats := idx.segments[routingIndexKey{s: seg.s, pos: i}] + wpats := idx.segments[routingIndexKey{s: "", pos: i}] + if sum := len(lpats) + len(wpats); sum < min { + lmin = lpats + wmin = wpats + min = sum + } + } + } + if hasLit { + apply(lmin) + apply(wmin) + return err + } + + // This pattern is all wildcards. + // Check it against everything. + for _, pats := range idx.segments { + apply(pats) + } + return err +} diff --git a/routing_index_test.go b/routing_index_test.go new file mode 100644 index 00000000..7030fc8a --- /dev/null +++ b/routing_index_test.go @@ -0,0 +1,207 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +import ( + "bytes" + "fmt" + "slices" + "sort" + "strings" + "testing" +) + +func TestIndex(t *testing.T) { + pats := []string{"HEAD /", "/a"} + + var patterns []*pattern + var idx routingIndex + for _, p := range pats { + pat := mustParsePattern(t, p) + patterns = append(patterns, pat) + idx.addPattern(pat) + } + + compare := func(pat *pattern) { + t.Helper() + got := indexConflicts(pat, &idx) + want := trueConflicts(pat, patterns) + if !slices.Equal(got, want) { + t.Errorf("%q:\ngot %q\nwant %q", pat, got, want) + } + } + + compare(mustParsePattern(t, "GET /foo")) + compare(mustParsePattern(t, "GET /{x}")) +} + +// This test works by comparing possiblyConflictingPatterns with +// an exhaustive loop through all patterns. +func FuzzIndex(f *testing.F) { + inits := []string{"/a", "/a/b", "/{x0}", "/{x0}/b", "/a/{x0}", "/a/{$}", "/a/b/{$}", + "/a/", "/a/b/", "/{x}/b/c/{$}", "GET /{x0}/", "HEAD /a"} + + var patterns []*pattern + var idx routingIndex + + // compare takes a fatalf function because fuzzing doesn't like + // it when the fuzz function calls f.Fatalf. + compare := func(pat *pattern, fatalf func(string, ...any)) { + got := indexConflicts(pat, &idx) + want := trueConflicts(pat, patterns) + if !slices.Equal(got, want) { + fatalf("%q:\ngot %q\nwant %q", pat, got, want) + } + } + + for _, p := range inits { + pat, err := parsePattern(p) + if err != nil { + f.Fatal(err) + } + compare(pat, f.Fatalf) + patterns = append(patterns, pat) + idx.addPattern(pat) + f.Add(bytesFromPattern(pat)) + } + + f.Fuzz(func(t *testing.T, pb []byte) { + pat := bytesToPattern(pb) + if pat == nil { + return + } + compare(pat, t.Fatalf) + }) +} + +func trueConflicts(pat *pattern, pats []*pattern) []string { + var s []string + for _, p := range pats { + if pat.conflictsWith(p) { + s = append(s, p.String()) + } + } + sort.Strings(s) + return s +} + +func indexConflicts(pat *pattern, idx *routingIndex) []string { + var s []string + idx.possiblyConflictingPatterns(pat, func(p *pattern) error { + if pat.conflictsWith(p) { + s = append(s, p.String()) + } + return nil + }) + sort.Strings(s) + return slices.Compact(s) +} + +// TODO: incorporate host and method; make encoding denser. +func bytesToPattern(bs []byte) *pattern { + if len(bs) == 0 { + return nil + } + var sb strings.Builder + wc := 0 + for _, b := range bs[:len(bs)-1] { + sb.WriteByte('/') + switch b & 0x3 { + case 0: + fmt.Fprintf(&sb, "{x%d}", wc) + wc++ + case 1: + sb.WriteString("a") + case 2: + sb.WriteString("b") + case 3: + sb.WriteString("c") + } + } + sb.WriteByte('/') + switch bs[len(bs)-1] & 0x7 { + case 0: + fmt.Fprintf(&sb, "{x%d}", wc) + case 1: + sb.WriteString("a") + case 2: + sb.WriteString("b") + case 3: + sb.WriteString("c") + case 4, 5: + fmt.Fprintf(&sb, "{x%d...}", wc) + default: + sb.WriteString("{$}") + } + pat, err := parsePattern(sb.String()) + if err != nil { + panic(err) + } + return pat +} + +func bytesFromPattern(p *pattern) []byte { + var bs []byte + for _, s := range p.segments { + var b byte + switch { + case s.multi: + b = 4 + case s.wild: + b = 0 + case s.s == "/": + b = 7 + case s.s == "a": + b = 1 + case s.s == "b": + b = 2 + case s.s == "c": + b = 3 + default: + panic("bad pattern") + } + bs = append(bs, b) + } + return bs +} + +func TestBytesPattern(t *testing.T) { + tests := []struct { + bs []byte + pat string + }{ + {[]byte{0, 1, 2, 3}, "/{x0}/a/b/c"}, + {[]byte{16, 17, 18, 19}, "/{x0}/a/b/c"}, + {[]byte{4, 4}, "/{x0}/{x1...}"}, + {[]byte{6, 7}, "/b/{$}"}, + } + t.Run("To", func(t *testing.T) { + for _, test := range tests { + p := bytesToPattern(test.bs) + got := p.String() + if got != test.pat { + t.Errorf("%v: got %q, want %q", test.bs, got, test.pat) + } + } + }) + t.Run("From", func(t *testing.T) { + for _, test := range tests { + p, err := parsePattern(test.pat) + if err != nil { + t.Fatal(err) + } + got := bytesFromPattern(p) + var want []byte + for _, b := range test.bs[:len(test.bs)-1] { + want = append(want, b%4) + + } + want = append(want, test.bs[len(test.bs)-1]%8) + if !bytes.Equal(got, want) { + t.Errorf("%s: got %v, want %v", test.pat, got, want) + } + } + }) +} diff --git a/server.go b/server.go index bc5bcb9a..629d8d3c 100644 --- a/server.go +++ b/server.go @@ -2347,7 +2347,8 @@ func RedirectHandler(url string, code int) Handler { type ServeMux struct { mu sync.RWMutex tree routingNode - patterns []*pattern + index routingIndex + patterns []*pattern // TODO(jba): remove if possible } // NewServeMux allocates and returns a new ServeMux. @@ -2624,8 +2625,8 @@ func (mux *ServeMux) register(pattern string, handler Handler) { } } -func (mux *ServeMux) registerErr(pattern string, handler Handler) error { - if pattern == "" { +func (mux *ServeMux) registerErr(patstr string, handler Handler) error { + if patstr == "" { return errors.New("http: invalid pattern") } if handler == nil { @@ -2635,9 +2636,9 @@ func (mux *ServeMux) registerErr(pattern string, handler Handler) error { return errors.New("http: nil handler") } - pat, err := parsePattern(pattern) + pat, err := parsePattern(patstr) if err != nil { - return fmt.Errorf("parsing %q: %w", pattern, err) + return fmt.Errorf("parsing %q: %w", patstr, err) } // Get the caller's location, for better conflict error messages. @@ -2652,16 +2653,17 @@ func (mux *ServeMux) registerErr(pattern string, handler Handler) error { mux.mu.Lock() defer mux.mu.Unlock() // Check for conflict. - // This makes a quadratic number of calls to conflictsWith: we check - // each pattern against every other pattern. - // TODO(jba): add indexing to speed this up. - for _, pat2 := range mux.patterns { + if err := mux.index.possiblyConflictingPatterns(pat, func(pat2 *pattern) error { if pat.conflictsWith(pat2) { return fmt.Errorf("pattern %q (registered at %s) conflicts with pattern %q (registered at %s)", pat, pat.loc, pat2, pat2.loc) } + return nil + }); err != nil { + return err } mux.tree.addPattern(pat, handler) + mux.index.addPattern(pat) mux.patterns = append(mux.patterns, pat) return nil } diff --git a/testdata/fuzz/FuzzIndex/48161038f0c8b2da b/testdata/fuzz/FuzzIndex/48161038f0c8b2da new file mode 100644 index 00000000..06a7336a --- /dev/null +++ b/testdata/fuzz/FuzzIndex/48161038f0c8b2da @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("101$") diff --git a/testdata/fuzz/FuzzIndex/716514f590ce7ab3 b/testdata/fuzz/FuzzIndex/716514f590ce7ab3 new file mode 100644 index 00000000..520bff17 --- /dev/null +++ b/testdata/fuzz/FuzzIndex/716514f590ce7ab3 @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("1010") From 7f22ddf9f2c5c2fe889102e49376121f6d04a36a Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Tue, 19 Sep 2023 07:54:09 -0400 Subject: [PATCH 43/93] net/http: explain why two patterns conflict It can be difficult to tell at a glance why two patterns conflict, so explain it with example paths. Change-Id: Ie384f0a4ef64f30e6e6898bce4b88027bc81034b Reviewed-on: https://go-review.googlesource.com/c/go/+/529122 Run-TryBot: Jonathan Amsterdam Reviewed-by: Damien Neil TryBot-Result: Gopher Robot --- pattern.go | 129 ++++++++++++++++++++++++++++++++++++++++++++++++ pattern_test.go | 112 +++++++++++++++++++++++++++++++++++++++++ server.go | 5 +- 3 files changed, 244 insertions(+), 2 deletions(-) diff --git a/pattern.go b/pattern.go index 6b9e535b..eca17918 100644 --- a/pattern.go +++ b/pattern.go @@ -388,3 +388,132 @@ func isLitOrSingle(seg segment) bool { } return seg.s != "/" } + +// describeConflict returns an explanation of why two patterns conflict. +func describeConflict(p1, p2 *pattern) string { + mrel := p1.compareMethods(p2) + prel := p1.comparePaths(p2) + rel := combineRelationships(mrel, prel) + if rel == equivalent { + return fmt.Sprintf("%s matches the same requests as %s", p1, p2) + } + if rel != overlaps { + panic("describeConflict called with non-conflicting patterns") + } + if prel == overlaps { + return fmt.Sprintf(`%[1]s and %[2]s both match some paths, like %[3]q. +But neither is more specific than the other. +%[1]s matches %[4]q, but %[2]s doesn't. +%[2]s matches %[5]q, but %[1]s doesn't.`, + p1, p2, commonPath(p1, p2), differencePath(p1, p2), differencePath(p2, p1)) + } + if mrel == moreGeneral && prel == moreSpecific { + return fmt.Sprintf("%s matches more methods than %s, but has a more specific path pattern", p1, p2) + } + if mrel == moreSpecific && prel == moreGeneral { + return fmt.Sprintf("%s matches fewer methods than %s, but has a more general path pattern", p1, p2) + } + return fmt.Sprintf("bug: unexpected way for two patterns %s and %s to conflict: methods %s, paths %s", p1, p2, mrel, prel) +} + +// writeMatchingPath writes to b a path that matches the segments. +func writeMatchingPath(b *strings.Builder, segs []segment) { + for _, s := range segs { + writeSegment(b, s) + } +} + +func writeSegment(b *strings.Builder, s segment) { + b.WriteByte('/') + if !s.multi && s.s != "/" { + b.WriteString(s.s) + } +} + +// commonPath returns a path that both p1 and p2 match. +// It assumes there is such a path. +func commonPath(p1, p2 *pattern) string { + var b strings.Builder + var segs1, segs2 []segment + for segs1, segs2 = p1.segments, p2.segments; len(segs1) > 0 && len(segs2) > 0; segs1, segs2 = segs1[1:], segs2[1:] { + if s1 := segs1[0]; s1.wild { + writeSegment(&b, segs2[0]) + } else { + writeSegment(&b, s1) + } + } + if len(segs1) > 0 { + writeMatchingPath(&b, segs1) + } else if len(segs2) > 0 { + writeMatchingPath(&b, segs2) + } + return b.String() +} + +// differencePath returns a path that p1 matches and p2 doesn't. +// It assumes there is such a path. +func differencePath(p1, p2 *pattern) string { + var b strings.Builder + + var segs1, segs2 []segment + for segs1, segs2 = p1.segments, p2.segments; len(segs1) > 0 && len(segs2) > 0; segs1, segs2 = segs1[1:], segs2[1:] { + s1 := segs1[0] + s2 := segs2[0] + if s1.multi && s2.multi { + // From here the patterns match the same paths, so we must have found a difference earlier. + b.WriteByte('/') + return b.String() + + } + if s1.multi && !s2.multi { + // s1 ends in a "..." wildcard but s2 does not. + // A trailing slash will distinguish them, unless s2 ends in "{$}", + // in which case any segment will do; prefer the wildcard name if + // it has one. + b.WriteByte('/') + if s2.s == "/" { + if s1.s != "" { + b.WriteString(s1.s) + } else { + b.WriteString("x") + } + } + return b.String() + } + if !s1.multi && s2.multi { + writeSegment(&b, s1) + } else if s1.wild && s2.wild { + // Both patterns will match whatever we put here; use + // the first wildcard name. + writeSegment(&b, s1) + } else if s1.wild && !s2.wild { + // s1 is a wildcard, s2 is a literal. + // Any segment other than s2.s will work. + // Prefer the wildcard name, but if it's the same as the literal, + // tweak the literal. + if s1.s != s2.s { + writeSegment(&b, s1) + } else { + b.WriteByte('/') + b.WriteString(s2.s + "x") + } + } else if !s1.wild && s2.wild { + writeSegment(&b, s1) + } else { + // Both are literals. A precondition of this function is that the + // patterns overlap, so they must be the same literal. Use it. + if s1.s != s2.s { + panic(fmt.Sprintf("literals differ: %q and %q", s1.s, s2.s)) + } + writeSegment(&b, s1) + } + } + if len(segs1) > 0 { + // p1 is longer than p2, and p2 does not end in a multi. + // Anything that matches the rest of p1 will do. + writeMatchingPath(&b, segs1) + } else if len(segs2) > 0 { + writeMatchingPath(&b, segs2) + } + return b.String() +} diff --git a/pattern_test.go b/pattern_test.go index 7c518979..f67a2b51 100644 --- a/pattern_test.go +++ b/pattern_test.go @@ -392,3 +392,115 @@ func TestConflictsWith(t *testing.T) { } } } + +func TestRegisterConflict(t *testing.T) { + mux := NewServeMux() + pat1 := "/a/{x}/" + if err := mux.registerErr(pat1, NotFoundHandler()); err != nil { + t.Fatal(err) + } + pat2 := "/a/{y}/{z...}" + err := mux.registerErr(pat2, NotFoundHandler()) + var got string + if err == nil { + got = "" + } else { + got = err.Error() + } + want := "matches the same requests as" + if !strings.Contains(got, want) { + t.Errorf("got\n%s\nwant\n%s", got, want) + } +} + +func TestDescribeConflict(t *testing.T) { + for _, test := range []struct { + p1, p2 string + want string + }{ + {"/a/{x}", "/a/{y}", "the same requests"}, + {"/", "/{m...}", "the same requests"}, + {"/a/{x}", "/{y}/b", "both match some paths"}, + {"/a", "GET /{x}", "matches more methods than GET /{x}, but has a more specific path pattern"}, + {"GET /a", "HEAD /", "matches more methods than HEAD /, but has a more specific path pattern"}, + {"POST /", "/a", "matches fewer methods than /a, but has a more general path pattern"}, + } { + got := describeConflict(mustParsePattern(t, test.p1), mustParsePattern(t, test.p2)) + if !strings.Contains(got, test.want) { + t.Errorf("%s vs. %s:\ngot:\n%s\nwhich does not contain %q", + test.p1, test.p2, got, test.want) + } + } +} + +func TestCommonPath(t *testing.T) { + for _, test := range []struct { + p1, p2 string + want string + }{ + {"/a/{x}", "/{x}/a", "/a/a"}, + {"/a/{z}/", "/{z}/a/", "/a/a/"}, + {"/a/{z}/{m...}", "/{z}/a/", "/a/a/"}, + {"/{z}/{$}", "/a/", "/a/"}, + {"/{z}/{$}", "/a/{x...}", "/a/"}, + {"/a/{z}/{$}", "/{z}/a/", "/a/a/"}, + {"/a/{x}/b/{y...}", "/{x}/c/{y...}", "/a/c/b/"}, + {"/a/{x}/b/", "/{x}/c/{y...}", "/a/c/b/"}, + {"/a/{x}/b/{$}", "/{x}/c/{y...}", "/a/c/b/"}, + {"/a/{z}/{x...}", "/{z}/b/{y...}", "/a/b/"}, + } { + pat1 := mustParsePattern(t, test.p1) + pat2 := mustParsePattern(t, test.p2) + if pat1.comparePaths(pat2) != overlaps { + t.Fatalf("%s does not overlap %s", test.p1, test.p2) + } + got := commonPath(pat1, pat2) + if got != test.want { + t.Errorf("%s vs. %s: got %q, want %q", test.p1, test.p2, got, test.want) + } + } +} + +func TestDifferencePath(t *testing.T) { + for _, test := range []struct { + p1, p2 string + want string + }{ + {"/a/{x}", "/{x}/a", "/a/x"}, + {"/{x}/a", "/a/{x}", "/x/a"}, + {"/a/{z}/", "/{z}/a/", "/a/z/"}, + {"/{z}/a/", "/a/{z}/", "/z/a/"}, + {"/{a}/a/", "/a/{z}/", "/ax/a/"}, + {"/a/{z}/{x...}", "/{z}/b/{y...}", "/a/z/"}, + {"/{z}/b/{y...}", "/a/{z}/{x...}", "/z/b/"}, + {"/a/b/", "/a/b/c", "/a/b/"}, + {"/a/b/{x...}", "/a/b/c", "/a/b/"}, + {"/a/b/{x...}", "/a/b/c/d", "/a/b/"}, + {"/a/b/{x...}", "/a/b/c/d/", "/a/b/"}, + {"/a/{z}/{m...}", "/{z}/a/", "/a/z/"}, + {"/{z}/a/", "/a/{z}/{m...}", "/z/a/"}, + {"/{z}/{$}", "/a/", "/z/"}, + {"/a/", "/{z}/{$}", "/a/x"}, + {"/{z}/{$}", "/a/{x...}", "/z/"}, + {"/a/{foo...}", "/{z}/{$}", "/a/foo"}, + {"/a/{z}/{$}", "/{z}/a/", "/a/z/"}, + {"/{z}/a/", "/a/{z}/{$}", "/z/a/x"}, + {"/a/{x}/b/{y...}", "/{x}/c/{y...}", "/a/x/b/"}, + {"/{x}/c/{y...}", "/a/{x}/b/{y...}", "/x/c/"}, + {"/a/{c}/b/", "/{x}/c/{y...}", "/a/cx/b/"}, + {"/{x}/c/{y...}", "/a/{c}/b/", "/x/c/"}, + {"/a/{x}/b/{$}", "/{x}/c/{y...}", "/a/x/b/"}, + {"/{x}/c/{y...}", "/a/{x}/b/{$}", "/x/c/"}, + } { + pat1 := mustParsePattern(t, test.p1) + pat2 := mustParsePattern(t, test.p2) + rel := pat1.comparePaths(pat2) + if rel != overlaps && rel != moreGeneral { + t.Fatalf("%s vs. %s are %s, need overlaps or moreGeneral", pat1, pat2, rel) + } + got := differencePath(pat1, pat2) + if got != test.want { + t.Errorf("%s vs. %s: got %q, want %q", test.p1, test.p2, got, test.want) + } + } +} diff --git a/server.go b/server.go index 629d8d3c..b9f4a6b4 100644 --- a/server.go +++ b/server.go @@ -2655,8 +2655,9 @@ func (mux *ServeMux) registerErr(patstr string, handler Handler) error { // Check for conflict. if err := mux.index.possiblyConflictingPatterns(pat, func(pat2 *pattern) error { if pat.conflictsWith(pat2) { - return fmt.Errorf("pattern %q (registered at %s) conflicts with pattern %q (registered at %s)", - pat, pat.loc, pat2, pat2.loc) + d := describeConflict(pat, pat2) + return fmt.Errorf("pattern %q (registered at %s) conflicts with pattern %q (registered at %s):\n%s", + pat, pat.loc, pat2, pat2.loc, d) } return nil }); err != nil { From b5201014f423e6979a00fe34fd3fcde281032fd5 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Tue, 19 Sep 2023 09:17:42 -0400 Subject: [PATCH 44/93] net/http: show offset in pattern parsing error Track the offset in the pattern string being parsed so we can show it in the error message. Change-Id: I495b99378d866f359f45974ffc33587e2c1e366d Reviewed-on: https://go-review.googlesource.com/c/go/+/529123 Run-TryBot: Jonathan Amsterdam Reviewed-by: Damien Neil TryBot-Result: Gopher Robot --- pattern.go | 20 ++++++++++++++++---- pattern_test.go | 34 ++++++++++++++++++---------------- server_test.go | 2 +- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/pattern.go b/pattern.go index eca17918..2993aecc 100644 --- a/pattern.go +++ b/pattern.go @@ -80,31 +80,42 @@ type segment struct { // The "{$}" and "{name...}" wildcard must occur at the end of PATH. // PATH may end with a '/'. // Wildcard names in a path must be distinct. -func parsePattern(s string) (*pattern, error) { +func parsePattern(s string) (_ *pattern, err error) { if len(s) == 0 { return nil, errors.New("empty pattern") } - // TODO(jba): record the rune offset in s to provide more information in errors. + off := 0 // offset into string + defer func() { + if err != nil { + err = fmt.Errorf("at offset %d: %w", off, err) + } + }() + method, rest, found := strings.Cut(s, " ") if !found { rest = method method = "" } if method != "" && !validMethod(method) { - return nil, fmt.Errorf("net/http: invalid method %q", method) + return nil, fmt.Errorf("invalid method %q", method) } p := &pattern{str: s, method: method} + if found { + off = len(method) + 1 + } i := strings.IndexByte(rest, '/') if i < 0 { return nil, errors.New("host/path missing /") } p.host = rest[:i] rest = rest[i:] - if strings.IndexByte(p.host, '{') >= 0 { + if j := strings.IndexByte(p.host, '{'); j >= 0 { + off += j return nil, errors.New("host contains '{' (missing initial '/'?)") } // At this point, rest is the path. + off += i // An unclean path with a method that is not CONNECT can never match, // because paths are cleaned before matching. @@ -116,6 +127,7 @@ func parsePattern(s string) (*pattern, error) { for len(rest) > 0 { // Invariant: rest[0] == '/'. rest = rest[1:] + off = len(s) - len(rest) if len(rest) == 0 { // Trailing slash. p.segments = append(p.segments, segment{wild: true, multi: true}) diff --git a/pattern_test.go b/pattern_test.go index f67a2b51..e71cba86 100644 --- a/pattern_test.go +++ b/pattern_test.go @@ -108,22 +108,24 @@ func TestParsePatternError(t *testing.T) { contains string }{ {"", "empty pattern"}, - {"A=B /", "invalid method"}, - {" ", "missing /"}, - {"/{w}x", "bad wildcard segment"}, - {"/x{w}", "bad wildcard segment"}, - {"/{wx", "bad wildcard segment"}, - {"/{a$}", "bad wildcard name"}, - {"/{}", "empty wildcard"}, - {"/{...}", "empty wildcard"}, - {"/{$...}", "bad wildcard"}, - {"/{$}/", "{$} not at end"}, - {"/{$}/x", "{$} not at end"}, - {"/{a...}/", "not at end"}, - {"/{a...}/x", "not at end"}, - {"{a}/b", "missing initial '/'"}, - {"/a/{x}/b/{x...}", "duplicate wildcard name"}, - {"GET //", "unclean path"}, + {"A=B /", "at offset 0: invalid method"}, + {" ", "at offset 1: host/path missing /"}, + {"/{w}x", "at offset 1: bad wildcard segment"}, + {"/x{w}", "at offset 1: bad wildcard segment"}, + {"/{wx", "at offset 1: bad wildcard segment"}, + {"/{a$}", "at offset 1: bad wildcard name"}, + {"/{}", "at offset 1: empty wildcard"}, + {"POST a.com/x/{}/y", "at offset 13: empty wildcard"}, + {"/{...}", "at offset 1: empty wildcard"}, + {"/{$...}", "at offset 1: bad wildcard"}, + {"/{$}/", "at offset 1: {$} not at end"}, + {"/{$}/x", "at offset 1: {$} not at end"}, + {"/abc/{$}/x", "at offset 5: {$} not at end"}, + {"/{a...}/", "at offset 1: {...} wildcard not at end"}, + {"/{a...}/x", "at offset 1: {...} wildcard not at end"}, + {"{a}/b", "at offset 0: host contains '{' (missing initial '/'?)"}, + {"/a/{x}/b/{x...}", "at offset 9: duplicate wildcard name"}, + {"GET //", "at offset 4: non-CONNECT pattern with unclean path"}, } { _, err := parsePattern(test.in) if err == nil || !strings.Contains(err.Error(), test.contains) { diff --git a/server_test.go b/server_test.go index 0c361c7d..a96d8765 100644 --- a/server_test.go +++ b/server_test.go @@ -131,7 +131,7 @@ func TestRegisterErr(t *testing.T) { {"", h, "invalid pattern"}, {"/", nil, "nil handler"}, {"/", HandlerFunc(nil), "nil handler"}, - {"/{x", h, `parsing "/\{x": bad wildcard segment`}, + {"/{x", h, `parsing "/\{x": at offset 1: bad wildcard segment`}, {"/a", h, `conflicts with pattern.* \(registered at .*/server_test.go:\d+`}, } { t.Run(fmt.Sprintf("%s:%#v", test.pattern, test.handler), func(t *testing.T) { From e0f72cca3225e3243c8661becd8bb0444aa1eb24 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Tue, 19 Sep 2023 17:03:45 -0400 Subject: [PATCH 45/93] net/http: test index exhaustively Replace the fuzz test with one that enumerates all relevant patterns up to a certain length. For conflict detection, we don't need to check every possible method, host and segment, only a few that cover all the possibilities. There are only 2400 distinct patterns in the corpus we generate, and the test generates, indexes and compares them all in about a quarter of a second. Change-Id: I9fde88e87cec07b1b244306119e4e71f7205bb77 Reviewed-on: https://go-review.googlesource.com/c/go/+/529556 Run-TryBot: Jonathan Amsterdam TryBot-Result: Gopher Robot Reviewed-by: Damien Neil --- routing_index_test.go | 240 +++++++++-------------- testdata/fuzz/FuzzIndex/48161038f0c8b2da | 2 - testdata/fuzz/FuzzIndex/716514f590ce7ab3 | 2 - 3 files changed, 93 insertions(+), 151 deletions(-) delete mode 100644 testdata/fuzz/FuzzIndex/48161038f0c8b2da delete mode 100644 testdata/fuzz/FuzzIndex/716514f590ce7ab3 diff --git a/routing_index_test.go b/routing_index_test.go index 7030fc8a..404574a6 100644 --- a/routing_index_test.go +++ b/routing_index_test.go @@ -5,7 +5,6 @@ package http import ( - "bytes" "fmt" "slices" "sort" @@ -14,66 +13,19 @@ import ( ) func TestIndex(t *testing.T) { - pats := []string{"HEAD /", "/a"} - - var patterns []*pattern - var idx routingIndex - for _, p := range pats { - pat := mustParsePattern(t, p) - patterns = append(patterns, pat) - idx.addPattern(pat) - } - - compare := func(pat *pattern) { - t.Helper() - got := indexConflicts(pat, &idx) - want := trueConflicts(pat, patterns) - if !slices.Equal(got, want) { - t.Errorf("%q:\ngot %q\nwant %q", pat, got, want) - } - } - - compare(mustParsePattern(t, "GET /foo")) - compare(mustParsePattern(t, "GET /{x}")) -} - -// This test works by comparing possiblyConflictingPatterns with -// an exhaustive loop through all patterns. -func FuzzIndex(f *testing.F) { - inits := []string{"/a", "/a/b", "/{x0}", "/{x0}/b", "/a/{x0}", "/a/{$}", "/a/b/{$}", - "/a/", "/a/b/", "/{x}/b/c/{$}", "GET /{x0}/", "HEAD /a"} - - var patterns []*pattern + // Generate every kind of pattern up to some number of segments, + // and compare conflicts found during indexing with those found + // by exhaustive comparison. + patterns := generatePatterns() var idx routingIndex - - // compare takes a fatalf function because fuzzing doesn't like - // it when the fuzz function calls f.Fatalf. - compare := func(pat *pattern, fatalf func(string, ...any)) { + for i, pat := range patterns { got := indexConflicts(pat, &idx) - want := trueConflicts(pat, patterns) + want := trueConflicts(pat, patterns[:i]) if !slices.Equal(got, want) { - fatalf("%q:\ngot %q\nwant %q", pat, got, want) - } - } - - for _, p := range inits { - pat, err := parsePattern(p) - if err != nil { - f.Fatal(err) + t.Fatalf("%q:\ngot %q\nwant %q", pat, got, want) } - compare(pat, f.Fatalf) - patterns = append(patterns, pat) idx.addPattern(pat) - f.Add(bytesFromPattern(pat)) } - - f.Fuzz(func(t *testing.T, pb []byte) { - pat := bytesToPattern(pb) - if pat == nil { - return - } - compare(pat, t.Fatalf) - }) } func trueConflicts(pat *pattern, pats []*pattern) []string { @@ -99,109 +51,103 @@ func indexConflicts(pat *pattern, idx *routingIndex) []string { return slices.Compact(s) } -// TODO: incorporate host and method; make encoding denser. -func bytesToPattern(bs []byte) *pattern { - if len(bs) == 0 { - return nil - } - var sb strings.Builder - wc := 0 - for _, b := range bs[:len(bs)-1] { - sb.WriteByte('/') - switch b & 0x3 { - case 0: - fmt.Fprintf(&sb, "{x%d}", wc) +// generatePatterns generates all possible patterns using a representative +// sample of parts. +func generatePatterns() []*pattern { + var pats []*pattern + + collect := func(s string) { + // Replace duplicate wildcards with unique ones. + var b strings.Builder + wc := 0 + for { + i := strings.Index(s, "{x}") + if i < 0 { + b.WriteString(s) + break + } + b.WriteString(s[:i]) + fmt.Fprintf(&b, "{x%d}", wc) wc++ - case 1: - sb.WriteString("a") - case 2: - sb.WriteString("b") - case 3: - sb.WriteString("c") + s = s[i+3:] } + pat, err := parsePattern(b.String()) + if err != nil { + panic(err) + } + pats = append(pats, pat) } - sb.WriteByte('/') - switch bs[len(bs)-1] & 0x7 { - case 0: - fmt.Fprintf(&sb, "{x%d}", wc) - case 1: - sb.WriteString("a") - case 2: - sb.WriteString("b") - case 3: - sb.WriteString("c") - case 4, 5: - fmt.Fprintf(&sb, "{x%d...}", wc) - default: - sb.WriteString("{$}") - } - pat, err := parsePattern(sb.String()) - if err != nil { - panic(err) + + var ( + methods = []string{"", "GET ", "HEAD ", "POST "} + hosts = []string{"", "h1", "h2"} + segs = []string{"/a", "/b", "/{x}"} + finalSegs = []string{"/a", "/b", "/{f}", "/{m...}", "/{$}"} + ) + + g := genConcat( + genChoice(methods), + genChoice(hosts), + genStar(3, genChoice(segs)), + genChoice(finalSegs)) + g(collect) + return pats +} + +// A generator is a function that calls its argument with the strings that it +// generates. +type generator func(collect func(string)) + +// genConst generates a single constant string. +func genConst(s string) generator { + return func(collect func(string)) { + collect(s) } - return pat } -func bytesFromPattern(p *pattern) []byte { - var bs []byte - for _, s := range p.segments { - var b byte - switch { - case s.multi: - b = 4 - case s.wild: - b = 0 - case s.s == "/": - b = 7 - case s.s == "a": - b = 1 - case s.s == "b": - b = 2 - case s.s == "c": - b = 3 - default: - panic("bad pattern") +// genChoice generates all the strings in its argument. +func genChoice(choices []string) generator { + return func(collect func(string)) { + for _, c := range choices { + collect(c) } - bs = append(bs, b) } - return bs } -func TestBytesPattern(t *testing.T) { - tests := []struct { - bs []byte - pat string - }{ - {[]byte{0, 1, 2, 3}, "/{x0}/a/b/c"}, - {[]byte{16, 17, 18, 19}, "/{x0}/a/b/c"}, - {[]byte{4, 4}, "/{x0}/{x1...}"}, - {[]byte{6, 7}, "/b/{$}"}, +// genConcat2 generates the cross product of the strings of g1 concatenated +// with those of g2. +func genConcat2(g1, g2 generator) generator { + return func(collect func(string)) { + g1(func(s1 string) { + g2(func(s2 string) { + collect(s1 + s2) + }) + }) } - t.Run("To", func(t *testing.T) { - for _, test := range tests { - p := bytesToPattern(test.bs) - got := p.String() - if got != test.pat { - t.Errorf("%v: got %q, want %q", test.bs, got, test.pat) - } - } - }) - t.Run("From", func(t *testing.T) { - for _, test := range tests { - p, err := parsePattern(test.pat) - if err != nil { - t.Fatal(err) - } - got := bytesFromPattern(p) - var want []byte - for _, b := range test.bs[:len(test.bs)-1] { - want = append(want, b%4) +} - } - want = append(want, test.bs[len(test.bs)-1]%8) - if !bytes.Equal(got, want) { - t.Errorf("%s: got %v, want %v", test.pat, got, want) - } +// genConcat generalizes genConcat2 to any number of generators. +func genConcat(gs ...generator) generator { + if len(gs) == 0 { + return genConst("") + } + return genConcat2(gs[0], genConcat(gs[1:]...)) +} + +// genRepeat generates strings of exactly n copies of g's strings. +func genRepeat(n int, g generator) generator { + if n == 0 { + return genConst("") + } + return genConcat(g, genRepeat(n-1, g)) +} + +// genStar (named after the Kleene star) generates 0, 1, 2, ..., max +// copies of the strings of g. +func genStar(max int, g generator) generator { + return func(collect func(string)) { + for i := 0; i <= max; i++ { + genRepeat(i, g)(collect) } - }) + } } diff --git a/testdata/fuzz/FuzzIndex/48161038f0c8b2da b/testdata/fuzz/FuzzIndex/48161038f0c8b2da deleted file mode 100644 index 06a7336a..00000000 --- a/testdata/fuzz/FuzzIndex/48161038f0c8b2da +++ /dev/null @@ -1,2 +0,0 @@ -go test fuzz v1 -[]byte("101$") diff --git a/testdata/fuzz/FuzzIndex/716514f590ce7ab3 b/testdata/fuzz/FuzzIndex/716514f590ce7ab3 deleted file mode 100644 index 520bff17..00000000 --- a/testdata/fuzz/FuzzIndex/716514f590ce7ab3 +++ /dev/null @@ -1,2 +0,0 @@ -go test fuzz v1 -[]byte("1010") From 6207ea6e48564f6ba62ae0854cf2599b66d473b2 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Tue, 19 Sep 2023 17:23:58 -0400 Subject: [PATCH 46/93] net/http: add a benchmark for multi indexing We don't index multis, so a corpus full of them will take quadratic time to check for conflicts. How slow is that going to be in practice? This benchmark indexes and checks a thousand multi patterns, all disjoint. It runs in about 35ms. Change-Id: Id27940ab19ad003627bd5c43c53466e01456b796 Reviewed-on: https://go-review.googlesource.com/c/go/+/529477 Run-TryBot: Jonathan Amsterdam TryBot-Result: Gopher Robot Reviewed-by: Damien Neil --- pattern_test.go | 6 +++--- routing_index_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/pattern_test.go b/pattern_test.go index e71cba86..abda4d87 100644 --- a/pattern_test.go +++ b/pattern_test.go @@ -159,11 +159,11 @@ func TestIsValidHTTPToken(t *testing.T) { } } -func mustParsePattern(t *testing.T, s string) *pattern { - t.Helper() +func mustParsePattern(tb testing.TB, s string) *pattern { + tb.Helper() p, err := parsePattern(s) if err != nil { - t.Fatal(err) + tb.Fatal(err) } return p } diff --git a/routing_index_test.go b/routing_index_test.go index 404574a6..1ffb9272 100644 --- a/routing_index_test.go +++ b/routing_index_test.go @@ -151,3 +151,29 @@ func genStar(max int, g generator) generator { } } } + +func BenchmarkMultiConflicts(b *testing.B) { + // How fast is indexing if the corpus is all multis? + const nMultis = 1000 + var pats []*pattern + for i := 0; i < nMultis; i++ { + pats = append(pats, mustParsePattern(b, fmt.Sprintf("/a/b/{x}/d%d/", i))) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + var idx routingIndex + for _, p := range pats { + got := indexConflicts(p, &idx) + if len(got) != 0 { + b.Fatalf("got %d conflicts, want 0", len(got)) + } + idx.addPattern(p) + } + if i == 0 { + // Confirm that all the multis ended up where they belong. + if g, w := len(idx.multis), nMultis; g != w { + b.Fatalf("got %d multis, want %d", g, w) + } + } + } +} From f77dc2ccdb236f38e1ce765c31dfa7786febb21c Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Wed, 20 Sep 2023 10:44:06 -0400 Subject: [PATCH 47/93] net/http: eliminate a goroutine leak in (*persistConn.addTLS) In case of a handshake timeout, the goroutine running addTLS closes the underlying connection, which should unblock the call to tlsConn.HandshakeContext. However, it didn't then wait for HandshakeContext to actually return. I thought this might have something to do with #57602, but as far as I can tell it does not. Still, it seems best to avoid the leak: if tracing is enabled we emit a TLSHandshakeDone event, and it seems misleading to produce that event when the handshake is still in progress. For #57602. Change-Id: Ibfc0cf4ef8df2ccf11d8897f23d7d79ee482d5fb Reviewed-on: https://go-review.googlesource.com/c/go/+/529755 LUCI-TryBot-Result: Go LUCI Reviewed-by: Damien Neil Auto-Submit: Bryan Mills Commit-Queue: Bryan Mills --- transport.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/transport.go b/transport.go index ac7477ea..1cf41a54 100644 --- a/transport.go +++ b/transport.go @@ -1577,6 +1577,11 @@ func (pconn *persistConn) addTLS(ctx context.Context, name string, trace *httptr }() if err := <-errc; err != nil { plainConn.Close() + if err == (tlsHandshakeTimeoutError{}) { + // Now that we have closed the connection, + // wait for the call to HandshakeContext to return. + <-errc + } if trace != nil && trace.TLSHandshakeDone != nil { trace.TLSHandshakeDone(tls.ConnectionState{}, err) } From 5211518264350a6acc5a5c7dedf6387d23483a0d Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Wed, 20 Sep 2023 10:48:20 -0400 Subject: [PATCH 48/93] net/http: eliminate more clientServerTest leaks in tests that use runTimeSensitiveTest Change-Id: I77684a095af03d5c4e50da8e7af210b10639ff23 Reviewed-on: https://go-review.googlesource.com/c/go/+/529756 LUCI-TryBot-Result: Go LUCI Auto-Submit: Bryan Mills Reviewed-by: Damien Neil --- serve_test.go | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/serve_test.go b/serve_test.go index 8fa40e61..93503d65 100644 --- a/serve_test.go +++ b/serve_test.go @@ -663,12 +663,14 @@ func testServerTimeouts(t *testing.T, mode testMode) { func testServerTimeoutsWithTimeout(t *testing.T, timeout time.Duration, mode testMode) error { var reqNum atomic.Int32 - ts := newClientServerTest(t, mode, HandlerFunc(func(res ResponseWriter, req *Request) { + cst := newClientServerTest(t, mode, HandlerFunc(func(res ResponseWriter, req *Request) { fmt.Fprintf(res, "req=%d", reqNum.Add(1)) }), func(ts *httptest.Server) { ts.Config.ReadTimeout = timeout ts.Config.WriteTimeout = timeout - }).ts + }) + defer cst.close() + ts := cst.ts // Hit the HTTP server successfully. c := ts.Client() @@ -865,7 +867,7 @@ func TestWriteDeadlineEnforcedPerStream(t *testing.T) { func testWriteDeadlineEnforcedPerStream(t *testing.T, mode testMode, timeout time.Duration) error { firstRequest := make(chan bool, 1) - ts := newClientServerTest(t, mode, HandlerFunc(func(res ResponseWriter, req *Request) { + cst := newClientServerTest(t, mode, HandlerFunc(func(res ResponseWriter, req *Request) { select { case firstRequest <- true: // first request succeeds @@ -875,7 +877,9 @@ func testWriteDeadlineEnforcedPerStream(t *testing.T, mode testMode, timeout tim } }), func(ts *httptest.Server) { ts.Config.WriteTimeout = timeout / 2 - }).ts + }) + defer cst.close() + ts := cst.ts c := ts.Client() @@ -923,7 +927,7 @@ func TestNoWriteDeadline(t *testing.T) { func testNoWriteDeadline(t *testing.T, mode testMode, timeout time.Duration) error { firstRequest := make(chan bool, 1) - ts := newClientServerTest(t, mode, HandlerFunc(func(res ResponseWriter, req *Request) { + cst := newClientServerTest(t, mode, HandlerFunc(func(res ResponseWriter, req *Request) { select { case firstRequest <- true: // first request succeeds @@ -931,7 +935,9 @@ func testNoWriteDeadline(t *testing.T, mode testMode, timeout time.Duration) err // second request times out time.Sleep(timeout) } - })).ts + })) + defer cst.close() + ts := cst.ts c := ts.Client() @@ -5399,13 +5405,15 @@ func testServerIdleTimeout(t *testing.T, mode testMode) { 1 * time.Second, 10 * time.Second, }, func(t *testing.T, readHeaderTimeout time.Duration) error { - ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { io.Copy(io.Discard, r.Body) io.WriteString(w, r.RemoteAddr) }), func(ts *httptest.Server) { ts.Config.ReadHeaderTimeout = readHeaderTimeout ts.Config.IdleTimeout = 2 * readHeaderTimeout - }).ts + }) + defer cst.close() + ts := cst.ts t.Logf("ReadHeaderTimeout = %v", ts.Config.ReadHeaderTimeout) t.Logf("IdleTimeout = %v", ts.Config.IdleTimeout) c := ts.Client() @@ -5719,7 +5727,7 @@ func testServerCancelsReadTimeoutWhenIdle(t *testing.T, mode testMode) { time.Second, 2 * time.Second, }, func(t *testing.T, timeout time.Duration) error { - ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { select { case <-time.After(2 * timeout): fmt.Fprint(w, "ok") @@ -5728,7 +5736,9 @@ func testServerCancelsReadTimeoutWhenIdle(t *testing.T, mode testMode) { } }), func(ts *httptest.Server) { ts.Config.ReadTimeout = timeout - }).ts + }) + defer cst.close() + ts := cst.ts c := ts.Client() @@ -5762,10 +5772,12 @@ func testServerCancelsReadHeaderTimeoutWhenIdle(t *testing.T, mode testMode) { time.Second, 2 * time.Second, }, func(t *testing.T, timeout time.Duration) error { - ts := newClientServerTest(t, mode, serve(200), func(ts *httptest.Server) { + cst := newClientServerTest(t, mode, serve(200), func(ts *httptest.Server) { ts.Config.ReadHeaderTimeout = timeout ts.Config.IdleTimeout = 0 // disable idle timeout - }).ts + }) + defer cst.close() + ts := cst.ts // rather than using an http.Client, create a single connection, so that // we can ensure this connection is not closed. From 1f52f9fe44254e80747c279509dc1f8a4778c5c8 Mon Sep 17 00:00:00 2001 From: Oleksandr Redko Date: Wed, 6 Sep 2023 15:14:28 +0300 Subject: [PATCH 49/93] all: simplify bool conditions Change-Id: Id2079f7012392dea8dfe2386bb9fb1ea3f487a4a Reviewed-on: https://go-review.googlesource.com/c/go/+/526015 Reviewed-by: Matthew Dempsky Auto-Submit: Ian Lance Taylor Reviewed-by: Ian Lance Taylor LUCI-TryBot-Result: Go LUCI Reviewed-by: qiulaidongfeng <2645477756@qq.com> --- response_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/response_test.go b/response_test.go index ddd31808..f3425c3c 100644 --- a/response_test.go +++ b/response_test.go @@ -849,7 +849,7 @@ func TestReadResponseErrors(t *testing.T) { type testCase struct { name string // optional, defaults to in in string - wantErr any // nil, err value, or string substring + wantErr any // nil, err value, bool value, or string substring } status := func(s string, wantErr any) testCase { From bac15b086fcf7aac7933e1c3f16dfb436bcad57b Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Fri, 22 Sep 2023 15:57:46 -0400 Subject: [PATCH 50/93] net/http: unescape paths and patterns by segment When parsing patterns and matching, split the path into segments at slashes, then unescape each segment. This behaves as most people would expect: - The pattern "/%61" matches the paths "/a" and "/%61". - The pattern "/%7B" matches the path "/{". (If we did not unescape patterns, there would be no way to write that pattern: because "/{" is a parse error because it is an invalid wildcard.) - The pattern "/user/{u}" matches "/user/john%2Fdoe" with u set to "john/doe". - The unexpected redirections of #21955 will not occur. A later CL will restore the old behavior behind a GODEBUG setting. Updates #61410. Fixes #21955. Change-Id: I99025e149021fc94bf87d351699270460db532d9 Reviewed-on: https://go-review.googlesource.com/c/go/+/530575 Run-TryBot: Jonathan Amsterdam TryBot-Result: Gopher Robot Reviewed-by: Damien Neil --- pattern.go | 12 +++++++ pattern_test.go | 6 ++++ request_test.go | 84 +++++++++++++++++++++++++++++++++++--------- routing_tree.go | 19 +++------- routing_tree_test.go | 2 ++ server.go | 12 +++---- 6 files changed, 97 insertions(+), 38 deletions(-) diff --git a/pattern.go b/pattern.go index 2993aecc..0c8644d9 100644 --- a/pattern.go +++ b/pattern.go @@ -9,6 +9,7 @@ package http import ( "errors" "fmt" + "net/url" "strings" "unicode" ) @@ -141,6 +142,7 @@ func parsePattern(s string) (_ *pattern, err error) { seg, rest = rest[:i], rest[i:] if i := strings.IndexByte(seg, '{'); i < 0 { // Literal. + seg = pathUnescape(seg) p.segments = append(p.segments, segment{s: seg}) } else { // Wildcard. @@ -178,6 +180,7 @@ func parsePattern(s string) (_ *pattern, err error) { return p, nil } +// TODO(jba): remove this; it is unused. func isValidHTTPToken(s string) bool { if s == "" { return false @@ -204,6 +207,15 @@ func isValidWildcardName(s string) bool { return true } +func pathUnescape(path string) string { + u, err := url.PathUnescape(path) + if err != nil { + // Invalidly escaped path; use the original + return path + } + return u +} + // relationship is a relationship between two patterns, p1 and p2. type relationship string diff --git a/pattern_test.go b/pattern_test.go index abda4d87..b219648f 100644 --- a/pattern_test.go +++ b/pattern_test.go @@ -94,6 +94,10 @@ func TestParsePattern(t *testing.T) { "a.com/foo//", pattern{host: "a.com", segments: []segment{lit("foo"), lit(""), multi("")}}, }, + { + "/%61%62/%7b/%", + pattern{segments: []segment{lit("ab"), lit("{"), lit("%")}}, + }, } { got := mustParsePattern(t, test.in) if !got.equal(&test.want) { @@ -113,6 +117,8 @@ func TestParsePatternError(t *testing.T) { {"/{w}x", "at offset 1: bad wildcard segment"}, {"/x{w}", "at offset 1: bad wildcard segment"}, {"/{wx", "at offset 1: bad wildcard segment"}, + {"/a/{/}/c", "at offset 3: bad wildcard segment"}, + {"/a/{%61}/c", "at offset 3: bad wildcard name"}, // wildcard names aren't unescaped {"/{a$}", "at offset 1: bad wildcard name"}, {"/{}", "at offset 1: empty wildcard"}, {"POST a.com/x/{}/y", "at offset 13: empty wildcard"}, diff --git a/request_test.go b/request_test.go index 18034ce1..835db91a 100644 --- a/request_test.go +++ b/request_test.go @@ -1445,23 +1445,22 @@ func TestPathValue(t *testing.T) { "d": "", }, }, - // TODO(jba): uncomment these tests when we implement path escaping (forthcoming). - // { - // "/names/{name}/{other...}", - // "/names/" + url.PathEscape("/john") + "/address", - // map[string]string{ - // "name": "/john", - // "other": "address", - // }, - // }, - // { - // "/names/{name}/{other...}", - // "/names/" + url.PathEscape("john/doe") + "/address", - // map[string]string{ - // "name": "john/doe", - // "other": "address", - // }, - // }, + { + "/names/{name}/{other...}", + "/names/%2fjohn/address", + map[string]string{ + "name": "/john", + "other": "address", + }, + }, + { + "/names/{name}/{other...}", + "/names/john%2Fdoe/there/is%2F/more", + map[string]string{ + "name": "john/doe", + "other": "there/is//more", + }, + }, } { mux := NewServeMux() mux.HandleFunc(test.pattern, func(w ResponseWriter, r *Request) { @@ -1555,3 +1554,54 @@ func TestStatus(t *testing.T) { } } } + +func TestEscapedPathsAndPatterns(t *testing.T) { + matches := []struct { + pattern string + paths []string + }{ + { + "/a", // this pattern matches a path that unescapes to "/a" + []string{"/a", "/%61"}, + }, + { + "/%62", // patterns are unescaped by segment; matches paths that unescape to "/b" + []string{"/b", "/%62"}, + }, + { + "/%7B/%7D", // the only way to write a pattern that matches '{' or '}' + []string{"/{/}", "/%7b/}", "/{/%7d", "/%7B/%7D"}, + }, + { + "/%x", // patterns that do not unescape are left unchanged + []string{"/%25x"}, + }, + } + + mux := NewServeMux() + var gotPattern string + for _, m := range matches { + mux.HandleFunc(m.pattern, func(w ResponseWriter, r *Request) { + gotPattern = m.pattern + }) + } + + server := httptest.NewServer(mux) + defer server.Close() + + for _, m := range matches { + for _, p := range m.paths { + res, err := Get(server.URL + p) + if err != nil { + t.Fatal(err) + } + if res.StatusCode != 200 { + t.Errorf("%s: got code %d, want 200", p, res.StatusCode) + continue + } + if g, w := gotPattern, m.pattern; g != w { + t.Errorf("%s: pattern: got %q, want %q", p, g, w) + } + } + } +} diff --git a/routing_tree.go b/routing_tree.go index 46287174..8812ed04 100644 --- a/routing_tree.go +++ b/routing_tree.go @@ -19,7 +19,6 @@ package http import ( - "net/url" "strings" ) @@ -180,7 +179,7 @@ func (n *routingNode) matchPath(path string, matches []string) (*routingNode, [] // We skip this step if the segment is a trailing slash, because single wildcards // don't match trailing slashes. if seg != "/" { - if n, m := n.emptyChild.matchPath(rest, append(matches, matchValue(seg))); n != nil { + if n, m := n.emptyChild.matchPath(rest, append(matches, seg)); n != nil { return n, m } } @@ -190,25 +189,17 @@ func (n *routingNode) matchPath(path string, matches []string) (*routingNode, [] // Don't record a match for a nameless wildcard (which arises from a // trailing slash in the pattern). if c.pattern.lastSegment().s != "" { - matches = append(matches, matchValue(path[1:])) // remove initial slash + matches = append(matches, pathUnescape(path[1:])) // remove initial slash } return c, matches } return nil, nil } -func matchValue(path string) string { - m, err := url.PathUnescape(path) - if err != nil { - // Path is not properly escaped, so use the original. - return path - } - return m -} - // firstSegment splits path into its first segment, and the rest. // The path must begin with "/". // If path consists of only a slash, firstSegment returns ("/", ""). +// The segment is returned unescaped, if possible. func firstSegment(path string) (seg, rest string) { if path == "/" { return "/", "" @@ -216,9 +207,9 @@ func firstSegment(path string) (seg, rest string) { path = path[1:] // drop initial slash i := strings.IndexByte(path, '/') if i < 0 { - return path, "" + i = len(path) } - return path[:i], path[i:] + return pathUnescape(path[:i]), path[i:] } // matchingMethods adds to methodSet all the methods that would result in a diff --git a/routing_tree_test.go b/routing_tree_test.go index 149349f3..2aac8b6c 100644 --- a/routing_tree_test.go +++ b/routing_tree_test.go @@ -22,6 +22,8 @@ func TestRoutingFirstSegment(t *testing.T) { {"/a/b/c", []string{"a", "b", "c"}}, {"/a/b/", []string{"a", "b", "/"}}, {"/", []string{"/"}}, + {"/a/%62/c", []string{"a", "b", "c"}}, + {"/a%2Fb%2fc", []string{"a/b/c"}}, } { var got []string rest := test.in diff --git a/server.go b/server.go index b9f4a6b4..ee02d776 100644 --- a/server.go +++ b/server.go @@ -2422,10 +2422,9 @@ func (mux *ServeMux) Handler(r *Request) (h Handler, pattern string) { // after the redirect. func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string, _ *pattern, matches []string) { var n *routingNode - // TODO(jba): use escaped path. This is an independent change that is also part - // of proposal https://go.dev/issue/61410. - path := r.URL.Path host := r.URL.Host + escapedPath := r.URL.EscapedPath() + path := escapedPath // CONNECT requests are not canonicalized. if r.Method == "CONNECT" { // If r.URL.Path is /tree and its handler is not registered, @@ -2451,7 +2450,7 @@ func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string, _ *patte if u != nil { return RedirectHandler(u.String(), StatusMovedPermanently), u.Path, nil, nil } - if path != r.URL.Path { + if path != escapedPath { // Redirect to cleaned path. patStr := "" if n != nil { @@ -2478,8 +2477,7 @@ func (mux *ServeMux) findHandler(r *Request) (h Handler, patStr string, _ *patte } // matchOrRedirect looks up a node in the tree that matches the host, method and path. -// The path is known to be in canonical form, except for CONNECT methods. - +// // If the url argument is non-nil, handler also deals with trailing-slash // redirection: when a path doesn't match exactly, the match is tried again // after appending "/" to the path. If that second match succeeds, the last @@ -2496,7 +2494,7 @@ func (mux *ServeMux) matchOrRedirect(host, method, path string, u *url.URL) (_ * path += "/" n2, _ := mux.tree.match(host, method, path) if exactMatch(n2, path) { - return nil, nil, &url.URL{Path: path, RawQuery: u.RawQuery} + return nil, nil, &url.URL{Path: cleanPath(u.Path) + "/", RawQuery: u.RawQuery} } } return n, matches, nil From 49d52c0cd277a9ca3da67fa07f469ea3284c9af3 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Fri, 22 Sep 2023 16:11:36 -0400 Subject: [PATCH 51/93] net/http: remove unused function Change-Id: I4364d94663282249e632d12026a810147844ad2e Reviewed-on: https://go-review.googlesource.com/c/go/+/530615 Run-TryBot: Jonathan Amsterdam TryBot-Result: Gopher Robot Reviewed-by: Damien Neil --- pattern.go | 14 -------------- pattern_test.go | 20 -------------------- 2 files changed, 34 deletions(-) diff --git a/pattern.go b/pattern.go index 0c8644d9..f6af19b0 100644 --- a/pattern.go +++ b/pattern.go @@ -180,20 +180,6 @@ func parsePattern(s string) (_ *pattern, err error) { return p, nil } -// TODO(jba): remove this; it is unused. -func isValidHTTPToken(s string) bool { - if s == "" { - return false - } - // See https://www.rfc-editor.org/rfc/rfc9110#section-5.6.2. - for _, r := range s { - if !unicode.IsLetter(r) && !unicode.IsDigit(r) && !strings.ContainsRune("!#$%&'*+.^_`|~-", r) { - return false - } - } - return true -} - func isValidWildcardName(s string) bool { if s == "" { return false diff --git a/pattern_test.go b/pattern_test.go index b219648f..f0c84d24 100644 --- a/pattern_test.go +++ b/pattern_test.go @@ -145,26 +145,6 @@ func (p1 *pattern) equal(p2 *pattern) bool { slices.Equal(p1.segments, p2.segments) } -func TestIsValidHTTPToken(t *testing.T) { - for _, test := range []struct { - in string - want bool - }{ - {"", false}, - {"GET", true}, - {"get", true}, - {"white space", false}, - {"#!~", true}, - {"a-b1_2", true}, - {"notok)", false}, - } { - got := isValidHTTPToken(test.in) - if g, w := got, test.want; g != w { - t.Errorf("%q: got %t, want %t", test.in, g, w) - } - } -} - func mustParsePattern(tb testing.TB, s string) *pattern { tb.Helper() p, err := parsePattern(s) From 495a3332a03aaf0bcbe1de745b2029ca69335096 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Tue, 26 Sep 2023 12:35:57 -0400 Subject: [PATCH 52/93] net/http: add extra synchronization for a Logf call in TestTransportAndServerSharedBodyRace This race was reported in https://build.golang.org/log/6f043170946b665edb85b50804a62db68348c52f. As best as I can tell, it is another instance of #38370. The deferred call to backend.close() ought to be enough to ensure that the t.Logf happens before the end of the test, but in practice it is not, and with enough scheduling delay we can manage to trip the race detector on a call to Logf after the test function has returned. Updates #38370. Change-Id: I5ee45df45c6bfad3239d665df65a138f1c4573a3 Reviewed-on: https://go-review.googlesource.com/c/go/+/531195 LUCI-TryBot-Result: Go LUCI Auto-Submit: Bryan Mills Reviewed-by: Damien Neil --- serve_test.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/serve_test.go b/serve_test.go index 93503d65..00230020 100644 --- a/serve_test.go +++ b/serve_test.go @@ -3978,14 +3978,29 @@ func testTransportAndServerSharedBodyRace(t *testing.T, mode testMode) { const bodySize = 1 << 20 + var wg sync.WaitGroup backend := newClientServerTest(t, mode, HandlerFunc(func(rw ResponseWriter, req *Request) { + // Work around https://go.dev/issue/38370: clientServerTest uses + // an httptest.Server under the hood, and in HTTP/2 mode it does not always + // “[block] until all outstanding requests on this server have completed”, + // causing the call to Logf below to race with the end of the test. + // + // Since the client doesn't cancel the request until we have copied half + // the body, this call to add happens before the test is cleaned up, + // preventing the race. + wg.Add(1) + defer wg.Done() + n, err := io.CopyN(rw, req.Body, bodySize) t.Logf("backend CopyN: %v, %v", n, err) <-req.Context().Done() })) // We need to close explicitly here so that in-flight server // requests don't race with the call to SetRSTAvoidanceDelay for a retry. - defer backend.close() + defer func() { + wg.Wait() + backend.close() + }() var proxy *clientServerTest proxy = newClientServerTest(t, mode, HandlerFunc(func(rw ResponseWriter, req *Request) { From 0b34f7d45fb028d71e0f7dd9b8826a68f0962cdc Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Sat, 23 Sep 2023 17:05:42 -0400 Subject: [PATCH 53/93] net/http: add GODEBUG setting for old ServeMux behavior Add the GODEBUG setting httpmuxgo121. When set to "1", ServeMux behaves exactly like it did in Go 1.21. Implemented by defining a new, unexported type, serveMux121, that uses the original code. Updates #61410. Change-Id: I0a9d0fe2a2286e442d680393e62895ab50683cea Reviewed-on: https://go-review.googlesource.com/c/go/+/530461 Run-TryBot: Jonathan Amsterdam TryBot-Result: Gopher Robot Reviewed-by: Damien Neil --- request_test.go | 51 ------------ servemux121.go | 211 ++++++++++++++++++++++++++++++++++++++++++++++++ server.go | 29 ++++++- server_test.go | 62 ++++++++++++++ 4 files changed, 298 insertions(+), 55 deletions(-) create mode 100644 servemux121.go diff --git a/request_test.go b/request_test.go index 835db91a..1531da3d 100644 --- a/request_test.go +++ b/request_test.go @@ -1554,54 +1554,3 @@ func TestStatus(t *testing.T) { } } } - -func TestEscapedPathsAndPatterns(t *testing.T) { - matches := []struct { - pattern string - paths []string - }{ - { - "/a", // this pattern matches a path that unescapes to "/a" - []string{"/a", "/%61"}, - }, - { - "/%62", // patterns are unescaped by segment; matches paths that unescape to "/b" - []string{"/b", "/%62"}, - }, - { - "/%7B/%7D", // the only way to write a pattern that matches '{' or '}' - []string{"/{/}", "/%7b/}", "/{/%7d", "/%7B/%7D"}, - }, - { - "/%x", // patterns that do not unescape are left unchanged - []string{"/%25x"}, - }, - } - - mux := NewServeMux() - var gotPattern string - for _, m := range matches { - mux.HandleFunc(m.pattern, func(w ResponseWriter, r *Request) { - gotPattern = m.pattern - }) - } - - server := httptest.NewServer(mux) - defer server.Close() - - for _, m := range matches { - for _, p := range m.paths { - res, err := Get(server.URL + p) - if err != nil { - t.Fatal(err) - } - if res.StatusCode != 200 { - t.Errorf("%s: got code %d, want 200", p, res.StatusCode) - continue - } - if g, w := gotPattern, m.pattern; g != w { - t.Errorf("%s: pattern: got %q, want %q", p, g, w) - } - } - } -} diff --git a/servemux121.go b/servemux121.go new file mode 100644 index 00000000..c0a4b770 --- /dev/null +++ b/servemux121.go @@ -0,0 +1,211 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package http + +// This file implements ServeMux behavior as in Go 1.21. +// The behavior is controlled by a GODEBUG setting. +// Most of this code is derived from commit 08e35cc334. +// Changes are minimal: aside from the different receiver type, +// they mostly involve renaming functions, usually by unexporting them. + +import ( + "internal/godebug" + "net/url" + "sort" + "strings" + "sync" +) + +var httpmuxgo121 = godebug.New("httpmuxgo121") + +var use121 bool + +// Read httpmuxgo121 once at startup, since dealing with changes to it during +// program execution is too complex and error-prone. +func init() { + if httpmuxgo121.Value() == "1" { + use121 = true + httpmuxgo121.IncNonDefault() + } +} + +// serveMux121 holds the state of a ServeMux needed for Go 1.21 behavior. +type serveMux121 struct { + mu sync.RWMutex + m map[string]muxEntry + es []muxEntry // slice of entries sorted from longest to shortest. + hosts bool // whether any patterns contain hostnames +} + +type muxEntry struct { + h Handler + pattern string +} + +// Formerly ServeMux.Handle. +func (mux *serveMux121) handle(pattern string, handler Handler) { + mux.mu.Lock() + defer mux.mu.Unlock() + + if pattern == "" { + panic("http: invalid pattern") + } + if handler == nil { + panic("http: nil handler") + } + if _, exist := mux.m[pattern]; exist { + panic("http: multiple registrations for " + pattern) + } + + if mux.m == nil { + mux.m = make(map[string]muxEntry) + } + e := muxEntry{h: handler, pattern: pattern} + mux.m[pattern] = e + if pattern[len(pattern)-1] == '/' { + mux.es = appendSorted(mux.es, e) + } + + if pattern[0] != '/' { + mux.hosts = true + } +} + +func appendSorted(es []muxEntry, e muxEntry) []muxEntry { + n := len(es) + i := sort.Search(n, func(i int) bool { + return len(es[i].pattern) < len(e.pattern) + }) + if i == n { + return append(es, e) + } + // we now know that i points at where we want to insert + es = append(es, muxEntry{}) // try to grow the slice in place, any entry works. + copy(es[i+1:], es[i:]) // Move shorter entries down + es[i] = e + return es +} + +// Formerly ServeMux.HandleFunc. +func (mux *serveMux121) handleFunc(pattern string, handler func(ResponseWriter, *Request)) { + if handler == nil { + panic("http: nil handler") + } + mux.handle(pattern, HandlerFunc(handler)) +} + +// Formerly ServeMux.Handler. +func (mux *serveMux121) findHandler(r *Request) (h Handler, pattern string) { + + // CONNECT requests are not canonicalized. + if r.Method == "CONNECT" { + // If r.URL.Path is /tree and its handler is not registered, + // the /tree -> /tree/ redirect applies to CONNECT requests + // but the path canonicalization does not. + if u, ok := mux.redirectToPathSlash(r.URL.Host, r.URL.Path, r.URL); ok { + return RedirectHandler(u.String(), StatusMovedPermanently), u.Path + } + + return mux.handler(r.Host, r.URL.Path) + } + + // All other requests have any port stripped and path cleaned + // before passing to mux.handler. + host := stripHostPort(r.Host) + path := cleanPath(r.URL.Path) + + // If the given path is /tree and its handler is not registered, + // redirect for /tree/. + if u, ok := mux.redirectToPathSlash(host, path, r.URL); ok { + return RedirectHandler(u.String(), StatusMovedPermanently), u.Path + } + + if path != r.URL.Path { + _, pattern = mux.handler(host, path) + u := &url.URL{Path: path, RawQuery: r.URL.RawQuery} + return RedirectHandler(u.String(), StatusMovedPermanently), pattern + } + + return mux.handler(host, r.URL.Path) +} + +// handler is the main implementation of findHandler. +// The path is known to be in canonical form, except for CONNECT methods. +func (mux *serveMux121) handler(host, path string) (h Handler, pattern string) { + mux.mu.RLock() + defer mux.mu.RUnlock() + + // Host-specific pattern takes precedence over generic ones + if mux.hosts { + h, pattern = mux.match(host + path) + } + if h == nil { + h, pattern = mux.match(path) + } + if h == nil { + h, pattern = NotFoundHandler(), "" + } + return +} + +// Find a handler on a handler map given a path string. +// Most-specific (longest) pattern wins. +func (mux *serveMux121) match(path string) (h Handler, pattern string) { + // Check for exact match first. + v, ok := mux.m[path] + if ok { + return v.h, v.pattern + } + + // Check for longest valid match. mux.es contains all patterns + // that end in / sorted from longest to shortest. + for _, e := range mux.es { + if strings.HasPrefix(path, e.pattern) { + return e.h, e.pattern + } + } + return nil, "" +} + +// redirectToPathSlash determines if the given path needs appending "/" to it. +// This occurs when a handler for path + "/" was already registered, but +// not for path itself. If the path needs appending to, it creates a new +// URL, setting the path to u.Path + "/" and returning true to indicate so. +func (mux *serveMux121) redirectToPathSlash(host, path string, u *url.URL) (*url.URL, bool) { + mux.mu.RLock() + shouldRedirect := mux.shouldRedirectRLocked(host, path) + mux.mu.RUnlock() + if !shouldRedirect { + return u, false + } + path = path + "/" + u = &url.URL{Path: path, RawQuery: u.RawQuery} + return u, true +} + +// shouldRedirectRLocked reports whether the given path and host should be redirected to +// path+"/". This should happen if a handler is registered for path+"/" but +// not path -- see comments at ServeMux. +func (mux *serveMux121) shouldRedirectRLocked(host, path string) bool { + p := []string{path, host + path} + + for _, c := range p { + if _, exist := mux.m[c]; exist { + return false + } + } + + n := len(path) + if n == 0 { + return false + } + for _, c := range p { + if _, exist := mux.m[c+"/"]; exist { + return path[n-1] != '/' + } + } + + return false +} diff --git a/server.go b/server.go index ee02d776..f456e43c 100644 --- a/server.go +++ b/server.go @@ -33,6 +33,8 @@ import ( "golang.org/x/net/http/httpguts" ) +// TODO(jba): test + // Errors used by the HTTP server. var ( // ErrBodyNotAllowed is returned by ResponseWriter.Write calls @@ -2348,7 +2350,8 @@ type ServeMux struct { mu sync.RWMutex tree routingNode index routingIndex - patterns []*pattern // TODO(jba): remove if possible + patterns []*pattern // TODO(jba): remove if possible + mux121 serveMux121 // used only when GODEBUG=httpmuxgo121=1 } // NewServeMux allocates and returns a new ServeMux. @@ -2412,6 +2415,9 @@ func stripHostPort(h string) string { // If there is no registered handler that applies to the request, // Handler returns a “page not found” handler and an empty pattern. func (mux *ServeMux) Handler(r *Request) (h Handler, pattern string) { + if use121 { + return mux.mux121.findHandler(r) + } h, p, _, _ := mux.findHandler(r) return h, p } @@ -2585,9 +2591,12 @@ func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { w.WriteHeader(StatusBadRequest) return } - h, _, pat, matches := mux.findHandler(r) - r.pat = pat - r.matches = matches + var h Handler + if use121 { + h, _ = mux.mux121.findHandler(r) + } else { + h, _, r.pat, r.matches = mux.findHandler(r) + } h.ServeHTTP(w, r) } @@ -2597,23 +2606,35 @@ func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { // Handle registers the handler for the given pattern. // If a handler already exists for pattern, Handle panics. func (mux *ServeMux) Handle(pattern string, handler Handler) { + if use121 { + mux.mux121.handle(pattern, handler) + } mux.register(pattern, handler) } // HandleFunc registers the handler function for the given pattern. func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { + if use121 { + mux.mux121.handleFunc(pattern, handler) + } mux.register(pattern, HandlerFunc(handler)) } // Handle registers the handler for the given pattern in [DefaultServeMux]. // The documentation for [ServeMux] explains how patterns are matched. func Handle(pattern string, handler Handler) { + if use121 { + DefaultServeMux.mux121.handle(pattern, handler) + } DefaultServeMux.register(pattern, handler) } // HandleFunc registers the handler function for the given pattern in [DefaultServeMux]. // The documentation for [ServeMux] explains how patterns are matched. func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { + if use121 { + DefaultServeMux.mux121.handleFunc(pattern, handler) + } DefaultServeMux.register(pattern, HandlerFunc(handler)) } diff --git a/server_test.go b/server_test.go index a96d8765..d4185734 100644 --- a/server_test.go +++ b/server_test.go @@ -174,6 +174,68 @@ func TestExactMatch(t *testing.T) { } } +func TestEscapedPathsAndPatterns(t *testing.T) { + matches := []struct { + pattern string + paths []string // paths that match the pattern + paths121 []string // paths that matched the pattern in Go 1.21. + }{ + { + "/a", // this pattern matches a path that unescapes to "/a" + []string{"/a", "/%61"}, + []string{"/a", "/%61"}, + }, + { + "/%62", // patterns are unescaped by segment; matches paths that unescape to "/b" + []string{"/b", "/%62"}, + []string{"/%2562"}, // In 1.21, patterns were not unescaped but paths were. + }, + { + "/%7B/%7D", // the only way to write a pattern that matches '{' or '}' + []string{"/{/}", "/%7b/}", "/{/%7d", "/%7B/%7D"}, + []string{"/%257B/%257D"}, // In 1.21, patterns were not unescaped. + }, + { + "/%x", // patterns that do not unescape are left unchanged + []string{"/%25x"}, + []string{"/%25x"}, + }, + } + + run := func(t *testing.T, test121 bool) { + defer func(u bool) { use121 = u }(use121) + use121 = test121 + + mux := NewServeMux() + for _, m := range matches { + mux.HandleFunc(m.pattern, func(w ResponseWriter, r *Request) {}) + } + + for _, m := range matches { + paths := m.paths + if use121 { + paths = m.paths121 + } + for _, p := range paths { + u, err := url.ParseRequestURI(p) + if err != nil { + t.Fatal(err) + } + req := &Request{ + URL: u, + } + _, gotPattern := mux.Handler(req) + if g, w := gotPattern, m.pattern; g != w { + t.Errorf("%s: pattern: got %q, want %q", p, g, w) + } + } + } + } + + t.Run("latest", func(t *testing.T) { run(t, false) }) + t.Run("1.21", func(t *testing.T) { run(t, true) }) +} + func BenchmarkServerMatch(b *testing.B) { fn := func(w ResponseWriter, r *Request) { fmt.Fprintf(w, "OK") From 982fb054268705568598e367818a213233ceaf6d Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Mon, 25 Sep 2023 09:46:32 -0400 Subject: [PATCH 54/93] net/http: add a test for an empty ServeMux Make sure a ServeMux with no patterns is well-behaved. Updates #61410. Change-Id: Ib3eb85b384e1309e785663902d2c45ae01e64807 Reviewed-on: https://go-review.googlesource.com/c/go/+/530479 Reviewed-by: Damien Neil Run-TryBot: Jonathan Amsterdam TryBot-Result: Gopher Robot --- server.go | 2 -- server_test.go | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/server.go b/server.go index f456e43c..017a8188 100644 --- a/server.go +++ b/server.go @@ -33,8 +33,6 @@ import ( "golang.org/x/net/http/httpguts" ) -// TODO(jba): test - // Errors used by the HTTP server. var ( // ErrBodyNotAllowed is returned by ResponseWriter.Write calls diff --git a/server_test.go b/server_test.go index d4185734..e81e3bb6 100644 --- a/server_test.go +++ b/server_test.go @@ -118,6 +118,20 @@ func TestFindHandler(t *testing.T) { } } +func TestEmptyServeMux(t *testing.T) { + // Verify that a ServeMux with nothing registered + // doesn't panic. + mux := NewServeMux() + var r Request + r.Method = "GET" + r.Host = "example.com" + r.URL = &url.URL{Path: "/"} + _, p := mux.Handler(&r) + if p != "" { + t.Errorf(`got %q, want ""`, p) + } +} + func TestRegisterErr(t *testing.T) { mux := NewServeMux() h := &handler{} From 807697b0105cf4399c8334c0a81bcfe2709face6 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Mon, 25 Sep 2023 12:39:43 -0400 Subject: [PATCH 55/93] net/http: document new ServeMux patterns Updates #61410. Change-Id: Ib9dd8ebca43cec6e27c6fdfcf01ee6a1539c2fa0 Reviewed-on: https://go-review.googlesource.com/c/go/+/530481 Reviewed-by: Eli Bendersky Run-TryBot: Jonathan Amsterdam Reviewed-by: Alan Donovan Reviewed-by: Damien Neil TryBot-Result: Gopher Robot --- server.go | 142 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 112 insertions(+), 30 deletions(-) diff --git a/server.go b/server.go index 017a8188..bfd27453 100644 --- a/server.go +++ b/server.go @@ -2306,44 +2306,123 @@ func RedirectHandler(url string, code int) Handler { return &redirectHandler{url, code} } -// TODO(jba): rewrite the following doc for enhanced patterns (proposal -// https://go.dev/issue/61410). - // ServeMux is an HTTP request multiplexer. // It matches the URL of each incoming request against a list of registered // patterns and calls the handler for the pattern that // most closely matches the URL. // -// Patterns name fixed, rooted paths, like "/favicon.ico", -// or rooted subtrees, like "/images/" (note the trailing slash). -// Longer patterns take precedence over shorter ones, so that -// if there are handlers registered for both "/images/" -// and "/images/thumbnails/", the latter handler will be -// called for paths beginning with "/images/thumbnails/" and the -// former will receive requests for any other paths in the -// "/images/" subtree. -// -// Note that since a pattern ending in a slash names a rooted subtree, -// the pattern "/" matches all paths not matched by other registered -// patterns, not just the URL with Path == "/". -// -// If a subtree has been registered and a request is received naming the -// subtree root without its trailing slash, ServeMux redirects that -// request to the subtree root (adding the trailing slash). This behavior can -// be overridden with a separate registration for the path without -// the trailing slash. For example, registering "/images/" causes ServeMux +// # Patterns +// +// Patterns can match the method, host and path of a request. +// Some examples: +// +// - "/index.html" matches the path "/index.html" for any host and method. +// - "GET /static/" matches a GET request whose path begins with "/static/". +// - "example.com/" matches any request to the host "example.com". +// - "example.com/{$}" matches requests with host "example.com" and path "/". +// - "/b/{bucket}/o/{objectname...}" matches paths whose first segment is "b" +// and whose third segment is "o". The name "bucket" denotes the second +// segment and "objectname" denotes the remainder of the path. +// +// In general, a pattern looks like +// +// [METHOD ][HOST]/[PATH] +// +// All three parts are optional; "/" is a valid pattern. +// If METHOD is present, it must be followed by a single space. +// +// Literal (that is, non-wildcard) parts of a pattern match +// the corresponding parts of a request case-sensitively. +// +// A pattern with no method matches every method. A pattern +// with the method GET matches both GET and HEAD requests. +// Otherwise, the method must match exactly. +// +// A pattern with no host matches every host. +// A pattern with a host matches URLs on that host only. +// +// A path can include wildcard segments of the form {NAME} or {NAME...}. +// For example, "/b/{bucket}/o/{objectname...}". +// The wildcard name must be a valid Go identifier. +// Wildcards must be full path segments: they must be preceded by a slash and followed by +// either a slash or the end of the string. +// For example, "/b_{bucket}" is not a valid pattern. +// +// Normally a wildcard matches only a single path segment, +// ending at the next literal slash (not %2F) in the request URL. +// But if the "..." is present, then the wildcard matches the remainder of the URL path, including slashes. +// (Therefore it is invalid for a "..." wildcard to appear anywhere but at the end of a pattern.) +// The match for a wildcard can be obtained by calling [Request.PathValue] with the wildcard's name. +// A trailing slash in a path acts as an anonymous "..." wildcard. +// +// The special wildcard {$} matches only the end of the URL. +// For example, the pattern "/{$}" matches only the path "/", +// whereas the pattern "/" matches every path. +// +// For matching, both pattern paths and incoming request paths are unescaped segment by segment. +// So, for example, the path "/a%2Fb/100%25" is treated as having two segments, "a/b" and "100%". +// The pattern "/a%2fb/" matches it, but the pattern "/a/b/" does not. +// +// # Precedence +// +// If two or more patterns match a request, then the most specific pattern takes precedence. +// A pattern P1 is more specific than P2 if P1 matches a strict subset of P2’s requests; +// that is, if P2 matches all the requests of P1 and more. +// If neither is more specific, then the patterns conflict. +// There is one exception to this rule, for backwards compatibility: +// if two patterns would otherwise conflict and one has a host while the other does not, +// then the pattern with the host takes precedence. +// If a pattern passed [ServeMux.Handle] or [ServeMux.HandleFunc] conflicts with +// another pattern that is already registered, those functions panic. +// +// As an example of the general rule, "/images/thumbnails/" is more specific than "/images/", +// so both can be registered. +// The former matches paths beginning with "/images/thumbnails/" +// and the latter will match any other path in the "/images/" subtree. +// +// As another example, consider the patterns "GET /" and "/index.html": +// both match a GET request for "/index.html", but the former pattern +// matches all other GET and HEAD requests, while the latter matches any +// request for "/index.html" that uses a different method. +// The patterns conflict. +// +// # Trailing-slash redirection +// +// Consider a ServeMux with a handler for a subtree, registered using a trailing slash or "..." wildcard. +// If the ServeMux receives a request for the subtree root without a trailing slash, +// it redirects the request by adding the trailing slash. +// This behavior can be overridden with a separate registration for the path without +// the trailing slash or "..." wildcard. For example, registering "/images/" causes ServeMux // to redirect a request for "/images" to "/images/", unless "/images" has // been registered separately. // -// Patterns may optionally begin with a host name, restricting matches to -// URLs on that host only. Host-specific patterns take precedence over -// general patterns, so that a handler might register for the two patterns -// "/codesearch" and "codesearch.google.com/" without also taking over -// requests for "http://www.google.com/". +// # Request sanitizing // // ServeMux also takes care of sanitizing the URL request path and the Host // header, stripping the port number and redirecting any request containing . or -// .. elements or repeated slashes to an equivalent, cleaner URL. +// .. segments or repeated slashes to an equivalent, cleaner URL. +// +// # Compatibility +// +// The pattern syntax and matching behavior of ServeMux changed significantly +// in Go 1.22. To restore the old behavior, set the GODEBUG environment variable +// to "httpmuxgo121=1". This setting is read once, at program startup; changes +// during execution will be ignored. +// +// The backwards-incompatible changes include: +// - Wildcards are just ordinary literal path segments in 1.21. +// For example, the pattern "/{x}" will match only that path in 1.21, +// but will match any one-segment path in 1.22. +// - In 1.21, no pattern was rejected, unless it was empty or conflicted with an existing pattern. +// In 1.22, syntactically invalid patterns will cause [ServeMux.Handle] and [ServeMux.HandleFunc] to panic. +// For example, in 1.21, the patterns "/{" and "/a{x}" match themselves, +// but in 1.22 they are invalid and will cause a panic when registered. +// - In 1.22, each segment of a pattern is unescaped; this was not done in 1.21. +// For example, in 1.22 the pattern "/%61" matches the path "/a" ("%61" being the URL escape sequence for "a"), +// but in 1.21 it would match only the path "/%2561" (where "%25" is the escape for the percent sign). +// - When matching patterns to paths, in 1.22 each segment of the path is unescaped; in 1.21, the entire path is unescaped. +// This change mostly affects how paths with %2F escapes adjacent to slashes are treated. +// See https://go.dev/issue/21955 for details. type ServeMux struct { mu sync.RWMutex tree routingNode @@ -2570,7 +2649,7 @@ func (mux *ServeMux) matchingMethods(host, path string) []string { return methods } -// TODO: replace with maps.Keys when it is defined. +// TODO(jba): replace with maps.Keys when it is defined. func mapKeys[K comparable, V any](m map[K]V) []K { var ks []K for k := range m { @@ -2598,11 +2677,12 @@ func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { h.ServeHTTP(w, r) } -// The four functions below all call register so that callerLocation +// The four functions below all call ServeMux.register so that callerLocation // always refers to user code. // Handle registers the handler for the given pattern. -// If a handler already exists for pattern, Handle panics. +// If the given pattern conflicts, with one that is already registered, Handle +// panics. func (mux *ServeMux) Handle(pattern string, handler Handler) { if use121 { mux.mux121.handle(pattern, handler) @@ -2611,6 +2691,8 @@ func (mux *ServeMux) Handle(pattern string, handler Handler) { } // HandleFunc registers the handler function for the given pattern. +// If the given pattern conflicts, with one that is already registered, HandleFunc +// panics. func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { if use121 { mux.mux121.handleFunc(pattern, handler) From d34bfb937520a7af62ee1cc9124c77d585f60bfa Mon Sep 17 00:00:00 2001 From: edef Date: Sat, 3 Dec 2022 00:09:22 +0000 Subject: [PATCH 56/93] net/http/cgi: set SERVER_PORT to 443 when req.TLS != nil A hostname without a port leaves the port implied by the protocol. For HTTPS, the implied port is 443, not 80. Change-Id: I873a076068f84c8041abf10a435d9499635730a0 Reviewed-on: https://go-review.googlesource.com/c/go/+/454975 Auto-Submit: Damien Neil Reviewed-by: Dmitri Shuralyov Reviewed-by: Damien Neil LUCI-TryBot-Result: Go LUCI --- cgi/host.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cgi/host.go b/cgi/host.go index 073952a7..085658ee 100644 --- a/cgi/host.go +++ b/cgi/host.go @@ -132,6 +132,9 @@ func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { } port := "80" + if req.TLS != nil { + port = "443" + } if matches := trailingPort.FindStringSubmatch(req.Host); len(matches) != 0 { port = matches[1] } From 6b77343705f09a435d8ba5e1cc60cdcf0720e366 Mon Sep 17 00:00:00 2001 From: Jonathan Amsterdam Date: Thu, 5 Oct 2023 11:27:36 -0400 Subject: [PATCH 57/93] net/http: fix ServeMux pattern registration When the httpmuxgo121 GODEBUG setting was active, we were registering patterns in the old and the new way. Fix to register only in the old way. Change-Id: Ibc1fd41e7f4d162ee5bc34575df409e1db5657cd Reviewed-on: https://go-review.googlesource.com/c/go/+/533095 Run-TryBot: Jonathan Amsterdam TryBot-Result: Gopher Robot Reviewed-by: Olena Synenka Reviewed-by: Damien Neil --- server.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/server.go b/server.go index bfd27453..7fa785df 100644 --- a/server.go +++ b/server.go @@ -2686,8 +2686,9 @@ func (mux *ServeMux) ServeHTTP(w ResponseWriter, r *Request) { func (mux *ServeMux) Handle(pattern string, handler Handler) { if use121 { mux.mux121.handle(pattern, handler) + } else { + mux.register(pattern, handler) } - mux.register(pattern, handler) } // HandleFunc registers the handler function for the given pattern. @@ -2696,8 +2697,9 @@ func (mux *ServeMux) Handle(pattern string, handler Handler) { func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { if use121 { mux.mux121.handleFunc(pattern, handler) + } else { + mux.register(pattern, HandlerFunc(handler)) } - mux.register(pattern, HandlerFunc(handler)) } // Handle registers the handler for the given pattern in [DefaultServeMux]. @@ -2705,8 +2707,9 @@ func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Re func Handle(pattern string, handler Handler) { if use121 { DefaultServeMux.mux121.handle(pattern, handler) + } else { + DefaultServeMux.register(pattern, handler) } - DefaultServeMux.register(pattern, handler) } // HandleFunc registers the handler function for the given pattern in [DefaultServeMux]. @@ -2714,8 +2717,9 @@ func Handle(pattern string, handler Handler) { func HandleFunc(pattern string, handler func(ResponseWriter, *Request)) { if use121 { DefaultServeMux.mux121.handleFunc(pattern, handler) + } else { + DefaultServeMux.register(pattern, HandlerFunc(handler)) } - DefaultServeMux.register(pattern, HandlerFunc(handler)) } func (mux *ServeMux) register(pattern string, handler Handler) { From f75d9bbf2b881368be34d6f52864e2fd5a4dcda4 Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Tue, 10 Oct 2023 11:22:17 -0400 Subject: [PATCH 58/93] all: pull in x/net v0.17.0 and its dependencies Pull in a security fix from x/net/http2: http2: limit maximum handler goroutines to MaxConcurrentStreams Fixes #63417. Fixes CVE-2023-39325. Change-Id: I01e7774912e81007a7cf70f33e5989fb50a0b708 Cq-Include-Trybots: luci.golang.try:gotip-linux-amd64-longtest Reviewed-on: https://go-review.googlesource.com/c/go/+/534295 Auto-Submit: Dmitri Shuralyov Reviewed-by: Michael Pratt Reviewed-by: Dmitri Shuralyov LUCI-TryBot-Result: Go LUCI --- h2_bundle.go | 89 +++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 71 insertions(+), 18 deletions(-) diff --git a/h2_bundle.go b/h2_bundle.go index c95fbc47..54ac12f5 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -4394,9 +4394,11 @@ type http2serverConn struct { advMaxStreams uint32 // our SETTINGS_MAX_CONCURRENT_STREAMS advertised the client curClientStreams uint32 // number of open streams initiated by the client curPushedStreams uint32 // number of open streams initiated by server push + curHandlers uint32 // number of running handler goroutines maxClientStreamID uint32 // max ever seen from client (odd), or 0 if there have been no client requests maxPushPromiseID uint32 // ID of the last push promise (even), or 0 if there have been no pushes streams map[uint32]*http2stream + unstartedHandlers []http2unstartedHandler initialStreamSendWindowSize int32 maxFrameSize int32 peerMaxHeaderListSize uint32 // zero means unknown (default) @@ -4797,6 +4799,8 @@ func (sc *http2serverConn) serve() { return case http2gracefulShutdownMsg: sc.startGracefulShutdownInternal() + case http2handlerDoneMsg: + sc.handlerDone() default: panic("unknown timer") } @@ -4828,14 +4832,6 @@ func (sc *http2serverConn) serve() { } } -func (sc *http2serverConn) awaitGracefulShutdown(sharedCh <-chan struct{}, privateCh chan struct{}) { - select { - case <-sc.doneServing: - case <-sharedCh: - close(privateCh) - } -} - type http2serverMessage int // Message values sent to serveMsgCh. @@ -4844,6 +4840,7 @@ var ( http2idleTimerMsg = new(http2serverMessage) http2shutdownTimerMsg = new(http2serverMessage) http2gracefulShutdownMsg = new(http2serverMessage) + http2handlerDoneMsg = new(http2serverMessage) ) func (sc *http2serverConn) onSettingsTimer() { sc.sendServeMsg(http2settingsTimerMsg) } @@ -5718,9 +5715,11 @@ func (st *http2stream) copyTrailersToHandlerRequest() { // onReadTimeout is run on its own goroutine (from time.AfterFunc) // when the stream's ReadTimeout has fired. func (st *http2stream) onReadTimeout() { - // Wrap the ErrDeadlineExceeded to avoid callers depending on us - // returning the bare error. - st.body.CloseWithError(fmt.Errorf("%w", os.ErrDeadlineExceeded)) + if st.body != nil { + // Wrap the ErrDeadlineExceeded to avoid callers depending on us + // returning the bare error. + st.body.CloseWithError(fmt.Errorf("%w", os.ErrDeadlineExceeded)) + } } // onWriteTimeout is run on its own goroutine (from time.AfterFunc) @@ -5838,13 +5837,10 @@ func (sc *http2serverConn) processHeaders(f *http2MetaHeadersFrame) error { // (in Go 1.8), though. That's a more sane option anyway. if sc.hs.ReadTimeout != 0 { sc.conn.SetReadDeadline(time.Time{}) - if st.body != nil { - st.readDeadline = time.AfterFunc(sc.hs.ReadTimeout, st.onReadTimeout) - } + st.readDeadline = time.AfterFunc(sc.hs.ReadTimeout, st.onReadTimeout) } - go sc.runHandler(rw, req, handler) - return nil + return sc.scheduleHandler(id, rw, req, handler) } func (sc *http2serverConn) upgradeRequest(req *Request) { @@ -5864,6 +5860,10 @@ func (sc *http2serverConn) upgradeRequest(req *Request) { sc.conn.SetReadDeadline(time.Time{}) } + // This is the first request on the connection, + // so start the handler directly rather than going + // through scheduleHandler. + sc.curHandlers++ go sc.runHandler(rw, req, sc.handler.ServeHTTP) } @@ -6104,8 +6104,62 @@ func (sc *http2serverConn) newResponseWriter(st *http2stream, req *Request) *htt return &http2responseWriter{rws: rws} } +type http2unstartedHandler struct { + streamID uint32 + rw *http2responseWriter + req *Request + handler func(ResponseWriter, *Request) +} + +// scheduleHandler starts a handler goroutine, +// or schedules one to start as soon as an existing handler finishes. +func (sc *http2serverConn) scheduleHandler(streamID uint32, rw *http2responseWriter, req *Request, handler func(ResponseWriter, *Request)) error { + sc.serveG.check() + maxHandlers := sc.advMaxStreams + if sc.curHandlers < maxHandlers { + sc.curHandlers++ + go sc.runHandler(rw, req, handler) + return nil + } + if len(sc.unstartedHandlers) > int(4*sc.advMaxStreams) { + return sc.countError("too_many_early_resets", http2ConnectionError(http2ErrCodeEnhanceYourCalm)) + } + sc.unstartedHandlers = append(sc.unstartedHandlers, http2unstartedHandler{ + streamID: streamID, + rw: rw, + req: req, + handler: handler, + }) + return nil +} + +func (sc *http2serverConn) handlerDone() { + sc.serveG.check() + sc.curHandlers-- + i := 0 + maxHandlers := sc.advMaxStreams + for ; i < len(sc.unstartedHandlers); i++ { + u := sc.unstartedHandlers[i] + if sc.streams[u.streamID] == nil { + // This stream was reset before its goroutine had a chance to start. + continue + } + if sc.curHandlers >= maxHandlers { + break + } + sc.curHandlers++ + go sc.runHandler(u.rw, u.req, u.handler) + sc.unstartedHandlers[i] = http2unstartedHandler{} // don't retain references + } + sc.unstartedHandlers = sc.unstartedHandlers[i:] + if len(sc.unstartedHandlers) == 0 { + sc.unstartedHandlers = nil + } +} + // Run on its own goroutine. func (sc *http2serverConn) runHandler(rw *http2responseWriter, req *Request, handler func(ResponseWriter, *Request)) { + defer sc.sendServeMsg(http2handlerDoneMsg) didPanic := true defer func() { rw.rws.stream.cancelCtx() @@ -7312,8 +7366,7 @@ func (t *http2Transport) initConnPool() { // HTTP/2 server. type http2ClientConn struct { t *http2Transport - tconn net.Conn // usually *tls.Conn, except specialized impls - tconnClosed bool + tconn net.Conn // usually *tls.Conn, except specialized impls tlsState *tls.ConnectionState // nil only for specialized impls reused uint32 // whether conn is being reused; atomic singleUse bool // whether being used for a single http.Request From 376c7e89a9f9bd45011b42410d830fab6692832f Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Tue, 17 Oct 2023 15:19:33 -0400 Subject: [PATCH 59/93] all: drop old +build lines Running 'go fix' on the cmd+std packages handled much of this change. Also update code generators to use only the new go:build lines, not the old +build ones. For #41184. For #60268. Change-Id: If35532abe3012e7357b02c79d5992ff5ac37ca23 Cq-Include-Trybots: luci.golang.try:gotip-linux-386-longtest,gotip-linux-amd64-longtest,gotip-windows-amd64-longtest Reviewed-on: https://go-review.googlesource.com/c/go/+/536237 Reviewed-by: Ian Lance Taylor Reviewed-by: Dmitri Shuralyov Auto-Submit: Dmitri Shuralyov LUCI-TryBot-Result: Go LUCI --- h2_error.go | 1 - h2_error_test.go | 1 - 2 files changed, 2 deletions(-) diff --git a/h2_error.go b/h2_error.go index 0391d31e..2c0b21ec 100644 --- a/h2_error.go +++ b/h2_error.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !nethttpomithttp2 -// +build !nethttpomithttp2 package http diff --git a/h2_error_test.go b/h2_error_test.go index 0d85e2f3..5e400683 100644 --- a/h2_error_test.go +++ b/h2_error_test.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !nethttpomithttp2 -// +build !nethttpomithttp2 package http From 85f6c0b01a2891012412b1530831471b4becca07 Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Tue, 17 Oct 2023 15:39:54 -0400 Subject: [PATCH 60/93] all: update vendored dependencies One of the remaining uses of the old +build syntax was in the bundled copy of golang.org/x/net/http2 in net/http. Pull in a newer version of bundle with CL 536075 that drops said +build syntax line. Also pull in newer x/sys and other golang.org/x modules where old +build lines were recently dropped. Generated with: go install golang.org/x/build/cmd/updatestd@latest go install golang.org/x/tools/cmd/bundle@latest updatestd -goroot=$(pwd) -branch=master For #36905. For #41184. For #60268. Change-Id: Ia18d1ce9eadce85b38176058ad1fe38562b004e9 Cq-Include-Trybots: luci.golang.try:gotip-linux-386-longtest,gotip-linux-amd64-longtest,gotip-windows-amd64-longtest Reviewed-on: https://go-review.googlesource.com/c/go/+/536575 Reviewed-by: Dmitri Shuralyov Auto-Submit: Dmitri Shuralyov LUCI-TryBot-Result: Go LUCI Reviewed-by: Ian Lance Taylor --- h2_bundle.go | 1 - 1 file changed, 1 deletion(-) diff --git a/h2_bundle.go b/h2_bundle.go index 54ac12f5..83f0bc09 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -1,5 +1,4 @@ //go:build !nethttpomithttp2 -// +build !nethttpomithttp2 // Code generated by golang.org/x/tools/cmd/bundle. DO NOT EDIT. // $ bundle -o=h2_bundle.go -prefix=http2 -tags=!nethttpomithttp2 golang.org/x/net/http2 From 32d89f5c74d242397956c1a7ab0aaa24fa7bee8e Mon Sep 17 00:00:00 2001 From: Keiichi Hirobe Date: Sun, 22 Oct 2023 17:30:37 +0900 Subject: [PATCH 61/93] net/http: remove outdated comment about a support of CONNECT method The net/http.Transport already supports CONNECT after https://go-review.googlesource.com/c/go/+/123156 was merged, which deleted comments in transport.go. Change-Id: I784fdb9b044bc8a4a29bf252328c80a11aaf6901 Reviewed-on: https://go-review.googlesource.com/c/go/+/536057 Auto-Submit: Damien Neil LUCI-TryBot-Result: Go LUCI Reviewed-by: David Chase Reviewed-by: Brad Fitzpatrick Reviewed-by: Damien Neil --- request.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/request.go b/request.go index b66e6853..ed2cdac1 100644 --- a/request.go +++ b/request.go @@ -111,10 +111,6 @@ var reqWriteExcludeHeader = map[string]bool{ type Request struct { // Method specifies the HTTP method (GET, POST, PUT, etc.). // For client requests, an empty string means GET. - // - // Go's HTTP client does not support sending a request with - // the CONNECT method. See the documentation on Transport for - // details. Method string // URL specifies either the URI being requested (for server From b4d8baac403a08687d19a8cd95f6a7472d124d4e Mon Sep 17 00:00:00 2001 From: Mauri de Souza Meneguzzo Date: Thu, 26 Oct 2023 01:52:57 +0000 Subject: [PATCH 62/93] net/http: pull http2 underflow fix from x/net/http2 After CL 534295 was merged to fix a CVE it introduced an underflow when we try to decrement sc.curHandlers in handlerDone. Pull in a fix from x/net/http2: http2: fix underflow in http2 server push https://go-review.googlesource.com/c/net/+/535595 Fixes #63511 Change-Id: I5c678ce7dcc53635f3ad5e4999857cb120dfc1ab GitHub-Last-Rev: 587ffa3cafbb9da6bc82ba8a5b83313f81e5c89b GitHub-Pull-Request: golang/go#63561 Reviewed-on: https://go-review.googlesource.com/c/go/+/535575 Run-TryBot: Mauri de Souza Meneguzzo Reviewed-by: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov Reviewed-by: David Chase Auto-Submit: Dmitri Shuralyov TryBot-Result: Gopher Robot --- h2_bundle.go | 1 + 1 file changed, 1 insertion(+) diff --git a/h2_bundle.go b/h2_bundle.go index 83f0bc09..fea33276 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -7004,6 +7004,7 @@ func (sc *http2serverConn) startPush(msg *http2startPushRequest) { panic(fmt.Sprintf("newWriterAndRequestNoBody(%+v): %v", msg.url, err)) } + sc.curHandlers++ go sc.runHandler(rw, req, sc.handler.ServeHTTP) return promisedID, nil } From c8cdf46839c1980c84cb17ffe9f7c9e837bbacec Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Thu, 2 Nov 2023 10:29:08 -0400 Subject: [PATCH 63/93] net/http: remove arbitrary timeouts in tests of Server.ErrorLog This also allows us to remove the chanWriter helper from the test, using a simpler strings.Builder instead, relying on clientServerTest.close for synchronization. (I don't think this runs afoul of #38370, because the handler functions themselves in these tests should never be executed, let alone result in an asynchronous write to the error log.) Fixes #57599. Change-Id: I45c6cefca0bb218f6f9a9659de6bde454547f704 Reviewed-on: https://go-review.googlesource.com/c/go/+/539436 Run-TryBot: Bryan Mills TryBot-Result: Gopher Robot Auto-Submit: Bryan Mills Reviewed-by: Damien Neil LUCI-TryBot-Result: Go LUCI --- client_test.go | 46 ++++++++++++++++------------------------------ serve_test.go | 17 +++++++++-------- 2 files changed, 25 insertions(+), 38 deletions(-) diff --git a/client_test.go b/client_test.go index 0fe555af..df2a670a 100644 --- a/client_test.go +++ b/client_test.go @@ -60,13 +60,6 @@ func pedanticReadAll(r io.Reader) (b []byte, err error) { } } -type chanWriter chan string - -func (w chanWriter) Write(p []byte) (n int, err error) { - w <- string(p) - return len(p), nil -} - func TestClient(t *testing.T) { run(t, testClient) } func testClient(t *testing.T, mode testMode) { ts := newClientServerTest(t, mode, robotsTxtHandler).ts @@ -827,12 +820,12 @@ func TestClientInsecureTransport(t *testing.T) { run(t, testClientInsecureTransport, []testMode{https1Mode, http2Mode}) } func testClientInsecureTransport(t *testing.T, mode testMode) { - ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { w.Write([]byte("Hello")) - })).ts - errc := make(chanWriter, 10) // but only expecting 1 - ts.Config.ErrorLog = log.New(errc, "", 0) - defer ts.Close() + })) + ts := cst.ts + errLog := new(strings.Builder) + ts.Config.ErrorLog = log.New(errLog, "", 0) // TODO(bradfitz): add tests for skipping hostname checks too? // would require a new cert for testing, and probably @@ -851,15 +844,10 @@ func testClientInsecureTransport(t *testing.T, mode testMode) { } } - select { - case v := <-errc: - if !strings.Contains(v, "TLS handshake error") { - t.Errorf("expected an error log message containing 'TLS handshake error'; got %q", v) - } - case <-time.After(5 * time.Second): - t.Errorf("timeout waiting for logged error") + cst.close() + if !strings.Contains(errLog.String(), "TLS handshake error") { + t.Errorf("expected an error log message containing 'TLS handshake error'; got %q", errLog) } - } func TestClientErrorWithRequestURI(t *testing.T) { @@ -897,9 +885,10 @@ func TestClientWithIncorrectTLSServerName(t *testing.T) { run(t, testClientWithIncorrectTLSServerName, []testMode{https1Mode, http2Mode}) } func testClientWithIncorrectTLSServerName(t *testing.T, mode testMode) { - ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) {})).ts - errc := make(chanWriter, 10) // but only expecting 1 - ts.Config.ErrorLog = log.New(errc, "", 0) + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) {})) + ts := cst.ts + errLog := new(strings.Builder) + ts.Config.ErrorLog = log.New(errLog, "", 0) c := ts.Client() c.Transport.(*Transport).TLSClientConfig.ServerName = "badserver" @@ -910,13 +899,10 @@ func testClientWithIncorrectTLSServerName(t *testing.T, mode testMode) { if !strings.Contains(err.Error(), "127.0.0.1") || !strings.Contains(err.Error(), "badserver") { t.Errorf("wanted error mentioning 127.0.0.1 and badserver; got error: %v", err) } - select { - case v := <-errc: - if !strings.Contains(v, "TLS handshake error") { - t.Errorf("expected an error log message containing 'TLS handshake error'; got %q", v) - } - case <-time.After(5 * time.Second): - t.Errorf("timeout waiting for logged error") + + cst.close() + if !strings.Contains(errLog.String(), "TLS handshake error") { + t.Errorf("expected an error log message containing 'TLS handshake error'; got %q", errLog) } } diff --git a/serve_test.go b/serve_test.go index 00230020..0c76f1bc 100644 --- a/serve_test.go +++ b/serve_test.go @@ -1400,27 +1400,28 @@ func TestTLSHandshakeTimeout(t *testing.T) { run(t, testTLSHandshakeTimeout, []testMode{https1Mode, http2Mode}) } func testTLSHandshakeTimeout(t *testing.T, mode testMode) { - errc := make(chanWriter, 10) // but only expecting 1 - ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) {}), + errLog := new(strings.Builder) + cst := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) {}), func(ts *httptest.Server) { ts.Config.ReadTimeout = 250 * time.Millisecond - ts.Config.ErrorLog = log.New(errc, "", 0) + ts.Config.ErrorLog = log.New(errLog, "", 0) }, - ).ts + ) + ts := cst.ts + conn, err := net.Dial("tcp", ts.Listener.Addr().String()) if err != nil { t.Fatalf("Dial: %v", err) } - defer conn.Close() - var buf [1]byte n, err := conn.Read(buf[:]) if err == nil || n != 0 { t.Errorf("Read = %d, %v; want an error and no bytes", n, err) } + conn.Close() - v := <-errc - if !strings.Contains(v, "timeout") && !strings.Contains(v, "TLS handshake") { + cst.close() + if v := errLog.String(); !strings.Contains(v, "timeout") && !strings.Contains(v, "TLS handshake") { t.Errorf("expected a TLS handshake timeout error; got %q", v) } } From 5eaaed62068fcfd62c9510eb560b40de657aa0ca Mon Sep 17 00:00:00 2001 From: Zeke Lu Date: Wed, 1 Nov 2023 13:47:23 +0000 Subject: [PATCH 64/93] net/http/httptest: remove unnecessary creation of http.Transport In (*Server).StartTLS, it's unnecessary to create an http.Client with a Transport, because a new one will be created with the TLSClientConfig later. Change-Id: I086e28717e9739787529006c3f0296c8224cd790 GitHub-Last-Rev: 33724596bd901a05a91654f8c2df233aa6563ea6 GitHub-Pull-Request: golang/go#60124 Reviewed-on: https://go-review.googlesource.com/c/go/+/494355 Run-TryBot: t hepudds Reviewed-by: Heschi Kreinick TryBot-Result: Gopher Robot Reviewed-by: Damien Neil Reviewed-by: qiulaidongfeng <2645477756@qq.com> --- httptest/server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/httptest/server.go b/httptest/server.go index 79749a03..c962749e 100644 --- a/httptest/server.go +++ b/httptest/server.go @@ -144,7 +144,7 @@ func (s *Server) StartTLS() { panic("Server already started") } if s.client == nil { - s.client = &http.Client{Transport: &http.Transport{}} + s.client = &http.Client{} } cert, err := tls.X509KeyPair(testcert.LocalhostCert, testcert.LocalhostKey) if err != nil { From dd842048eb9c39272d626561c38e30fbce0a0bc8 Mon Sep 17 00:00:00 2001 From: Mitar Date: Mon, 6 Nov 2023 11:00:51 +0000 Subject: [PATCH 65/93] net/http: set/override Content-Length for encoded range requests Currently, http.ServeContent returns invalid Content-Length header if: * Request is a range request. * Content is encoded (e.g., gzip compressed). * Content-Length of the encoded content has been set before calling http.ServeContent, as suggested in https://github.com/golang/go/issues/19420. Example: w.Header().Set("Content-Type", "application/json") w.Header().Set("Content-Length", strconv.Itoa(len(compressedJsonBody))) w.Header().Set("Content-Encoding", "gzip") w.Header().Set("Etag", etag) http.ServeContent( w, req, "", time.Time{}, bytes.NewReader(compressedJsonBody), ) The issue is that http.ServeContent currently sees Content-Length as something optional when Content-Encoding is set, but that is a problem with range request which can send a payload of different size. So this reverts https://go.dev/cl/4538111 and makes Content-Length be set always to the number of bytes which will actually be send (both for range and non-range requests). Without this fix, this is an example response: HTTP/1.1 206 Partial Content Accept-Ranges: bytes Content-Encoding: gzip Content-Length: 351 Content-Range: bytes 100-350/351 Content-Type: application/json; charset=UTF-8 Etag: "amCTP_vgT5PQt5OsAEI7NFJ6Hx1UfEpR5nIaYEInfOA" Date: Sat, 29 Jan 2022 14:42:15 GMT As you see, Content-Length is invalid and should be 251. Change-Id: I4d2ea3a8489a115f92ef1f7e98250d555b47a94e GitHub-Last-Rev: 3aff9126f5d62725c7d539df2d0eb2b860a84ca6 GitHub-Pull-Request: golang/go#50904 Reviewed-on: https://go-review.googlesource.com/c/go/+/381956 Reviewed-by: Damien Neil Run-TryBot: t hepudds Reviewed-by: David Chase TryBot-Result: Gopher Robot --- fs.go | 4 +--- fs_test.go | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/fs.go b/fs.go index c605fe3a..20da5600 100644 --- a/fs.go +++ b/fs.go @@ -343,9 +343,7 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, } w.Header().Set("Accept-Ranges", "bytes") - if w.Header().Get("Content-Encoding") == "" { - w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) - } + w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) w.WriteHeader(code) diff --git a/fs_test.go b/fs_test.go index cfabaae3..d29664c1 100644 --- a/fs_test.go +++ b/fs_test.go @@ -571,7 +571,7 @@ func testServeDirWithoutTrailingSlash(t *testing.T, mode testMode) { } } -// Tests that ServeFile doesn't add a Content-Length if a Content-Encoding is +// Tests that ServeFile adds a Content-Length even if a Content-Encoding is // specified. func TestServeFileWithContentEncoding(t *testing.T) { run(t, testServeFileWithContentEncoding) } func testServeFileWithContentEncoding(t *testing.T, mode testMode) { @@ -593,7 +593,7 @@ func testServeFileWithContentEncoding(t *testing.T, mode testMode) { t.Fatal(err) } resp.Body.Close() - if g, e := resp.ContentLength, int64(-1); g != e { + if g, e := resp.ContentLength, int64(11); g != e { t.Errorf("Content-Length mismatch: got %d, want %d", g, e) } } From b8d3b92234331503a0c5ecc1e884f0a497235aad Mon Sep 17 00:00:00 2001 From: "carl.tao" Date: Thu, 21 Sep 2023 23:38:33 +0800 Subject: [PATCH 66/93] net/http: remove Content-Encoding header in roundtrip_js The fetch api will decode the gzip, but Content-Encoding not be deleted. To ensure that the behavior of roundtrip_js is consistent with native. delete the Content-Encoding header when the response body is decompressed by js fetch api. Fixes #63139 Change-Id: Ie35b3aa050786e2ef865f9ffa992e30ab060506e Reviewed-on: https://go-review.googlesource.com/c/go/+/530155 Commit-Queue: Damien Neil Reviewed-by: Bryan Mills Reviewed-by: Damien Neil LUCI-TryBot-Result: Go LUCI Auto-Submit: Damien Neil --- roundtrip_js.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/roundtrip_js.go b/roundtrip_js.go index 9f9f0cb6..cbf978af 100644 --- a/roundtrip_js.go +++ b/roundtrip_js.go @@ -10,6 +10,7 @@ import ( "errors" "fmt" "io" + "net/http/internal/ascii" "strconv" "strings" "syscall/js" @@ -184,11 +185,22 @@ func (t *Transport) RoundTrip(req *Request) (*Response, error) { } code := result.Get("status").Int() + + uncompressed := false + if ascii.EqualFold(header.Get("Content-Encoding"), "gzip") { + // The fetch api will decode the gzip, but Content-Encoding not be deleted. + header.Del("Content-Encoding") + header.Del("Content-Length") + contentLength = -1 + uncompressed = true + } + respCh <- &Response{ Status: fmt.Sprintf("%d %s", code, StatusText(code)), StatusCode: code, Header: header, ContentLength: contentLength, + Uncompressed: uncompressed, Body: body, Request: req, } From fa70e90396ab58576bad3d6d5af533079582a7df Mon Sep 17 00:00:00 2001 From: aimuz Date: Fri, 3 Nov 2023 23:42:21 +0000 Subject: [PATCH 67/93] net/http/cgi: the PATH_INFO should be empty or start with a slash fixed PATH_INFO not starting with a slash as described in RFC 3875 for PATH_INFO. Fixes #63925 Change-Id: I1ead98dff190c53eb7a50546569ef6ded3199a0a GitHub-Last-Rev: 1c532e330b0d74ee42afc412611a005bc565bb26 GitHub-Pull-Request: golang/go#63926 Reviewed-on: https://go-review.googlesource.com/c/go/+/539615 Reviewed-by: Bryan Mills Reviewed-by: Damien Neil Auto-Submit: Damien Neil LUCI-TryBot-Result: Go LUCI --- cgi/host.go | 11 ++--------- cgi/host_test.go | 8 ++++---- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/cgi/host.go b/cgi/host.go index 085658ee..ef222ab7 100644 --- a/cgi/host.go +++ b/cgi/host.go @@ -115,21 +115,14 @@ func removeLeadingDuplicates(env []string) (ret []string) { } func (h *Handler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - root := h.Root - if root == "" { - root = "/" - } - if len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked" { rw.WriteHeader(http.StatusBadRequest) rw.Write([]byte("Chunked request bodies are not supported by CGI.")) return } - pathInfo := req.URL.Path - if root != "/" && strings.HasPrefix(pathInfo, root) { - pathInfo = pathInfo[len(root):] - } + root := strings.TrimRight(h.Root, "/") + pathInfo := strings.TrimPrefix(req.URL.Path, root) port := "80" if req.TLS != nil { diff --git a/cgi/host_test.go b/cgi/host_test.go index 707af71d..f310a83d 100644 --- a/cgi/host_test.go +++ b/cgi/host_test.go @@ -210,14 +210,14 @@ func TestPathInfoDirRoot(t *testing.T) { check(t) h := &Handler{ Path: "testdata/test.cgi", - Root: "/myscript/", + Root: "/myscript//", } expectedMap := map[string]string{ - "env-PATH_INFO": "bar", + "env-PATH_INFO": "/bar", "env-QUERY_STRING": "a=b", "env-REQUEST_URI": "/myscript/bar?a=b", "env-SCRIPT_FILENAME": "testdata/test.cgi", - "env-SCRIPT_NAME": "/myscript/", + "env-SCRIPT_NAME": "/myscript", } runCgiTest(t, h, "GET /myscript/bar?a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) } @@ -278,7 +278,7 @@ func TestPathInfoNoRoot(t *testing.T) { "env-QUERY_STRING": "a=b", "env-REQUEST_URI": "/bar?a=b", "env-SCRIPT_FILENAME": "testdata/test.cgi", - "env-SCRIPT_NAME": "/", + "env-SCRIPT_NAME": "", } runCgiTest(t, h, "GET /bar?a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) } From 63597586a723cf0697a2da1b6051c95c56dcb67a Mon Sep 17 00:00:00 2001 From: aimuz Date: Tue, 7 Nov 2023 13:25:32 +0000 Subject: [PATCH 68/93] net/http/cgi: eliminate use of Perl in tests Previously, a Perl script was used to test the net/http/cgi package. This sometimes led to hidden failures as these tests were not run on builders without Perl. Also, this approach posed maintenance difficulties for those unfamiliar with Perl. We have now replaced Perl-based tests with a Go handler to simplify maintenance and ensure consistent testing environments. It's part of our ongoing effort to reduce reliance on Perl throughout the Go codebase (see #20032,#25586,#25669,#27779), thus improving reliability and ease of maintenance. Fixes #63800 Fixes #63828 Change-Id: I8d554af93d4070036cf0cc3aaa9c9b256affbd17 GitHub-Last-Rev: a8034083d824da7d68e5995a7997a1199d78de15 GitHub-Pull-Request: golang/go#63869 Reviewed-on: https://go-review.googlesource.com/c/go/+/538861 Reviewed-by: Bryan Mills Run-TryBot: qiulaidongfeng <2645477756@qq.com> TryBot-Result: Gopher Robot Reviewed-by: Damien Neil Auto-Submit: Bryan Mills Commit-Queue: Bryan Mills --- cgi/cgi_main.go | 145 ++++++++++++++++++++++++++++++++ cgi/host_test.go | 180 +++++++++++++--------------------------- cgi/integration_test.go | 67 +-------------- cgi/testdata/test.cgi | 95 --------------------- 4 files changed, 203 insertions(+), 284 deletions(-) create mode 100644 cgi/cgi_main.go delete mode 100755 cgi/testdata/test.cgi diff --git a/cgi/cgi_main.go b/cgi/cgi_main.go new file mode 100644 index 00000000..8997d66a --- /dev/null +++ b/cgi/cgi_main.go @@ -0,0 +1,145 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cgi + +import ( + "fmt" + "io" + "net/http" + "os" + "path" + "sort" + "strings" + "time" +) + +func cgiMain() { + switch path.Join(os.Getenv("SCRIPT_NAME"), os.Getenv("PATH_INFO")) { + case "/bar", "/test.cgi", "/myscript/bar", "/test.cgi/extrapath": + testCGI() + return + } + childCGIProcess() +} + +// testCGI is a CGI program translated from a Perl program to complete host_test. +// test cases in host_test should be provided by testCGI. +func testCGI() { + req, err := Request() + if err != nil { + panic(err) + } + + err = req.ParseForm() + if err != nil { + panic(err) + } + + params := req.Form + if params.Get("loc") != "" { + fmt.Printf("Location: %s\r\n\r\n", params.Get("loc")) + return + } + + fmt.Printf("Content-Type: text/html\r\n") + fmt.Printf("X-CGI-Pid: %d\r\n", os.Getpid()) + fmt.Printf("X-Test-Header: X-Test-Value\r\n") + fmt.Printf("\r\n") + + if params.Get("writestderr") != "" { + fmt.Fprintf(os.Stderr, "Hello, stderr!\n") + } + + if params.Get("bigresponse") != "" { + // 17 MB, for OS X: golang.org/issue/4958 + line := strings.Repeat("A", 1024) + for i := 0; i < 17*1024; i++ { + fmt.Printf("%s\r\n", line) + } + return + } + + fmt.Printf("test=Hello CGI\r\n") + + keys := make([]string, 0, len(params)) + for k := range params { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + fmt.Printf("param-%s=%s\r\n", key, params.Get(key)) + } + + envs := envMap(os.Environ()) + keys = make([]string, 0, len(envs)) + for k := range envs { + keys = append(keys, k) + } + sort.Strings(keys) + for _, key := range keys { + fmt.Printf("env-%s=%s\r\n", key, envs[key]) + } + + cwd, _ := os.Getwd() + fmt.Printf("cwd=%s\r\n", cwd) +} + +type neverEnding byte + +func (b neverEnding) Read(p []byte) (n int, err error) { + for i := range p { + p[i] = byte(b) + } + return len(p), nil +} + +// childCGIProcess is used by integration_test to complete unit tests. +func childCGIProcess() { + if os.Getenv("REQUEST_METHOD") == "" { + // Not in a CGI environment; skipping test. + return + } + switch os.Getenv("REQUEST_URI") { + case "/immediate-disconnect": + os.Exit(0) + case "/no-content-type": + fmt.Printf("Content-Length: 6\n\nHello\n") + os.Exit(0) + case "/empty-headers": + fmt.Printf("\nHello") + os.Exit(0) + } + Serve(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + if req.FormValue("nil-request-body") == "1" { + fmt.Fprintf(rw, "nil-request-body=%v\n", req.Body == nil) + return + } + rw.Header().Set("X-Test-Header", "X-Test-Value") + req.ParseForm() + if req.FormValue("no-body") == "1" { + return + } + if eb, ok := req.Form["exact-body"]; ok { + io.WriteString(rw, eb[0]) + return + } + if req.FormValue("write-forever") == "1" { + io.Copy(rw, neverEnding('a')) + for { + time.Sleep(5 * time.Second) // hang forever, until killed + } + } + fmt.Fprintf(rw, "test=Hello CGI-in-CGI\n") + for k, vv := range req.Form { + for _, v := range vv { + fmt.Fprintf(rw, "param-%s=%s\n", k, v) + } + } + for _, kv := range os.Environ() { + fmt.Fprintf(rw, "env-%s\n", kv) + } + })) + os.Exit(0) +} diff --git a/cgi/host_test.go b/cgi/host_test.go index f310a83d..78e05d59 100644 --- a/cgi/host_test.go +++ b/cgi/host_test.go @@ -15,7 +15,6 @@ import ( "net/http" "net/http/httptest" "os" - "os/exec" "path/filepath" "reflect" "runtime" @@ -25,6 +24,18 @@ import ( "time" ) +// TestMain executes the test binary as the cgi server if +// SERVER_SOFTWARE is set, and runs the tests otherwise. +func TestMain(m *testing.M) { + // SERVER_SOFTWARE swap variable is set when starting the cgi server. + if os.Getenv("SERVER_SOFTWARE") != "" { + cgiMain() + os.Exit(0) + } + + os.Exit(m.Run()) +} + func newRequest(httpreq string) *http.Request { buf := bufio.NewReader(strings.NewReader(httpreq)) req, err := http.ReadRequest(buf) @@ -89,24 +100,10 @@ readlines: } } -var cgiTested, cgiWorks bool - -func check(t *testing.T) { - if !cgiTested { - cgiTested = true - cgiWorks = testenv.Command(t, "./testdata/test.cgi").Run() == nil - } - if !cgiWorks { - // No Perl on Windows, needed by test.cgi - // TODO: make the child process be Go, not Perl. - t.Skip("Skipping test: test.cgi failed.") - } -} - func TestCGIBasicGet(t *testing.T) { - check(t) + testenv.MustHaveExec(t) h := &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], Root: "/test.cgi", } expectedMap := map[string]string{ @@ -122,7 +119,7 @@ func TestCGIBasicGet(t *testing.T) { "env-REMOTE_PORT": "1234", "env-REQUEST_METHOD": "GET", "env-REQUEST_URI": "/test.cgi?foo=bar&a=b", - "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_FILENAME": os.Args[0], "env-SCRIPT_NAME": "/test.cgi", "env-SERVER_NAME": "example.com", "env-SERVER_PORT": "80", @@ -139,9 +136,9 @@ func TestCGIBasicGet(t *testing.T) { } func TestCGIEnvIPv6(t *testing.T) { - check(t) + testenv.MustHaveExec(t) h := &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], Root: "/test.cgi", } expectedMap := map[string]string{ @@ -157,7 +154,7 @@ func TestCGIEnvIPv6(t *testing.T) { "env-REMOTE_PORT": "12345", "env-REQUEST_METHOD": "GET", "env-REQUEST_URI": "/test.cgi?foo=bar&a=b", - "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_FILENAME": os.Args[0], "env-SCRIPT_NAME": "/test.cgi", "env-SERVER_NAME": "example.com", "env-SERVER_PORT": "80", @@ -172,27 +169,27 @@ func TestCGIEnvIPv6(t *testing.T) { } func TestCGIBasicGetAbsPath(t *testing.T) { - check(t) - pwd, err := os.Getwd() + absPath, err := filepath.Abs(os.Args[0]) if err != nil { - t.Fatalf("getwd error: %v", err) + t.Fatal(err) } + testenv.MustHaveExec(t) h := &Handler{ - Path: pwd + "/testdata/test.cgi", + Path: absPath, Root: "/test.cgi", } expectedMap := map[string]string{ "env-REQUEST_URI": "/test.cgi?foo=bar&a=b", - "env-SCRIPT_FILENAME": pwd + "/testdata/test.cgi", + "env-SCRIPT_FILENAME": absPath, "env-SCRIPT_NAME": "/test.cgi", } runCgiTest(t, h, "GET /test.cgi?foo=bar&a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) } func TestPathInfo(t *testing.T) { - check(t) + testenv.MustHaveExec(t) h := &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], Root: "/test.cgi", } expectedMap := map[string]string{ @@ -200,36 +197,36 @@ func TestPathInfo(t *testing.T) { "env-PATH_INFO": "/extrapath", "env-QUERY_STRING": "a=b", "env-REQUEST_URI": "/test.cgi/extrapath?a=b", - "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_FILENAME": os.Args[0], "env-SCRIPT_NAME": "/test.cgi", } runCgiTest(t, h, "GET /test.cgi/extrapath?a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) } func TestPathInfoDirRoot(t *testing.T) { - check(t) + testenv.MustHaveExec(t) h := &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], Root: "/myscript//", } expectedMap := map[string]string{ "env-PATH_INFO": "/bar", "env-QUERY_STRING": "a=b", "env-REQUEST_URI": "/myscript/bar?a=b", - "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_FILENAME": os.Args[0], "env-SCRIPT_NAME": "/myscript", } runCgiTest(t, h, "GET /myscript/bar?a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) } func TestDupHeaders(t *testing.T) { - check(t) + testenv.MustHaveExec(t) h := &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], } expectedMap := map[string]string{ "env-REQUEST_URI": "/myscript/bar?a=b", - "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_FILENAME": os.Args[0], "env-HTTP_COOKIE": "nom=NOM; yum=YUM", "env-HTTP_X_FOO": "val1, val2", } @@ -246,13 +243,13 @@ func TestDupHeaders(t *testing.T) { // Verify we don't set the HTTP_PROXY environment variable. // Hope nobody was depending on it. It's not a known header, though. func TestDropProxyHeader(t *testing.T) { - check(t) + testenv.MustHaveExec(t) h := &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], } expectedMap := map[string]string{ "env-REQUEST_URI": "/myscript/bar?a=b", - "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_FILENAME": os.Args[0], "env-HTTP_X_FOO": "a", } runCgiTest(t, h, "GET /myscript/bar?a=b HTTP/1.0\n"+ @@ -268,23 +265,23 @@ func TestDropProxyHeader(t *testing.T) { } func TestPathInfoNoRoot(t *testing.T) { - check(t) + testenv.MustHaveExec(t) h := &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], Root: "", } expectedMap := map[string]string{ "env-PATH_INFO": "/bar", "env-QUERY_STRING": "a=b", "env-REQUEST_URI": "/bar?a=b", - "env-SCRIPT_FILENAME": "testdata/test.cgi", + "env-SCRIPT_FILENAME": os.Args[0], "env-SCRIPT_NAME": "", } runCgiTest(t, h, "GET /bar?a=b HTTP/1.0\nHost: example.com\n\n", expectedMap) } func TestCGIBasicPost(t *testing.T) { - check(t) + testenv.MustHaveExec(t) postReq := `POST /test.cgi?a=b HTTP/1.0 Host: example.com Content-Type: application/x-www-form-urlencoded @@ -292,7 +289,7 @@ Content-Length: 15 postfoo=postbar` h := &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], Root: "/test.cgi", } expectedMap := map[string]string{ @@ -311,7 +308,7 @@ func chunk(s string) string { // The CGI spec doesn't allow chunked requests. func TestCGIPostChunked(t *testing.T) { - check(t) + testenv.MustHaveExec(t) postReq := `POST /test.cgi?a=b HTTP/1.1 Host: example.com Content-Type: application/x-www-form-urlencoded @@ -320,7 +317,7 @@ Transfer-Encoding: chunked ` + chunk("postfoo") + chunk("=") + chunk("postbar") + chunk("") h := &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], Root: "/test.cgi", } expectedMap := map[string]string{} @@ -332,9 +329,9 @@ Transfer-Encoding: chunked } func TestRedirect(t *testing.T) { - check(t) + testenv.MustHaveExec(t) h := &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], Root: "/test.cgi", } rec := runCgiTest(t, h, "GET /test.cgi?loc=http://foo.com/ HTTP/1.0\nHost: example.com\n\n", nil) @@ -347,13 +344,13 @@ func TestRedirect(t *testing.T) { } func TestInternalRedirect(t *testing.T) { - check(t) + testenv.MustHaveExec(t) baseHandler := http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { fmt.Fprintf(rw, "basepath=%s\n", req.URL.Path) fmt.Fprintf(rw, "remoteaddr=%s\n", req.RemoteAddr) }) h := &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], Root: "/test.cgi", PathLocationHandler: baseHandler, } @@ -367,12 +364,12 @@ func TestInternalRedirect(t *testing.T) { // TestCopyError tests that we kill the process if there's an error copying // its output. (for example, from the client having gone away) func TestCopyError(t *testing.T) { - check(t) + testenv.MustHaveExec(t) if runtime.GOOS == "windows" { t.Skipf("skipping test on %q", runtime.GOOS) } h := &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], Root: "/test.cgi", } ts := httptest.NewServer(h) @@ -427,14 +424,11 @@ func TestCopyError(t *testing.T) { } } -func TestDirUnix(t *testing.T) { - check(t) - if runtime.GOOS == "windows" { - t.Skipf("skipping test on %q", runtime.GOOS) - } +func TestDir(t *testing.T) { + testenv.MustHaveExec(t) cwd, _ := os.Getwd() h := &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], Root: "/test.cgi", Dir: cwd, } @@ -444,9 +438,9 @@ func TestDirUnix(t *testing.T) { runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) cwd, _ = os.Getwd() - cwd = filepath.Join(cwd, "testdata") + cwd, _ = filepath.Split(os.Args[0]) h = &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], Root: "/test.cgi", } expectedMap = map[string]string{ @@ -455,75 +449,15 @@ func TestDirUnix(t *testing.T) { runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) } -func findPerl(t *testing.T) string { - t.Helper() - perl, err := exec.LookPath("perl") - if err != nil { - t.Skip("Skipping test: perl not found.") - } - perl, _ = filepath.Abs(perl) - - cmd := testenv.Command(t, perl, "-e", "print 123") - cmd.Env = []string{"PATH=/garbage"} - out, err := cmd.Output() - if err != nil || string(out) != "123" { - t.Skipf("Skipping test: %s is not functional", perl) - } - return perl -} - -func TestDirWindows(t *testing.T) { - if runtime.GOOS != "windows" { - t.Skip("Skipping windows specific test.") - } - - cgifile, _ := filepath.Abs("testdata/test.cgi") - - perl := findPerl(t) - - cwd, _ := os.Getwd() - h := &Handler{ - Path: perl, - Root: "/test.cgi", - Dir: cwd, - Args: []string{cgifile}, - Env: []string{"SCRIPT_FILENAME=" + cgifile}, - } - expectedMap := map[string]string{ - "cwd": cwd, - } - runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) - - // If not specify Dir on windows, working directory should be - // base directory of perl. - cwd, _ = filepath.Split(perl) - if cwd != "" && cwd[len(cwd)-1] == filepath.Separator { - cwd = cwd[:len(cwd)-1] - } - h = &Handler{ - Path: perl, - Root: "/test.cgi", - Args: []string{cgifile}, - Env: []string{"SCRIPT_FILENAME=" + cgifile}, - } - expectedMap = map[string]string{ - "cwd": cwd, - } - runCgiTest(t, h, "GET /test.cgi HTTP/1.0\nHost: example.com\n\n", expectedMap) -} - func TestEnvOverride(t *testing.T) { - check(t) + testenv.MustHaveExec(t) cgifile, _ := filepath.Abs("testdata/test.cgi") - perl := findPerl(t) - cwd, _ := os.Getwd() h := &Handler{ - Path: perl, + Path: os.Args[0], Root: "/test.cgi", Dir: cwd, - Args: []string{cgifile}, Env: []string{ "SCRIPT_FILENAME=" + cgifile, "REQUEST_URI=/foo/bar", @@ -539,10 +473,10 @@ func TestEnvOverride(t *testing.T) { } func TestHandlerStderr(t *testing.T) { - check(t) + testenv.MustHaveExec(t) var stderr strings.Builder h := &Handler{ - Path: "testdata/test.cgi", + Path: os.Args[0], Root: "/test.cgi", Stderr: &stderr, } diff --git a/cgi/integration_test.go b/cgi/integration_test.go index 4890ae07..68f908e2 100644 --- a/cgi/integration_test.go +++ b/cgi/integration_test.go @@ -20,7 +20,6 @@ import ( "os" "strings" "testing" - "time" ) // This test is a CGI host (testing host.go) that runs its own binary @@ -31,7 +30,6 @@ func TestHostingOurselves(t *testing.T) { h := &Handler{ Path: os.Args[0], Root: "/test.go", - Args: []string{"-test.run=^TestBeChildCGIProcess$"}, } expectedMap := map[string]string{ "test": "Hello CGI-in-CGI", @@ -98,9 +96,8 @@ func TestKillChildAfterCopyError(t *testing.T) { h := &Handler{ Path: os.Args[0], Root: "/test.go", - Args: []string{"-test.run=^TestBeChildCGIProcess$"}, } - req, _ := http.NewRequest("GET", "http://example.com/test.cgi?write-forever=1", nil) + req, _ := http.NewRequest("GET", "http://example.com/test.go?write-forever=1", nil) rec := httptest.NewRecorder() var out bytes.Buffer const writeLen = 50 << 10 @@ -120,7 +117,6 @@ func TestChildOnlyHeaders(t *testing.T) { h := &Handler{ Path: os.Args[0], Root: "/test.go", - Args: []string{"-test.run=^TestBeChildCGIProcess$"}, } expectedMap := map[string]string{ "_body": "", @@ -139,7 +135,6 @@ func TestNilRequestBody(t *testing.T) { h := &Handler{ Path: os.Args[0], Root: "/test.go", - Args: []string{"-test.run=^TestBeChildCGIProcess$"}, } expectedMap := map[string]string{ "nil-request-body": "false", @@ -154,7 +149,6 @@ func TestChildContentType(t *testing.T) { h := &Handler{ Path: os.Args[0], Root: "/test.go", - Args: []string{"-test.run=^TestBeChildCGIProcess$"}, } var tests = []struct { name string @@ -202,7 +196,6 @@ func want500Test(t *testing.T, path string) { h := &Handler{ Path: os.Args[0], Root: "/test.go", - Args: []string{"-test.run=^TestBeChildCGIProcess$"}, } expectedMap := map[string]string{ "_body": "", @@ -212,61 +205,3 @@ func want500Test(t *testing.T, path string) { t.Errorf("Got code %d; want 500", replay.Code) } } - -type neverEnding byte - -func (b neverEnding) Read(p []byte) (n int, err error) { - for i := range p { - p[i] = byte(b) - } - return len(p), nil -} - -// Note: not actually a test. -func TestBeChildCGIProcess(t *testing.T) { - if os.Getenv("REQUEST_METHOD") == "" { - // Not in a CGI environment; skipping test. - return - } - switch os.Getenv("REQUEST_URI") { - case "/immediate-disconnect": - os.Exit(0) - case "/no-content-type": - fmt.Printf("Content-Length: 6\n\nHello\n") - os.Exit(0) - case "/empty-headers": - fmt.Printf("\nHello") - os.Exit(0) - } - Serve(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - if req.FormValue("nil-request-body") == "1" { - fmt.Fprintf(rw, "nil-request-body=%v\n", req.Body == nil) - return - } - rw.Header().Set("X-Test-Header", "X-Test-Value") - req.ParseForm() - if req.FormValue("no-body") == "1" { - return - } - if eb, ok := req.Form["exact-body"]; ok { - io.WriteString(rw, eb[0]) - return - } - if req.FormValue("write-forever") == "1" { - io.Copy(rw, neverEnding('a')) - for { - time.Sleep(5 * time.Second) // hang forever, until killed - } - } - fmt.Fprintf(rw, "test=Hello CGI-in-CGI\n") - for k, vv := range req.Form { - for _, v := range vv { - fmt.Fprintf(rw, "param-%s=%s\n", k, v) - } - } - for _, kv := range os.Environ() { - fmt.Fprintf(rw, "env-%s\n", kv) - } - })) - os.Exit(0) -} diff --git a/cgi/testdata/test.cgi b/cgi/testdata/test.cgi deleted file mode 100755 index 667fce21..00000000 --- a/cgi/testdata/test.cgi +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/perl -# Copyright 2011 The Go Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. -# -# Test script run as a child process under cgi_test.go - -use strict; -use Cwd; - -binmode STDOUT; - -my $q = MiniCGI->new; -my $params = $q->Vars; - -if ($params->{"loc"}) { - print "Location: $params->{loc}\r\n\r\n"; - exit(0); -} - -print "Content-Type: text/html\r\n"; -print "X-CGI-Pid: $$\r\n"; -print "X-Test-Header: X-Test-Value\r\n"; -print "\r\n"; - -if ($params->{"writestderr"}) { - print STDERR "Hello, stderr!\n"; -} - -if ($params->{"bigresponse"}) { - # 17 MB, for OS X: golang.org/issue/4958 - for (1..(17 * 1024)) { - print "A" x 1024, "\r\n"; - } - exit 0; -} - -print "test=Hello CGI\r\n"; - -foreach my $k (sort keys %$params) { - print "param-$k=$params->{$k}\r\n"; -} - -foreach my $k (sort keys %ENV) { - my $clean_env = $ENV{$k}; - $clean_env =~ s/[\n\r]//g; - print "env-$k=$clean_env\r\n"; -} - -# NOTE: msys perl returns /c/go/src/... not C:\go\.... -my $dir = getcwd(); -if ($^O eq 'MSWin32' || $^O eq 'msys' || $^O eq 'cygwin') { - if ($dir =~ /^.:/) { - $dir =~ s!/!\\!g; - } else { - my $cmd = $ENV{'COMSPEC'} || 'c:\\windows\\system32\\cmd.exe'; - $cmd =~ s!\\!/!g; - $dir = `$cmd /c cd`; - chomp $dir; - } -} -print "cwd=$dir\r\n"; - -# A minimal version of CGI.pm, for people without the perl-modules -# package installed. (CGI.pm used to be part of the Perl core, but -# some distros now bundle perl-base and perl-modules separately...) -package MiniCGI; - -sub new { - my $class = shift; - return bless {}, $class; -} - -sub Vars { - my $self = shift; - my $pairs; - if ($ENV{CONTENT_LENGTH}) { - $pairs = do { local $/; }; - } else { - $pairs = $ENV{QUERY_STRING}; - } - my $vars = {}; - foreach my $kv (split(/&/, $pairs)) { - my ($k, $v) = split(/=/, $kv, 2); - $vars->{_urldecode($k)} = _urldecode($v); - } - return $vars; -} - -sub _urldecode { - my $v = shift; - $v =~ tr/+/ /; - $v =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C", hex($1))/eg; - return $v; -} From 61e2401b6584b3d9ac6b87e38f211f72fb364f3b Mon Sep 17 00:00:00 2001 From: wulianglongrd Date: Sun, 29 Oct 2023 22:57:22 +0800 Subject: [PATCH 69/93] net/http/cookiejar: remove unused variable The errNoHostname variable is not used, delete it. Change-Id: I62ca6390fd026e6a8cb1e8147f3fbfc3078c2249 Reviewed-on: https://go-review.googlesource.com/c/go/+/538455 Reviewed-by: Damien Neil Auto-Submit: Damien Neil LUCI-TryBot-Result: Go LUCI Reviewed-by: Bryan Mills --- cookiejar/jar.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cookiejar/jar.go b/cookiejar/jar.go index 273b54c8..46d11939 100644 --- a/cookiejar/jar.go +++ b/cookiejar/jar.go @@ -440,7 +440,6 @@ func (j *Jar) newEntry(c *http.Cookie, now time.Time, defPath, host string) (e e var ( errIllegalDomain = errors.New("cookiejar: illegal cookie domain attribute") errMalformedDomain = errors.New("cookiejar: malformed cookie domain attribute") - errNoHostname = errors.New("cookiejar: no host name available (IP only)") ) // endOfTime is the time when session (non-persistent) cookies expire. From 943f1d81c12c886a7b7275584249ca1d5c4eb874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Matczuk?= Date: Thu, 9 Nov 2023 09:43:26 +0000 Subject: [PATCH 70/93] net/http: use copyBufPool in transferWriter.doBodyCopy() This is a followup to CL 14177. It applies copyBufPool optimization to transferWriter.doBodyCopy(). The function is used every time Request or Response is written. Without this patch for every Request and Response processed, if there is a body, we need to allocate and GC a 32k buffer. This is quickly causing GC pressure. Fixes #57202 Change-Id: I4c30e1737726ac8d9937846106efd02effbae300 GitHub-Last-Rev: 908573cdbe2e8b6f91ce026cf8632ff5f2c41110 GitHub-Pull-Request: golang/go#57205 Reviewed-on: https://go-review.googlesource.com/c/go/+/456435 Reviewed-by: Damien Neil Reviewed-by: Bryan Mills TryBot-Result: Gopher Robot LUCI-TryBot-Result: Go LUCI Reviewed-by: qiulaidongfeng <2645477756@qq.com> --- transfer.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/transfer.go b/transfer.go index b2499817..dffff56b 100644 --- a/transfer.go +++ b/transfer.go @@ -410,7 +410,11 @@ func (t *transferWriter) writeBody(w io.Writer) (err error) { // // This function is only intended for use in writeBody. func (t *transferWriter) doBodyCopy(dst io.Writer, src io.Reader) (n int64, err error) { - n, err = io.Copy(dst, src) + bufp := copyBufPool.Get().(*[]byte) + buf := *bufp + defer copyBufPool.Put(bufp) + + n, err = io.CopyBuffer(dst, src, buf) if err != nil && err != io.EOF { t.bodyReadError = err } From 0d035c0cdbec7f382f0b1a5b75191e4ba018f052 Mon Sep 17 00:00:00 2001 From: Roland Shoemaker Date: Fri, 10 Nov 2023 10:42:42 -0800 Subject: [PATCH 71/93] crypto/tls: remove RSA KEX ciphers from the default list Removes the RSA KEX based ciphers from the default list. This can be reverted using the tlsrsakex GODEBUG. Fixes #63413 Change-Id: Id221be3eb2f6c24b91039d380313f0c87d339f98 Reviewed-on: https://go-review.googlesource.com/c/go/+/541517 LUCI-TryBot-Result: Go LUCI Reviewed-by: Damien Neil --- client_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client_test.go b/client_test.go index df2a670a..7459b9cb 100644 --- a/client_test.go +++ b/client_test.go @@ -946,7 +946,7 @@ func testResponseSetsTLSConnectionState(t *testing.T, mode testMode) { c := ts.Client() tr := c.Transport.(*Transport) - tr.TLSClientConfig.CipherSuites = []uint16{tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA} + tr.TLSClientConfig.CipherSuites = []uint16{tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA} tr.TLSClientConfig.MaxVersion = tls.VersionTLS12 // to get to pick the cipher suite tr.Dial = func(netw, addr string) (net.Conn, error) { return net.Dial(netw, ts.Listener.Addr().String()) @@ -959,7 +959,7 @@ func testResponseSetsTLSConnectionState(t *testing.T, mode testMode) { if res.TLS == nil { t.Fatal("Response didn't set TLS Connection State.") } - if got, want := res.TLS.CipherSuite, tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA; got != want { + if got, want := res.TLS.CipherSuite, tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA; got != want { t.Errorf("TLS Cipher Suite = %d; want %d", got, want) } } From 5e526452661f4f8d965f74c311ef8b28a6572305 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 15 Nov 2023 09:45:16 -0800 Subject: [PATCH 72/93] net/http: don't set length for non-range encoded content requests Historically, serveContent has not set Content-Length when the user provides Content-Encoding. This causes broken responses when the user sets both Content-Length and Content-Encoding, and the request is a range request, because the returned data doesn't match the declared length. CL 381956 fixed this case by changing serveContent to always set a Content-Length header. Unfortunately, I've discovered multiple cases in the wild of users setting Content-Encoding: gzip and passing serveContent a ResponseWriter wrapper that gzips the data written to it. This breaks serveContent in a number of ways. In particular, there's no way for it to respond to Range requests properly, because it doesn't know the recipient's view of the content. What the user should be doing in this case is just using io.Copy to send the gzipped data to the response. Or possibly setting Transfer-Encoding: gzip. But whatever they should be doing, what they are doing has mostly worked for non-Range requests, and setting Content-Length makes it stop working because the length of the file being served doesn't match the number of bytes being sent. So in the interests of not breaking users (even if they're misusing serveContent in ways that are already broken), partially revert CL 381956. For non-Range requests, don't set Content-Length when the user has set Content-Encoding. This matches our previous behavior and causes minimal harm in cases where we could have set Content-Length. (We will send using chunked encoding rather than identity, but that's fine.) For Range requests, set Content-Length unconditionally. Either the user isn't mangling the data in the ResponseWriter, in which case the length is correct, or they are, in which case the response isn't going to contain the right bytes anyway. (Note that a Range request for a Content-Length: gzip file is requesting a range of *gzipped* bytes, not a range from the uncompressed file.) Change-Id: I5e788e6756f34cee520aa7c456826f462a59f7eb Reviewed-on: https://go-review.googlesource.com/c/go/+/542595 Auto-Submit: Damien Neil LUCI-TryBot-Result: Go LUCI Reviewed-by: Jonathan Amsterdam --- fs.go | 29 ++++++++++++++++++++++++- fs_test.go | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/fs.go b/fs.go index 20da5600..ace74a7b 100644 --- a/fs.go +++ b/fs.go @@ -343,8 +343,35 @@ func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, } w.Header().Set("Accept-Ranges", "bytes") - w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) + // We should be able to unconditionally set the Content-Length here. + // + // However, there is a pattern observed in the wild that this breaks: + // The user wraps the ResponseWriter in one which gzips data written to it, + // and sets "Content-Encoding: gzip". + // + // The user shouldn't be doing this; the serveContent path here depends + // on serving seekable data with a known length. If you want to compress + // on the fly, then you shouldn't be using ServeFile/ServeContent, or + // you should compress the entire file up-front and provide a seekable + // view of the compressed data. + // + // However, since we've observed this pattern in the wild, and since + // setting Content-Length here breaks code that mostly-works today, + // skip setting Content-Length if the user set Content-Encoding. + // + // If this is a range request, always set Content-Length. + // If the user isn't changing the bytes sent in the ResponseWrite, + // the Content-Length will be correct. + // If the user is changing the bytes sent, then the range request wasn't + // going to work properly anyway and we aren't worse off. + // + // A possible future improvement on this might be to look at the type + // of the ResponseWriter, and always set Content-Length if it's one + // that we recognize. + if len(ranges) > 0 || w.Header().Get("Content-Encoding") == "" { + w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) + } w.WriteHeader(code) if r.Method != "HEAD" { diff --git a/fs_test.go b/fs_test.go index d29664c1..861e70ca 100644 --- a/fs_test.go +++ b/fs_test.go @@ -7,6 +7,7 @@ package http_test import ( "bufio" "bytes" + "compress/gzip" "errors" "fmt" "internal/testenv" @@ -15,6 +16,7 @@ import ( "mime" "mime/multipart" "net" + "net/http" . "net/http" "net/http/httptest" "net/url" @@ -571,7 +573,7 @@ func testServeDirWithoutTrailingSlash(t *testing.T, mode testMode) { } } -// Tests that ServeFile adds a Content-Length even if a Content-Encoding is +// Tests that ServeFile doesn't add a Content-Length if a Content-Encoding is // specified. func TestServeFileWithContentEncoding(t *testing.T) { run(t, testServeFileWithContentEncoding) } func testServeFileWithContentEncoding(t *testing.T, mode testMode) { @@ -593,7 +595,7 @@ func testServeFileWithContentEncoding(t *testing.T, mode testMode) { t.Fatal(err) } resp.Body.Close() - if g, e := resp.ContentLength, int64(11); g != e { + if g, e := resp.ContentLength, int64(-1); g != e { t.Errorf("Content-Length mismatch: got %d, want %d", g, e) } } @@ -1609,3 +1611,60 @@ func TestServeFileFS(t *testing.T) { } res.Body.Close() } + +func TestServeFileZippingResponseWriter(t *testing.T) { + // This test exercises a pattern which is incorrect, + // but has been observed enough in the world that we don't want to break it. + // + // The server is setting "Content-Encoding: gzip", + // wrapping the ResponseWriter in an implementation which gzips data written to it, + // and passing this ResponseWriter to ServeFile. + // + // This means ServeFile cannot properly set a Content-Length header, because it + // doesn't know what content it is going to send--the ResponseWriter is modifying + // the bytes sent. + // + // Range requests are always going to be broken in this scenario, + // but verify that we can serve non-range requests correctly. + filename := "index.html" + contents := []byte("contents will be sent with Content-Encoding: gzip") + fsys := fstest.MapFS{ + filename: {Data: contents}, + } + ts := newClientServerTest(t, http1Mode, HandlerFunc(func(w ResponseWriter, r *Request) { + w.Header().Set("Content-Encoding", "gzip") + gzw := gzip.NewWriter(w) + defer gzw.Close() + ServeFileFS(gzipResponseWriter{w: gzw, ResponseWriter: w}, r, fsys, filename) + })).ts + defer ts.Close() + + res, err := ts.Client().Get(ts.URL + "/" + filename) + if err != nil { + t.Fatal(err) + } + b, err := io.ReadAll(res.Body) + if err != nil { + t.Fatal("reading Body:", err) + } + if s := string(b); s != string(contents) { + t.Errorf("for path %q got %q, want %q", filename, s, contents) + } + res.Body.Close() +} + +type gzipResponseWriter struct { + ResponseWriter + w *gzip.Writer +} + +func (grw gzipResponseWriter) Write(b []byte) (int, error) { + return grw.w.Write(b) +} + +func (grw gzipResponseWriter) Flush() { + grw.w.Flush() + if fw, ok := grw.ResponseWriter.(http.Flusher); ok { + fw.Flush() + } +} From abd13dd85fca6153247c46ec262cbec37c956535 Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Tue, 28 Feb 2023 16:39:15 +0800 Subject: [PATCH 73/93] net,os: arrange zero-copy of os.File and net.TCPConn to net.UnixConn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #58808 goos: linux goarch: amd64 pkg: net cpu: DO-Premium-Intel │ old │ new │ │ sec/op │ sec/op vs base │ Splice/tcp-to-unix/1024-4 3.783µ ± 10% 3.201µ ± 7% -15.40% (p=0.001 n=10) Splice/tcp-to-unix/2048-4 3.967µ ± 13% 3.818µ ± 16% ~ (p=0.971 n=10) Splice/tcp-to-unix/4096-4 4.988µ ± 16% 4.590µ ± 11% ~ (p=0.089 n=10) Splice/tcp-to-unix/8192-4 6.981µ ± 13% 5.236µ ± 9% -25.00% (p=0.000 n=10) Splice/tcp-to-unix/16384-4 10.192µ ± 9% 7.350µ ± 7% -27.89% (p=0.000 n=10) Splice/tcp-to-unix/32768-4 19.65µ ± 13% 10.28µ ± 16% -47.69% (p=0.000 n=10) Splice/tcp-to-unix/65536-4 41.89µ ± 18% 15.70µ ± 13% -62.52% (p=0.000 n=10) Splice/tcp-to-unix/131072-4 90.05µ ± 11% 29.55µ ± 10% -67.18% (p=0.000 n=10) Splice/tcp-to-unix/262144-4 170.24µ ± 15% 52.66µ ± 4% -69.06% (p=0.000 n=10) Splice/tcp-to-unix/524288-4 326.4µ ± 13% 109.3µ ± 11% -66.52% (p=0.000 n=10) Splice/tcp-to-unix/1048576-4 651.4µ ± 9% 228.3µ ± 14% -64.95% (p=0.000 n=10) geomean 29.42µ 15.62µ -46.90% │ old │ new │ │ B/s │ B/s vs base │ Splice/tcp-to-unix/1024-4 258.2Mi ± 11% 305.2Mi ± 8% +18.21% (p=0.001 n=10) Splice/tcp-to-unix/2048-4 492.5Mi ± 15% 511.7Mi ± 13% ~ (p=0.971 n=10) Splice/tcp-to-unix/4096-4 783.5Mi ± 14% 851.2Mi ± 12% ~ (p=0.089 n=10) Splice/tcp-to-unix/8192-4 1.093Gi ± 11% 1.458Gi ± 8% +33.36% (p=0.000 n=10) Splice/tcp-to-unix/16384-4 1.497Gi ± 9% 2.076Gi ± 7% +38.67% (p=0.000 n=10) Splice/tcp-to-unix/32768-4 1.553Gi ± 11% 2.969Gi ± 14% +91.17% (p=0.000 n=10) Splice/tcp-to-unix/65536-4 1.458Gi ± 23% 3.888Gi ± 11% +166.69% (p=0.000 n=10) Splice/tcp-to-unix/131072-4 1.356Gi ± 10% 4.131Gi ± 9% +204.72% (p=0.000 n=10) Splice/tcp-to-unix/262144-4 1.434Gi ± 13% 4.637Gi ± 4% +223.32% (p=0.000 n=10) Splice/tcp-to-unix/524288-4 1.497Gi ± 15% 4.468Gi ± 10% +198.47% (p=0.000 n=10) Splice/tcp-to-unix/1048576-4 1.501Gi ± 10% 4.277Gi ± 16% +184.88% (p=0.000 n=10) geomean 1.038Gi 1.954Gi +88.28% │ old │ new │ │ B/op │ B/op vs base │ Splice/tcp-to-unix/1024-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/2048-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/4096-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/8192-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/16384-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/32768-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/65536-4 1.000 ± ? 0.000 ± 0% -100.00% (p=0.001 n=10) Splice/tcp-to-unix/131072-4 2.000 ± 0% 0.000 ± 0% -100.00% (p=0.000 n=10) Splice/tcp-to-unix/262144-4 4.000 ± 25% 0.000 ± 0% -100.00% (p=0.000 n=10) Splice/tcp-to-unix/524288-4 7.500 ± 33% 0.000 ± 0% -100.00% (p=0.000 n=10) Splice/tcp-to-unix/1048576-4 17.00 ± 12% 0.00 ± 0% -100.00% (p=0.000 n=10) geomean ² ? ² ³ ¹ all samples are equal ² summaries must be >0 to compute geomean ³ ratios must be >0 to compute geomean │ old │ new │ │ allocs/op │ allocs/op vs base │ Splice/tcp-to-unix/1024-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/2048-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/4096-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/8192-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/16384-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/32768-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/65536-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/131072-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/262144-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/524288-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Splice/tcp-to-unix/1048576-4 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ geomean ² +0.00% ² ¹ all samples are equal ² summaries must be >0 to compute geomean Change-Id: I829061b009a0929a8ef1a15c183793c0b9104dde Reviewed-on: https://go-review.googlesource.com/c/go/+/472475 Reviewed-by: Damien Neil Reviewed-by: Bryan Mills LUCI-TryBot-Result: Go LUCI --- transfer_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/transfer_test.go b/transfer_test.go index 3f9ebdea..b1a5a931 100644 --- a/transfer_test.go +++ b/transfer_test.go @@ -264,6 +264,12 @@ func TestTransferWriterWriteBodyReaderTypes(t *testing.T) { actualReader = reflect.TypeOf(lr.R) } else { actualReader = reflect.TypeOf(mw.CalledReader) + // We have to handle this special case for genericWriteTo in os, + // this struct is introduced to support a zero-copy optimization, + // check out https://go.dev/issue/58808 for details. + if actualReader.Kind() == reflect.Struct && actualReader.PkgPath() == "os" && actualReader.Name() == "fileWithoutWriteTo" { + actualReader = actualReader.Field(1).Type + } } if tc.expectedReader != actualReader { From 9704e81d17b9a5b0b7664976d536a8bbd0e8b2d7 Mon Sep 17 00:00:00 2001 From: Jes Cok Date: Mon, 13 Nov 2023 08:57:14 +0000 Subject: [PATCH 74/93] all: add missing copyright header Change-Id: Ic61fb181923159e80a86a41582e83ec466ab9bc4 GitHub-Last-Rev: 92469845665fa1f864d257c8bc175201a43b4d43 GitHub-Pull-Request: golang/go#64080 Reviewed-on: https://go-review.googlesource.com/c/go/+/541741 TryBot-Result: Gopher Robot Reviewed-by: Michael Pratt Reviewed-by: Than McIntosh Run-TryBot: Jes Cok --- responsecontroller_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/responsecontroller_test.go b/responsecontroller_test.go index 5828f379..f1dcc79e 100644 --- a/responsecontroller_test.go +++ b/responsecontroller_test.go @@ -1,3 +1,7 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package http_test import ( From 4efbfb2d47ab7edc772036fdc933fd3ec49f3bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Sat, 18 Nov 2023 11:58:34 +0000 Subject: [PATCH 75/93] src: a/an grammar fixes Change-Id: I179b50ae8e73677d4d408b83424afbbfe6aa17a1 GitHub-Last-Rev: 2e2d9c1e45556155d02db4df381b99f2d1bc5c0e GitHub-Pull-Request: golang/go#63478 Reviewed-on: https://go-review.googlesource.com/c/go/+/534015 Auto-Submit: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov LUCI-TryBot-Result: Go LUCI Reviewed-by: Michael Knyszek --- transport.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transport.go b/transport.go index 1cf41a54..170ba86c 100644 --- a/transport.go +++ b/transport.go @@ -237,7 +237,7 @@ type Transport struct { // TLSNextProto specifies how the Transport switches to an // alternate protocol (such as HTTP/2) after a TLS ALPN - // protocol negotiation. If Transport dials an TLS connection + // protocol negotiation. If Transport dials a TLS connection // with a non-empty protocol name and TLSNextProto contains a // map entry for that key (such as "h2"), then the func is // called with the request's authority (such as "example.com" From 4e0c9ea1af3447ebc93587a38039af57593c23bb Mon Sep 17 00:00:00 2001 From: Jorropo Date: Wed, 8 Nov 2023 16:08:26 +0100 Subject: [PATCH 76/93] net/http: use pointers to array for copyBufPool This is inspired by CL 539915, I'm only submitting now that CL 456435 has been merged. This divide the number of objects kept alive by the heap by two and remove the slice header allocation in New and in the put back. Change-Id: Ibcd5166bac5a37f365a533e09a28f3b79f81ad58 Reviewed-on: https://go-review.googlesource.com/c/go/+/543515 Reviewed-by: Damien Neil Auto-Submit: Damien Neil Reviewed-by: qiulaidongfeng <2645477756@qq.com> LUCI-TryBot-Result: Go LUCI Reviewed-by: Michael Knyszek --- server.go | 22 ++++++++++++++-------- transfer.go | 5 ++--- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/server.go b/server.go index 7fa785df..36a03f4a 100644 --- a/server.go +++ b/server.go @@ -575,9 +575,8 @@ type writerOnly struct { // to a *net.TCPConn with sendfile, or from a supported src type such // as a *net.TCPConn on Linux with splice. func (w *response) ReadFrom(src io.Reader) (n int64, err error) { - bufp := copyBufPool.Get().(*[]byte) - buf := *bufp - defer copyBufPool.Put(bufp) + buf := getCopyBuf() + defer putCopyBuf(buf) // Our underlying w.conn.rwc is usually a *TCPConn (with its // own ReadFrom method). If not, just fall back to the normal @@ -807,11 +806,18 @@ var ( bufioWriter4kPool sync.Pool ) -var copyBufPool = sync.Pool{ - New: func() any { - b := make([]byte, 32*1024) - return &b - }, +const copyBufPoolSize = 32 * 1024 + +var copyBufPool = sync.Pool{New: func() any { return new([copyBufPoolSize]byte) }} + +func getCopyBuf() []byte { + return copyBufPool.Get().(*[copyBufPoolSize]byte)[:] +} +func putCopyBuf(b []byte) { + if len(b) != copyBufPoolSize { + panic("trying to put back buffer of the wrong size in the copyBufPool") + } + copyBufPool.Put((*[copyBufPoolSize]byte)(b)) } func bufioWriterPool(size int) *sync.Pool { diff --git a/transfer.go b/transfer.go index dffff56b..d7872584 100644 --- a/transfer.go +++ b/transfer.go @@ -410,9 +410,8 @@ func (t *transferWriter) writeBody(w io.Writer) (err error) { // // This function is only intended for use in writeBody. func (t *transferWriter) doBodyCopy(dst io.Writer, src io.Reader) (n int64, err error) { - bufp := copyBufPool.Get().(*[]byte) - buf := *bufp - defer copyBufPool.Put(bufp) + buf := getCopyBuf() + defer putCopyBuf(buf) n, err = io.CopyBuffer(dst, src, buf) if err != nil && err != io.EOF { From d90121bb4c066d8fa81fa2c034dd1af583a846b5 Mon Sep 17 00:00:00 2001 From: Dmitri Shuralyov Date: Wed, 29 Nov 2023 15:42:53 -0500 Subject: [PATCH 77/93] all: update vendored dependencies The Go 1.22 code freeze has recently started. This is a time to update all golang.org/x/... module versions that contribute packages to the std and cmd modules in the standard library to latest master versions. Generated with: go install golang.org/x/build/cmd/updatestd@latest go install golang.org/x/tools/cmd/bundle@latest updatestd -goroot=$(pwd) -branch=master For #36905. Change-Id: I76525261b9a954ed21a3bd3cb6c4a12e6c031d80 Cq-Include-Trybots: luci.golang.try:gotip-windows-amd64-longtest,gotip-linux-amd64-longtest,gotip-linux-386-longtest Reviewed-on: https://go-review.googlesource.com/c/go/+/546055 LUCI-TryBot-Result: Go LUCI Reviewed-by: Michael Knyszek Reviewed-by: Dmitri Shuralyov Auto-Submit: Dmitri Shuralyov --- h2_bundle.go | 150 +++++++++++++++++++++++---------------------------- 1 file changed, 66 insertions(+), 84 deletions(-) diff --git a/h2_bundle.go b/h2_bundle.go index fea33276..ac41144d 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -1041,41 +1041,44 @@ func http2shouldRetryDial(call *http2dialCall, req *Request) bool { // TODO: Benchmark to determine if the pools are necessary. The GC may have // improved enough that we can instead allocate chunks like this: // make([]byte, max(16<<10, expectedBytesRemaining)) -var ( - http2dataChunkSizeClasses = []int{ - 1 << 10, - 2 << 10, - 4 << 10, - 8 << 10, - 16 << 10, - } - http2dataChunkPools = [...]sync.Pool{ - {New: func() interface{} { return make([]byte, 1<<10) }}, - {New: func() interface{} { return make([]byte, 2<<10) }}, - {New: func() interface{} { return make([]byte, 4<<10) }}, - {New: func() interface{} { return make([]byte, 8<<10) }}, - {New: func() interface{} { return make([]byte, 16<<10) }}, - } -) +var http2dataChunkPools = [...]sync.Pool{ + {New: func() interface{} { return new([1 << 10]byte) }}, + {New: func() interface{} { return new([2 << 10]byte) }}, + {New: func() interface{} { return new([4 << 10]byte) }}, + {New: func() interface{} { return new([8 << 10]byte) }}, + {New: func() interface{} { return new([16 << 10]byte) }}, +} func http2getDataBufferChunk(size int64) []byte { - i := 0 - for ; i < len(http2dataChunkSizeClasses)-1; i++ { - if size <= int64(http2dataChunkSizeClasses[i]) { - break - } + switch { + case size <= 1<<10: + return http2dataChunkPools[0].Get().(*[1 << 10]byte)[:] + case size <= 2<<10: + return http2dataChunkPools[1].Get().(*[2 << 10]byte)[:] + case size <= 4<<10: + return http2dataChunkPools[2].Get().(*[4 << 10]byte)[:] + case size <= 8<<10: + return http2dataChunkPools[3].Get().(*[8 << 10]byte)[:] + default: + return http2dataChunkPools[4].Get().(*[16 << 10]byte)[:] } - return http2dataChunkPools[i].Get().([]byte) } func http2putDataBufferChunk(p []byte) { - for i, n := range http2dataChunkSizeClasses { - if len(p) == n { - http2dataChunkPools[i].Put(p) - return - } + switch len(p) { + case 1 << 10: + http2dataChunkPools[0].Put((*[1 << 10]byte)(p)) + case 2 << 10: + http2dataChunkPools[1].Put((*[2 << 10]byte)(p)) + case 4 << 10: + http2dataChunkPools[2].Put((*[4 << 10]byte)(p)) + case 8 << 10: + http2dataChunkPools[3].Put((*[8 << 10]byte)(p)) + case 16 << 10: + http2dataChunkPools[4].Put((*[16 << 10]byte)(p)) + default: + panic(fmt.Sprintf("unexpected buffer len=%v", len(p))) } - panic(fmt.Sprintf("unexpected buffer len=%v", len(p))) } // dataBuffer is an io.ReadWriter backed by a list of data chunks. @@ -3058,41 +3061,6 @@ func http2summarizeFrame(f http2Frame) string { return buf.String() } -func http2traceHasWroteHeaderField(trace *httptrace.ClientTrace) bool { - return trace != nil && trace.WroteHeaderField != nil -} - -func http2traceWroteHeaderField(trace *httptrace.ClientTrace, k, v string) { - if trace != nil && trace.WroteHeaderField != nil { - trace.WroteHeaderField(k, []string{v}) - } -} - -func http2traceGot1xxResponseFunc(trace *httptrace.ClientTrace) func(int, textproto.MIMEHeader) error { - if trace != nil { - return trace.Got1xxResponse - } - return nil -} - -// dialTLSWithContext uses tls.Dialer, added in Go 1.15, to open a TLS -// connection. -func (t *http2Transport) dialTLSWithContext(ctx context.Context, network, addr string, cfg *tls.Config) (*tls.Conn, error) { - dialer := &tls.Dialer{ - Config: cfg, - } - cn, err := dialer.DialContext(ctx, network, addr) - if err != nil { - return nil, err - } - tlsCn := cn.(*tls.Conn) // DialContext comment promises this will always succeed - return tlsCn, nil -} - -func http2tlsUnderlyingConn(tc *tls.Conn) net.Conn { - return tc.NetConn() -} - var http2DebugGoroutines = os.Getenv("DEBUG_HTTP2_GOROUTINES") == "1" type http2goroutineLock uint64 @@ -6366,7 +6334,6 @@ type http2responseWriterState struct { wroteHeader bool // WriteHeader called (explicitly or implicitly). Not necessarily sent to user yet. sentHeader bool // have we sent the header frame? handlerDone bool // handler has finished - dirty bool // a Write failed; don't reuse this responseWriterState sentContentLen int64 // non-zero if handler set a Content-Length header wroteBytes int64 @@ -6486,7 +6453,6 @@ func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { date: date, }) if err != nil { - rws.dirty = true return 0, err } if endStream { @@ -6507,7 +6473,6 @@ func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { if len(p) > 0 || endStream { // only send a 0 byte DATA frame if we're ending the stream. if err := rws.conn.writeDataFromHandler(rws.stream, p, endStream); err != nil { - rws.dirty = true return 0, err } } @@ -6519,9 +6484,6 @@ func (rws *http2responseWriterState) writeChunk(p []byte) (n int, err error) { trailers: rws.trailers, endStream: true, }) - if err != nil { - rws.dirty = true - } return len(p), err } return len(p), nil @@ -6737,14 +6699,12 @@ func (rws *http2responseWriterState) writeHeader(code int) { h.Del("Transfer-Encoding") } - if rws.conn.writeHeaders(rws.stream, &http2writeResHeaders{ + rws.conn.writeHeaders(rws.stream, &http2writeResHeaders{ streamID: rws.stream.id, httpResCode: code, h: h, endStream: rws.handlerDone && !rws.hasTrailers(), - }) != nil { - rws.dirty = true - } + }) return } @@ -6809,19 +6769,10 @@ func (w *http2responseWriter) write(lenData int, dataB []byte, dataS string) (n func (w *http2responseWriter) handlerDone() { rws := w.rws - dirty := rws.dirty rws.handlerDone = true w.Flush() w.rws = nil - if !dirty { - // Only recycle the pool if all prior Write calls to - // the serverConn goroutine completed successfully. If - // they returned earlier due to resets from the peer - // there might still be write goroutines outstanding - // from the serverConn referencing the rws memory. See - // issue 20704. - http2responseWriterStatePool.Put(rws) - } + http2responseWriterStatePool.Put(rws) } // Push errors. @@ -8094,7 +8045,7 @@ func (cc *http2ClientConn) forceCloseConn() { if !ok { return } - if nc := http2tlsUnderlyingConn(tc); nc != nil { + if nc := tc.NetConn(); nc != nil { nc.Close() } } @@ -10282,6 +10233,37 @@ func http2traceFirstResponseByte(trace *httptrace.ClientTrace) { } } +func http2traceHasWroteHeaderField(trace *httptrace.ClientTrace) bool { + return trace != nil && trace.WroteHeaderField != nil +} + +func http2traceWroteHeaderField(trace *httptrace.ClientTrace, k, v string) { + if trace != nil && trace.WroteHeaderField != nil { + trace.WroteHeaderField(k, []string{v}) + } +} + +func http2traceGot1xxResponseFunc(trace *httptrace.ClientTrace) func(int, textproto.MIMEHeader) error { + if trace != nil { + return trace.Got1xxResponse + } + return nil +} + +// dialTLSWithContext uses tls.Dialer, added in Go 1.15, to open a TLS +// connection. +func (t *http2Transport) dialTLSWithContext(ctx context.Context, network, addr string, cfg *tls.Config) (*tls.Conn, error) { + dialer := &tls.Dialer{ + Config: cfg, + } + cn, err := dialer.DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + tlsCn := cn.(*tls.Conn) // DialContext comment promises this will always succeed + return tlsCn, nil +} + // writeFramer is implemented by any type that is used to write frames. type http2writeFramer interface { writeFrame(http2writeContext) error From f997504943bf61e1a2fc414a63d4964a0930ddc1 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Fri, 1 Dec 2023 16:27:43 -0500 Subject: [PATCH 78/93] net/http: avoid leaking io.Copy goroutines (and hijacked connections) in TestTransportNoReuseAfterEarlyResponse Fixes #64252 (maybe). Change-Id: Iba2a403a9347be4206f14acb11591dc2eb7f9fb8 Reviewed-on: https://go-review.googlesource.com/c/go/+/546616 Reviewed-by: Damien Neil LUCI-TryBot-Result: Go LUCI Auto-Submit: Bryan Mills --- transport_test.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/transport_test.go b/transport_test.go index 8c09de70..3057024b 100644 --- a/transport_test.go +++ b/transport_test.go @@ -3499,6 +3499,7 @@ func testTransportNoReuseAfterEarlyResponse(t *testing.T, mode testMode) { c net.Conn } var getOkay bool + var copying sync.WaitGroup closeConn := func() { sconn.Lock() defer sconn.Unlock() @@ -3510,7 +3511,10 @@ func testTransportNoReuseAfterEarlyResponse(t *testing.T, mode testMode) { } } } - defer closeConn() + defer func() { + closeConn() + copying.Wait() + }() ts := newClientServerTest(t, mode, HandlerFunc(func(w ResponseWriter, r *Request) { if r.Method == "GET" { @@ -3522,7 +3526,12 @@ func testTransportNoReuseAfterEarlyResponse(t *testing.T, mode testMode) { sconn.c = conn sconn.Unlock() conn.Write([]byte("HTTP/1.1 200 OK\r\nContent-Length: 3\r\n\r\nfoo")) // keep-alive - go io.Copy(io.Discard, conn) + + copying.Add(1) + go func() { + io.Copy(io.Discard, conn) + copying.Done() + }() })).ts c := ts.Client() From 72e4f415fb24881bdc5970239af439c2d870e75f Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Tue, 7 Nov 2023 10:47:56 -0800 Subject: [PATCH 79/93] net/http: limit chunked data overhead The chunked transfer encoding adds some overhead to the content transferred. When writing one byte per chunk, for example, there are five bytes of overhead per byte of data transferred: "1\r\nX\r\n" to send "X". Chunks may include "chunk extensions", which we skip over and do not use. For example: "1;chunk extension here\r\nX\r\n". A malicious sender can use chunk extensions to add about 4k of overhead per byte of data. (The maximum chunk header line size we will accept.) Track the amount of overhead read in chunked data, and produce an error if it seems excessive. Fixes #64433 Fixes CVE-2023-39326 Change-Id: I40f8d70eb6f9575fb43f506eb19132ccedafcf39 Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/2076135 Reviewed-by: Tatiana Bradley Reviewed-by: Roland Shoemaker Reviewed-on: https://go-review.googlesource.com/c/go/+/547335 Reviewed-by: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov LUCI-TryBot-Result: Go LUCI --- internal/chunked.go | 34 +++++++++++++++++++---- internal/chunked_test.go | 59 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/internal/chunked.go b/internal/chunked.go index 5a174415..aad8e5aa 100644 --- a/internal/chunked.go +++ b/internal/chunked.go @@ -39,7 +39,8 @@ type chunkedReader struct { n uint64 // unread bytes in chunk err error buf [2]byte - checkEnd bool // whether need to check for \r\n chunk footer + checkEnd bool // whether need to check for \r\n chunk footer + excess int64 // "excessive" chunk overhead, for malicious sender detection } func (cr *chunkedReader) beginChunk() { @@ -49,10 +50,36 @@ func (cr *chunkedReader) beginChunk() { if cr.err != nil { return } + cr.excess += int64(len(line)) + 2 // header, plus \r\n after the chunk data + line = trimTrailingWhitespace(line) + line, cr.err = removeChunkExtension(line) + if cr.err != nil { + return + } cr.n, cr.err = parseHexUint(line) if cr.err != nil { return } + // A sender who sends one byte per chunk will send 5 bytes of overhead + // for every byte of data. ("1\r\nX\r\n" to send "X".) + // We want to allow this, since streaming a byte at a time can be legitimate. + // + // A sender can use chunk extensions to add arbitrary amounts of additional + // data per byte read. ("1;very long extension\r\nX\r\n" to send "X".) + // We don't want to disallow extensions (although we discard them), + // but we also don't want to allow a sender to reduce the signal/noise ratio + // arbitrarily. + // + // We track the amount of excess overhead read, + // and produce an error if it grows too large. + // + // Currently, we say that we're willing to accept 16 bytes of overhead per chunk, + // plus twice the amount of real data in the chunk. + cr.excess -= 16 + (2 * int64(cr.n)) + cr.excess = max(cr.excess, 0) + if cr.excess > 16*1024 { + cr.err = errors.New("chunked encoding contains too much non-data") + } if cr.n == 0 { cr.err = io.EOF } @@ -140,11 +167,6 @@ func readChunkLine(b *bufio.Reader) ([]byte, error) { if len(p) >= maxLineLength { return nil, ErrLineTooLong } - p = trimTrailingWhitespace(p) - p, err = removeChunkExtension(p) - if err != nil { - return nil, err - } return p, nil } diff --git a/internal/chunked_test.go b/internal/chunked_test.go index 5e29a786..b99090c1 100644 --- a/internal/chunked_test.go +++ b/internal/chunked_test.go @@ -239,3 +239,62 @@ func TestChunkEndReadError(t *testing.T) { t.Errorf("expected %v, got %v", readErr, err) } } + +func TestChunkReaderTooMuchOverhead(t *testing.T) { + // If the sender is sending 100x as many chunk header bytes as chunk data, + // we should reject the stream at some point. + chunk := []byte("1;") + for i := 0; i < 100; i++ { + chunk = append(chunk, 'a') // chunk extension + } + chunk = append(chunk, "\r\nX\r\n"...) + const bodylen = 1 << 20 + r := NewChunkedReader(&funcReader{f: func(i int) ([]byte, error) { + if i < bodylen { + return chunk, nil + } + return []byte("0\r\n"), nil + }}) + _, err := io.ReadAll(r) + if err == nil { + t.Fatalf("successfully read body with excessive overhead; want error") + } +} + +func TestChunkReaderByteAtATime(t *testing.T) { + // Sending one byte per chunk should not trip the excess-overhead detection. + const bodylen = 1 << 20 + r := NewChunkedReader(&funcReader{f: func(i int) ([]byte, error) { + if i < bodylen { + return []byte("1\r\nX\r\n"), nil + } + return []byte("0\r\n"), nil + }}) + got, err := io.ReadAll(r) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if len(got) != bodylen { + t.Errorf("read %v bytes, want %v", len(got), bodylen) + } +} + +type funcReader struct { + f func(iteration int) ([]byte, error) + i int + b []byte + err error +} + +func (r *funcReader) Read(p []byte) (n int, err error) { + if len(r.b) == 0 && r.err == nil { + r.b, r.err = r.f(r.i) + r.i++ + } + n = copy(p, r.b) + r.b = r.b[n:] + if len(r.b) > 0 { + return n, nil + } + return n, r.err +} From 635f30d706f1d083f3bd0494a512ad1a7f191a0c Mon Sep 17 00:00:00 2001 From: Michael Pratt Date: Wed, 6 Dec 2023 13:29:03 -0500 Subject: [PATCH 80/93] internal/profile: fully decode proto even if there are no samples This is a partial revert of CL 483137. CL 483137 started checking errors in postDecode, which is good. Now we can catch more malformed pprof protos. However this made TestEmptyProfile fail, so an early return was added when the profile was "empty" (no samples). Unfortunately, this was problematic. Profiles with no samples can still be valid, but skipping postDecode meant that the resulting Profile was missing values from the string table. In particular, net/http/pprof needs to parse empty profiles in order to pass through the sample and period types to a final output proto. CL 483137 broke this behavior. internal/profile.Parse is only used in two places: in cmd/compile to parse PGO pprof profiles, and in net/http/pprof to parse before/after pprof profiles for delta profiles. In both cases, the input is never literally empty (0 bytes). Even a pprof proto with no samples still contains some header fields, such as sample and period type. Upstream github.com/google/pprof/profile even has an explicit error on 0 byte input, so `go tool pprof` will not support such an input. Thus TestEmptyProfile was misleading; this profile doesn't need to support empty input at all. Resolve this by removing TestEmptyProfile and replacing it with an explicit error on empty input, as upstream github.com/google/pprof/profile has. For non-empty input, always run postDecode to ensure the string table is processed. TestConvertCPUProfileEmpty is reverted back to assert the values from before CL 483137. Note that in this case "Empty" means no samples, not a 0 byte input. Continue to allow empty files for PGO in order to minimize the chance of last minute breakage if some users have empty files. Fixes #64566. Change-Id: I83a1f0200ae225ac6da0009d4b2431fe215b283f Reviewed-on: https://go-review.googlesource.com/c/go/+/547996 Reviewed-by: Michael Knyszek LUCI-TryBot-Result: Go LUCI Reviewed-by: Cherry Mui --- pprof/pprof_test.go | 63 +++++++++++++++++++++++++++++++++++ pprof/testdata/delta_mutex.go | 43 ++++++++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 pprof/testdata/delta_mutex.go diff --git a/pprof/pprof_test.go b/pprof/pprof_test.go index f82ad45b..24ad59ab 100644 --- a/pprof/pprof_test.go +++ b/pprof/pprof_test.go @@ -6,12 +6,14 @@ package pprof import ( "bytes" + "encoding/base64" "fmt" "internal/profile" "internal/testenv" "io" "net/http" "net/http/httptest" + "path/filepath" "runtime" "runtime/pprof" "strings" @@ -261,3 +263,64 @@ func seen(p *profile.Profile, fname string) bool { } return false } + +// TestDeltaProfileEmptyBase validates that we still receive a valid delta +// profile even if the base contains no samples. +// +// Regression test for https://go.dev/issue/64566. +func TestDeltaProfileEmptyBase(t *testing.T) { + if testing.Short() { + // Delta profile collection has a 1s minimum. + t.Skip("skipping in -short mode") + } + + testenv.MustHaveGoRun(t) + + gotool, err := testenv.GoTool() + if err != nil { + t.Fatalf("error finding go tool: %v", err) + } + + out, err := testenv.Command(t, gotool, "run", filepath.Join("testdata", "delta_mutex.go")).CombinedOutput() + if err != nil { + t.Fatalf("error running profile collection: %v\noutput: %s", err, out) + } + + // Log the binary output for debugging failures. + b64 := make([]byte, base64.StdEncoding.EncodedLen(len(out))) + base64.StdEncoding.Encode(b64, out) + t.Logf("Output in base64.StdEncoding: %s", b64) + + p, err := profile.Parse(bytes.NewReader(out)) + if err != nil { + t.Fatalf("Parse got err %v want nil", err) + } + + t.Logf("Output as parsed Profile: %s", p) + + if len(p.SampleType) != 2 { + t.Errorf("len(p.SampleType) got %d want 2", len(p.SampleType)) + } + if p.SampleType[0].Type != "contentions" { + t.Errorf(`p.SampleType[0].Type got %q want "contentions"`, p.SampleType[0].Type) + } + if p.SampleType[0].Unit != "count" { + t.Errorf(`p.SampleType[0].Unit got %q want "count"`, p.SampleType[0].Unit) + } + if p.SampleType[1].Type != "delay" { + t.Errorf(`p.SampleType[1].Type got %q want "delay"`, p.SampleType[1].Type) + } + if p.SampleType[1].Unit != "nanoseconds" { + t.Errorf(`p.SampleType[1].Unit got %q want "nanoseconds"`, p.SampleType[1].Unit) + } + + if p.PeriodType == nil { + t.Fatal("p.PeriodType got nil want not nil") + } + if p.PeriodType.Type != "contentions" { + t.Errorf(`p.PeriodType.Type got %q want "contentions"`, p.PeriodType.Type) + } + if p.PeriodType.Unit != "count" { + t.Errorf(`p.PeriodType.Unit got %q want "count"`, p.PeriodType.Unit) + } +} diff --git a/pprof/testdata/delta_mutex.go b/pprof/testdata/delta_mutex.go new file mode 100644 index 00000000..634069c8 --- /dev/null +++ b/pprof/testdata/delta_mutex.go @@ -0,0 +1,43 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This binary collects a 1s delta mutex profile and dumps it to os.Stdout. +// +// This is in a subprocess because we want the base mutex profile to be empty +// (as a regression test for https://go.dev/issue/64566) and the only way to +// force reset the profile is to create a new subprocess. +// +// This manually collects the HTTP response and dumps to stdout in order to +// avoid any flakiness around port selection for a real HTTP server. +package main + +import ( + "bytes" + "fmt" + "log" + "net/http" + "net/http/pprof" + "net/http/httptest" + "runtime" +) + +func main() { + // Disable the mutex profiler. This is the default, but that default is + // load-bearing for this test, which needs the base profile to be empty. + runtime.SetMutexProfileFraction(0) + + h := pprof.Handler("mutex") + + req := httptest.NewRequest("GET", "/debug/pprof/mutex?seconds=1", nil) + rec := httptest.NewRecorder() + rec.Body = new(bytes.Buffer) + + h.ServeHTTP(rec, req) + resp := rec.Result() + if resp.StatusCode != http.StatusOK { + log.Fatalf("Request failed: %s\n%s", resp.Status, rec.Body) + } + + fmt.Print(rec.Body) +} From 3db8a9718955ae18a60665b1dc2d17a8cdfaac94 Mon Sep 17 00:00:00 2001 From: Danil Timerbulatov Date: Fri, 8 Dec 2023 00:17:19 +0300 Subject: [PATCH 81/93] all: remove newline characters after return statements This commit is aimed at improving the readability and consistency of the code base. Extraneous newline characters were present after some return statements, creating unnecessary separation in the code. Fixes #64610 Change-Id: Ic1b05bf11761c4dff22691c2f1c3755f66d341f7 Reviewed-on: https://go-review.googlesource.com/c/go/+/548316 Auto-Submit: Dmitri Shuralyov LUCI-TryBot-Result: Go LUCI Reviewed-by: Michael Pratt Reviewed-by: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov --- server.go | 1 - 1 file changed, 1 deletion(-) diff --git a/server.go b/server.go index 36a03f4a..92457785 100644 --- a/server.go +++ b/server.go @@ -3812,7 +3812,6 @@ func numLeadingCRorLF(v []byte) (n int) { break } return - } func strSliceContains(ss []string, s string) bool { From 7dc0f34a9fc198db681a89af75807de886c8a157 Mon Sep 17 00:00:00 2001 From: Jes Cok Date: Wed, 3 Jan 2024 04:18:15 +0000 Subject: [PATCH 82/93] net/http: make Request.Clone create fresh copies for matches and otherValues This change fixes Request.Clone to correctly work with SetPathValue by creating fresh copies for matches and otherValues so that SetPathValue for cloned requests doesn't pollute the original request. While here, also added a doc for Request.SetPathValue. Fixes #64911 Change-Id: I2831b38e135935dfaea2b939bb9db554c75b65ef GitHub-Last-Rev: 1981db16475a49fe8d4b874a6bceec64d28a1332 GitHub-Pull-Request: golang/go#64913 Reviewed-on: https://go-review.googlesource.com/c/go/+/553375 Reviewed-by: Emmanuel Odeke Run-TryBot: Jes Cok Auto-Submit: Dmitri Shuralyov Reviewed-by: Jonathan Amsterdam TryBot-Result: Gopher Robot Reviewed-by: Dmitri Shuralyov --- request.go | 16 ++++++++++++++++ request_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/request.go b/request.go index ed2cdac1..fce2d16f 100644 --- a/request.go +++ b/request.go @@ -397,6 +397,20 @@ func (r *Request) Clone(ctx context.Context) *Request { r2.Form = cloneURLValues(r.Form) r2.PostForm = cloneURLValues(r.PostForm) r2.MultipartForm = cloneMultipartForm(r.MultipartForm) + + // Copy matches and otherValues. See issue 61410. + if s := r.matches; s != nil { + s2 := make([]string, len(s)) + copy(s2, s) + r2.matches = s2 + } + if s := r.otherValues; s != nil { + s2 := make(map[string]string, len(s)) + for k, v := range s { + s2[k] = v + } + r2.otherValues = s2 + } return r2 } @@ -1427,6 +1441,8 @@ func (r *Request) PathValue(name string) string { return r.otherValues[name] } +// SetPathValue sets name to value, so that subsequent calls to r.PathValue(name) +// return value. func (r *Request) SetPathValue(name, value string) { if i := r.patIndex(name); i >= 0 { r.matches[i] = value diff --git a/request_test.go b/request_test.go index 1531da3d..6ce32332 100644 --- a/request_test.go +++ b/request_test.go @@ -1053,6 +1053,33 @@ func TestRequestCloneTransferEncoding(t *testing.T) { } } +// Ensure that Request.Clone works correctly with PathValue. +// See issue 64911. +func TestRequestClonePathValue(t *testing.T) { + req, _ := http.NewRequest("GET", "https://example.org/", nil) + req.SetPathValue("p1", "orig") + + clonedReq := req.Clone(context.Background()) + clonedReq.SetPathValue("p2", "copy") + + // Ensure that any modifications to the cloned + // request do not pollute the original request. + if g, w := req.PathValue("p2"), ""; g != w { + t.Fatalf("p2 mismatch got %q, want %q", g, w) + } + if g, w := req.PathValue("p1"), "orig"; g != w { + t.Fatalf("p1 mismatch got %q, want %q", g, w) + } + + // Assert on the changes to the cloned request. + if g, w := clonedReq.PathValue("p1"), "orig"; g != w { + t.Fatalf("p1 mismatch got %q, want %q", g, w) + } + if g, w := clonedReq.PathValue("p2"), "copy"; g != w { + t.Fatalf("p2 mismatch got %q, want %q", g, w) + } +} + // Issue 34878: verify we don't panic when including basic auth (Go 1.13 regression) func TestNoPanicOnRoundTripWithBasicAuth(t *testing.T) { run(t, testNoPanicWithBasicAuth) } func testNoPanicWithBasicAuth(t *testing.T, mode testMode) { From b3da93083ee5bb06f166bf8c9a3f01863fc72382 Mon Sep 17 00:00:00 2001 From: "Bryan C. Mills" Date: Thu, 4 Jan 2024 11:41:54 -0500 Subject: [PATCH 83/93] net/http/cgi: in TestCopyError, check for a Handler.ServeHTTP goroutine instead of a running PID Previously, the test could fail spuriously if the CGI process's PID happened to be reused in between checks. That sort of reuse is highly unlikely on platforms that cycle through the PID space sequentially (such as Linux), but plausible on platforms that use randomized PIDs (such as OpenBSD). Also unskip the test on Windows, since it no longer relies on being able to send signal 0 to an arbitrary PID. Also change the expected failure mode of the test to a timeout instead of a call to t.Fatalf, so that on failure we get a useful goroutine dump for debugging instead of a non-actionable failure message. Fixes #57369 (maybe). Change-Id: Ib7e3fff556450b48cb5e6ea120fdf4d53547479b Reviewed-on: https://go-review.googlesource.com/c/go/+/554075 LUCI-TryBot-Result: Go LUCI Auto-Submit: Bryan Mills Reviewed-by: Damien Neil --- cgi/host_test.go | 56 ++++++++++++++++++++++++++--------------------- cgi/plan9_test.go | 17 -------------- cgi/posix_test.go | 20 ----------------- 3 files changed, 31 insertions(+), 62 deletions(-) delete mode 100644 cgi/plan9_test.go delete mode 100644 cgi/posix_test.go diff --git a/cgi/host_test.go b/cgi/host_test.go index 78e05d59..f29395fe 100644 --- a/cgi/host_test.go +++ b/cgi/host_test.go @@ -17,8 +17,8 @@ import ( "os" "path/filepath" "reflect" + "regexp" "runtime" - "strconv" "strings" "testing" "time" @@ -363,11 +363,12 @@ func TestInternalRedirect(t *testing.T) { // TestCopyError tests that we kill the process if there's an error copying // its output. (for example, from the client having gone away) +// +// If we fail to do so, the test will time out (and dump its goroutines) with a +// call to [Handler.ServeHTTP] blocked on a deferred call to [exec.Cmd.Wait]. func TestCopyError(t *testing.T) { testenv.MustHaveExec(t) - if runtime.GOOS == "windows" { - t.Skipf("skipping test on %q", runtime.GOOS) - } + h := &Handler{ Path: os.Args[0], Root: "/test.cgi", @@ -390,37 +391,42 @@ func TestCopyError(t *testing.T) { t.Fatalf("ReadResponse: %v", err) } - pidstr := res.Header.Get("X-CGI-Pid") - if pidstr == "" { - t.Fatalf("expected an X-CGI-Pid header in response") - } - pid, err := strconv.Atoi(pidstr) - if err != nil { - t.Fatalf("invalid X-CGI-Pid value") - } - var buf [5000]byte n, err := io.ReadFull(res.Body, buf[:]) if err != nil { t.Fatalf("ReadFull: %d bytes, %v", n, err) } - childRunning := func() bool { - return isProcessRunning(pid) - } - - if !childRunning() { - t.Fatalf("pre-conn.Close, expected child to be running") + if !handlerRunning() { + t.Fatalf("pre-conn.Close, expected handler to still be running") } conn.Close() + closed := time.Now() - tries := 0 - for tries < 25 && childRunning() { - time.Sleep(50 * time.Millisecond * time.Duration(tries)) - tries++ + nextSleep := 1 * time.Millisecond + for { + time.Sleep(nextSleep) + nextSleep *= 2 + if !handlerRunning() { + break + } + t.Logf("handler still running %v after conn.Close", time.Since(closed)) } - if childRunning() { - t.Fatalf("post-conn.Close, expected child to be gone") +} + +// handlerRunning reports whether any goroutine is currently running +// [Handler.ServeHTTP]. +func handlerRunning() bool { + r := regexp.MustCompile(`net/http/cgi\.\(\*Handler\)\.ServeHTTP`) + buf := make([]byte, 64<<10) + for { + n := runtime.Stack(buf, true) + if n < len(buf) { + return r.Match(buf[:n]) + } + // Buffer wasn't large enough for a full goroutine dump. + // Resize it and try again. + buf = make([]byte, 2*len(buf)) } } diff --git a/cgi/plan9_test.go b/cgi/plan9_test.go deleted file mode 100644 index b7ace3f8..00000000 --- a/cgi/plan9_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build plan9 - -package cgi - -import ( - "os" - "strconv" -) - -func isProcessRunning(pid int) bool { - _, err := os.Stat("/proc/" + strconv.Itoa(pid)) - return err == nil -} diff --git a/cgi/posix_test.go b/cgi/posix_test.go deleted file mode 100644 index 49b9470d..00000000 --- a/cgi/posix_test.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2013 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -//go:build !plan9 - -package cgi - -import ( - "os" - "syscall" -) - -func isProcessRunning(pid int) bool { - p, err := os.FindProcess(pid) - if err != nil { - return false - } - return p.Signal(syscall.Signal(0)) == nil -} From 760bd317f904528e477aca7cae74c81a03fa7145 Mon Sep 17 00:00:00 2001 From: Andy Pan Date: Thu, 4 Jan 2024 15:28:14 +0800 Subject: [PATCH 84/93] net/http: respond with 400 Bad Request for empty hex number of chunk length Fixes #64517 Change-Id: I78b8a6a83301deee05c3ff052a6adcd1f965aef2 Reviewed-on: https://go-review.googlesource.com/c/go/+/553835 Auto-Submit: Damien Neil Commit-Queue: Damien Neil LUCI-TryBot-Result: Go LUCI Reviewed-by: Damien Neil Reviewed-by: Bryan Mills --- internal/chunked.go | 3 +++ internal/chunked_test.go | 1 + 2 files changed, 4 insertions(+) diff --git a/internal/chunked.go b/internal/chunked.go index aad8e5aa..c0fa4cca 100644 --- a/internal/chunked.go +++ b/internal/chunked.go @@ -263,6 +263,9 @@ type FlushAfterChunkWriter struct { } func parseHexUint(v []byte) (n uint64, err error) { + if len(v) == 0 { + return 0, errors.New("empty hex number for chunk length") + } for i, b := range v { switch { case '0' <= b && b <= '9': diff --git a/internal/chunked_test.go b/internal/chunked_test.go index b99090c1..af797117 100644 --- a/internal/chunked_test.go +++ b/internal/chunked_test.go @@ -153,6 +153,7 @@ func TestParseHexUint(t *testing.T) { {"00000000000000000", 0, "http chunk length too large"}, // could accept if we wanted {"10000000000000000", 0, "http chunk length too large"}, {"00000000000000001", 0, "http chunk length too large"}, // could accept if we wanted + {"", 0, "empty hex number for chunk length"}, } for i := uint64(0); i <= 1234; i++ { tests = append(tests, testCase{in: fmt.Sprintf("%x", i), want: i}) From d1f9a93da85369eb43114659e2f6c530210b237b Mon Sep 17 00:00:00 2001 From: Jes Cok Date: Tue, 9 Jan 2024 05:33:53 +0000 Subject: [PATCH 85/93] net/http: clarify the precedence order for Request.FormValue Fixes #64575 Change-Id: I0eaec642a9dc8ae3b273a6d41131cc7cb8332947 GitHub-Last-Rev: 17aa5170cbfe42cb86d56f1804266850d33c3eb5 GitHub-Pull-Request: golang/go#64578 Reviewed-on: https://go-review.googlesource.com/c/go/+/547855 Reviewed-by: Damien Neil Reviewed-by: Dmitri Shuralyov Auto-Submit: Damien Neil LUCI-TryBot-Result: Go LUCI --- request.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/request.go b/request.go index fce2d16f..730f5128 100644 --- a/request.go +++ b/request.go @@ -1378,7 +1378,11 @@ func (r *Request) ParseMultipartForm(maxMemory int64) error { } // FormValue returns the first value for the named component of the query. -// POST, PUT, and PATCH body parameters take precedence over URL query string values. +// The precedence order: +// 1. application/x-www-form-urlencoded form body (POST, PUT, PATCH only) +// 2. query parameters (always) +// 3. multipart/form-data form body (always) +// // FormValue calls ParseMultipartForm and ParseForm if necessary and ignores // any errors returned by these functions. // If key is not present, FormValue returns the empty string. From fedd50f9ecc14a3bd9107dc008c30aff557d03f6 Mon Sep 17 00:00:00 2001 From: cui fliter Date: Mon, 6 Nov 2023 22:58:32 +0800 Subject: [PATCH 86/93] net: add available godoc link Change-Id: Ib7c4baf0247c421954aedabfbb6a6af8a08a8936 Reviewed-on: https://go-review.googlesource.com/c/go/+/540021 Reviewed-by: Damien Neil Run-TryBot: shuang cui TryBot-Result: Gopher Robot Auto-Submit: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov --- cgi/child.go | 6 +-- client.go | 84 ++++++++++++++--------------- cookie.go | 4 +- cookiejar/jar.go | 6 +-- doc.go | 16 +++--- fcgi/child.go | 2 +- filetransport.go | 10 ++-- fs.go | 44 +++++++-------- header.go | 14 ++--- http.go | 6 +-- httptest/httptest.go | 2 +- httptest/recorder.go | 18 +++---- httptest/server.go | 8 +-- httptrace/trace.go | 4 +- httputil/dump.go | 10 ++-- httputil/httputil.go | 2 +- httputil/persist.go | 32 +++++------ httputil/reverseproxy.go | 10 ++-- internal/ascii/print.go | 2 +- internal/chunked.go | 8 +-- pprof/pprof.go | 4 +- request.go | 76 +++++++++++++------------- response.go | 10 ++-- responsecontroller.go | 12 ++--- roundtrip.go | 4 +- roundtrip_js.go | 2 +- server.go | 114 +++++++++++++++++++-------------------- transfer.go | 6 +-- transport.go | 34 ++++++------ triv.go | 2 +- 30 files changed, 276 insertions(+), 276 deletions(-) diff --git a/cgi/child.go b/cgi/child.go index 1411f0b8..e29fe20d 100644 --- a/cgi/child.go +++ b/cgi/child.go @@ -46,7 +46,7 @@ func envMap(env []string) map[string]string { return m } -// RequestFromMap creates an http.Request from CGI variables. +// RequestFromMap creates an [http.Request] from CGI variables. // The returned Request's Body field is not populated. func RequestFromMap(params map[string]string) (*http.Request, error) { r := new(http.Request) @@ -138,10 +138,10 @@ func RequestFromMap(params map[string]string) (*http.Request, error) { return r, nil } -// Serve executes the provided Handler on the currently active CGI +// Serve executes the provided [Handler] on the currently active CGI // request, if any. If there's no current CGI environment // an error is returned. The provided handler may be nil to use -// http.DefaultServeMux. +// [http.DefaultServeMux]. func Serve(handler http.Handler) error { req, err := Request() if err != nil { diff --git a/client.go b/client.go index 5fd86a1e..ee6de24f 100644 --- a/client.go +++ b/client.go @@ -27,19 +27,19 @@ import ( "time" ) -// A Client is an HTTP client. Its zero value (DefaultClient) is a -// usable client that uses DefaultTransport. +// A Client is an HTTP client. Its zero value ([DefaultClient]) is a +// usable client that uses [DefaultTransport]. // -// The Client's Transport typically has internal state (cached TCP +// The [Client.Transport] typically has internal state (cached TCP // connections), so Clients should be reused instead of created as // needed. Clients are safe for concurrent use by multiple goroutines. // -// A Client is higher-level than a RoundTripper (such as Transport) +// A Client is higher-level than a [RoundTripper] (such as [Transport]) // and additionally handles HTTP details such as cookies and // redirects. // // When following redirects, the Client will forward all headers set on the -// initial Request except: +// initial [Request] except: // // - when forwarding sensitive headers like "Authorization", // "WWW-Authenticate", and "Cookie" to untrusted targets. @@ -105,11 +105,11 @@ type Client struct { Timeout time.Duration } -// DefaultClient is the default Client and is used by Get, Head, and Post. +// DefaultClient is the default [Client] and is used by [Get], [Head], and [Post]. var DefaultClient = &Client{} // RoundTripper is an interface representing the ability to execute a -// single HTTP transaction, obtaining the Response for a given Request. +// single HTTP transaction, obtaining the [Response] for a given [Request]. // // A RoundTripper must be safe for concurrent use by multiple // goroutines. @@ -439,7 +439,7 @@ func basicAuth(username, password string) string { // // An error is returned if there were too many redirects or if there // was an HTTP protocol error. A non-2xx response doesn't cause an -// error. Any returned error will be of type *url.Error. The url.Error +// error. Any returned error will be of type [*url.Error]. The url.Error // value's Timeout method will report true if the request timed out. // // When err is nil, resp always contains a non-nil resp.Body. @@ -447,10 +447,10 @@ func basicAuth(username, password string) string { // // Get is a wrapper around DefaultClient.Get. // -// To make a request with custom headers, use NewRequest and +// To make a request with custom headers, use [NewRequest] and // DefaultClient.Do. // -// To make a request with a specified context.Context, use NewRequestWithContext +// To make a request with a specified context.Context, use [NewRequestWithContext] // and DefaultClient.Do. func Get(url string) (resp *Response, err error) { return DefaultClient.Get(url) @@ -458,7 +458,7 @@ func Get(url string) (resp *Response, err error) { // Get issues a GET to the specified URL. If the response is one of the // following redirect codes, Get follows the redirect after calling the -// Client's CheckRedirect function: +// [Client.CheckRedirect] function: // // 301 (Moved Permanently) // 302 (Found) @@ -466,18 +466,18 @@ func Get(url string) (resp *Response, err error) { // 307 (Temporary Redirect) // 308 (Permanent Redirect) // -// An error is returned if the Client's CheckRedirect function fails +// An error is returned if the [Client.CheckRedirect] function fails // or if there was an HTTP protocol error. A non-2xx response doesn't -// cause an error. Any returned error will be of type *url.Error. The +// cause an error. Any returned error will be of type [*url.Error]. The // url.Error value's Timeout method will report true if the request // timed out. // // When err is nil, resp always contains a non-nil resp.Body. // Caller should close resp.Body when done reading from it. // -// To make a request with custom headers, use NewRequest and Client.Do. +// To make a request with custom headers, use [NewRequest] and [Client.Do]. // -// To make a request with a specified context.Context, use NewRequestWithContext +// To make a request with a specified context.Context, use [NewRequestWithContext] // and Client.Do. func (c *Client) Get(url string) (resp *Response, err error) { req, err := NewRequest("GET", url, nil) @@ -558,10 +558,10 @@ func urlErrorOp(method string) string { // connectivity problem). A non-2xx status code doesn't cause an // error. // -// If the returned error is nil, the Response will contain a non-nil +// If the returned error is nil, the [Response] will contain a non-nil // Body which the user is expected to close. If the Body is not both -// read to EOF and closed, the Client's underlying RoundTripper -// (typically Transport) may not be able to re-use a persistent TCP +// read to EOF and closed, the [Client]'s underlying [RoundTripper] +// (typically [Transport]) may not be able to re-use a persistent TCP // connection to the server for a subsequent "keep-alive" request. // // The request Body, if non-nil, will be closed by the underlying @@ -570,9 +570,9 @@ func urlErrorOp(method string) string { // // On error, any Response can be ignored. A non-nil Response with a // non-nil error only occurs when CheckRedirect fails, and even then -// the returned Response.Body is already closed. +// the returned [Response.Body] is already closed. // -// Generally Get, Post, or PostForm will be used instead of Do. +// Generally [Get], [Post], or [PostForm] will be used instead of Do. // // If the server replies with a redirect, the Client first uses the // CheckRedirect function to determine whether the redirect should be @@ -580,11 +580,11 @@ func urlErrorOp(method string) string { // subsequent requests to use HTTP method GET // (or HEAD if the original request was HEAD), with no body. // A 307 or 308 redirect preserves the original HTTP method and body, -// provided that the Request.GetBody function is defined. -// The NewRequest function automatically sets GetBody for common +// provided that the [Request.GetBody] function is defined. +// The [NewRequest] function automatically sets GetBody for common // standard library body types. // -// Any returned error will be of type *url.Error. The url.Error +// Any returned error will be of type [*url.Error]. The url.Error // value's Timeout method will report true if the request timed out. func (c *Client) Do(req *Request) (*Response, error) { return c.do(req) @@ -818,17 +818,17 @@ func defaultCheckRedirect(req *Request, via []*Request) error { // // Caller should close resp.Body when done reading from it. // -// If the provided body is an io.Closer, it is closed after the +// If the provided body is an [io.Closer], it is closed after the // request. // // Post is a wrapper around DefaultClient.Post. // -// To set custom headers, use NewRequest and DefaultClient.Do. +// To set custom headers, use [NewRequest] and DefaultClient.Do. // -// See the Client.Do method documentation for details on how redirects +// See the [Client.Do] method documentation for details on how redirects // are handled. // -// To make a request with a specified context.Context, use NewRequestWithContext +// To make a request with a specified context.Context, use [NewRequestWithContext] // and DefaultClient.Do. func Post(url, contentType string, body io.Reader) (resp *Response, err error) { return DefaultClient.Post(url, contentType, body) @@ -838,13 +838,13 @@ func Post(url, contentType string, body io.Reader) (resp *Response, err error) { // // Caller should close resp.Body when done reading from it. // -// If the provided body is an io.Closer, it is closed after the +// If the provided body is an [io.Closer], it is closed after the // request. // -// To set custom headers, use NewRequest and Client.Do. +// To set custom headers, use [NewRequest] and [Client.Do]. // -// To make a request with a specified context.Context, use NewRequestWithContext -// and Client.Do. +// To make a request with a specified context.Context, use [NewRequestWithContext] +// and [Client.Do]. // // See the Client.Do method documentation for details on how redirects // are handled. @@ -861,17 +861,17 @@ func (c *Client) Post(url, contentType string, body io.Reader) (resp *Response, // values URL-encoded as the request body. // // The Content-Type header is set to application/x-www-form-urlencoded. -// To set other headers, use NewRequest and DefaultClient.Do. +// To set other headers, use [NewRequest] and DefaultClient.Do. // // When err is nil, resp always contains a non-nil resp.Body. // Caller should close resp.Body when done reading from it. // // PostForm is a wrapper around DefaultClient.PostForm. // -// See the Client.Do method documentation for details on how redirects +// See the [Client.Do] method documentation for details on how redirects // are handled. // -// To make a request with a specified context.Context, use NewRequestWithContext +// To make a request with a specified [context.Context], use [NewRequestWithContext] // and DefaultClient.Do. func PostForm(url string, data url.Values) (resp *Response, err error) { return DefaultClient.PostForm(url, data) @@ -881,7 +881,7 @@ func PostForm(url string, data url.Values) (resp *Response, err error) { // with data's keys and values URL-encoded as the request body. // // The Content-Type header is set to application/x-www-form-urlencoded. -// To set other headers, use NewRequest and Client.Do. +// To set other headers, use [NewRequest] and [Client.Do]. // // When err is nil, resp always contains a non-nil resp.Body. // Caller should close resp.Body when done reading from it. @@ -889,7 +889,7 @@ func PostForm(url string, data url.Values) (resp *Response, err error) { // See the Client.Do method documentation for details on how redirects // are handled. // -// To make a request with a specified context.Context, use NewRequestWithContext +// To make a request with a specified context.Context, use [NewRequestWithContext] // and Client.Do. func (c *Client) PostForm(url string, data url.Values) (resp *Response, err error) { return c.Post(url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) @@ -907,7 +907,7 @@ func (c *Client) PostForm(url string, data url.Values) (resp *Response, err erro // // Head is a wrapper around DefaultClient.Head. // -// To make a request with a specified context.Context, use NewRequestWithContext +// To make a request with a specified [context.Context], use [NewRequestWithContext] // and DefaultClient.Do. func Head(url string) (resp *Response, err error) { return DefaultClient.Head(url) @@ -915,7 +915,7 @@ func Head(url string) (resp *Response, err error) { // Head issues a HEAD to the specified URL. If the response is one of the // following redirect codes, Head follows the redirect after calling the -// Client's CheckRedirect function: +// [Client.CheckRedirect] function: // // 301 (Moved Permanently) // 302 (Found) @@ -923,8 +923,8 @@ func Head(url string) (resp *Response, err error) { // 307 (Temporary Redirect) // 308 (Permanent Redirect) // -// To make a request with a specified context.Context, use NewRequestWithContext -// and Client.Do. +// To make a request with a specified [context.Context], use [NewRequestWithContext] +// and [Client.Do]. func (c *Client) Head(url string) (resp *Response, err error) { req, err := NewRequest("HEAD", url, nil) if err != nil { @@ -933,12 +933,12 @@ func (c *Client) Head(url string) (resp *Response, err error) { return c.Do(req) } -// CloseIdleConnections closes any connections on its Transport which +// CloseIdleConnections closes any connections on its [Transport] which // were previously connected from previous requests but are now // sitting idle in a "keep-alive" state. It does not interrupt any // connections currently in use. // -// If the Client's Transport does not have a CloseIdleConnections method +// If [Client.Transport] does not have a [Client.CloseIdleConnections] method // then this method does nothing. func (c *Client) CloseIdleConnections() { type closeIdler interface { diff --git a/cookie.go b/cookie.go index 912fde6b..c22897f3 100644 --- a/cookie.go +++ b/cookie.go @@ -163,7 +163,7 @@ func readSetCookies(h Header) []*Cookie { return cookies } -// SetCookie adds a Set-Cookie header to the provided ResponseWriter's headers. +// SetCookie adds a Set-Cookie header to the provided [ResponseWriter]'s headers. // The provided cookie must have a valid Name. Invalid cookies may be // silently dropped. func SetCookie(w ResponseWriter, cookie *Cookie) { @@ -172,7 +172,7 @@ func SetCookie(w ResponseWriter, cookie *Cookie) { } } -// String returns the serialization of the cookie for use in a Cookie +// String returns the serialization of the cookie for use in a [Cookie] // header (if only Name and Value are set) or a Set-Cookie response // header (if other fields are set). // If c is nil or c.Name is invalid, the empty string is returned. diff --git a/cookiejar/jar.go b/cookiejar/jar.go index 46d11939..59cde82c 100644 --- a/cookiejar/jar.go +++ b/cookiejar/jar.go @@ -73,7 +73,7 @@ type Jar struct { nextSeqNum uint64 } -// New returns a new cookie jar. A nil *Options is equivalent to a zero +// New returns a new cookie jar. A nil [*Options] is equivalent to a zero // Options. func New(o *Options) (*Jar, error) { jar := &Jar{ @@ -151,7 +151,7 @@ func hasDotSuffix(s, suffix string) bool { return len(s) > len(suffix) && s[len(s)-len(suffix)-1] == '.' && s[len(s)-len(suffix):] == suffix } -// Cookies implements the Cookies method of the http.CookieJar interface. +// Cookies implements the Cookies method of the [http.CookieJar] interface. // // It returns an empty slice if the URL's scheme is not HTTP or HTTPS. func (j *Jar) Cookies(u *url.URL) (cookies []*http.Cookie) { @@ -226,7 +226,7 @@ func (j *Jar) cookies(u *url.URL, now time.Time) (cookies []*http.Cookie) { return cookies } -// SetCookies implements the SetCookies method of the http.CookieJar interface. +// SetCookies implements the SetCookies method of the [http.CookieJar] interface. // // It does nothing if the URL's scheme is not HTTP or HTTPS. func (j *Jar) SetCookies(u *url.URL, cookies []*http.Cookie) { diff --git a/doc.go b/doc.go index d9e6aafb..f7ad3ae7 100644 --- a/doc.go +++ b/doc.go @@ -5,7 +5,7 @@ /* Package http provides HTTP client and server implementations. -Get, Head, Post, and PostForm make HTTP (or HTTPS) requests: +[Get], [Head], [Post], and [PostForm] make HTTP (or HTTPS) requests: resp, err := http.Get("http://example.com/") ... @@ -27,7 +27,7 @@ The caller must close the response body when finished with it: # Clients and Transports For control over HTTP client headers, redirect policy, and other -settings, create a Client: +settings, create a [Client]: client := &http.Client{ CheckRedirect: redirectPolicyFunc, @@ -43,7 +43,7 @@ settings, create a Client: // ... For control over proxies, TLS configuration, keep-alives, -compression, and other settings, create a Transport: +compression, and other settings, create a [Transport]: tr := &http.Transport{ MaxIdleConns: 10, @@ -59,8 +59,8 @@ goroutines and for efficiency should only be created once and re-used. # Servers ListenAndServe starts an HTTP server with a given address and handler. -The handler is usually nil, which means to use DefaultServeMux. -Handle and HandleFunc add handlers to DefaultServeMux: +The handler is usually nil, which means to use [DefaultServeMux]. +[Handle] and [HandleFunc] add handlers to [DefaultServeMux]: http.Handle("/foo", fooHandler) @@ -86,8 +86,8 @@ custom Server: Starting with Go 1.6, the http package has transparent support for the HTTP/2 protocol when using HTTPS. Programs that must disable HTTP/2 -can do so by setting Transport.TLSNextProto (for clients) or -Server.TLSNextProto (for servers) to a non-nil, empty +can do so by setting [Transport.TLSNextProto] (for clients) or +[Server.TLSNextProto] (for servers) to a non-nil, empty map. Alternatively, the following GODEBUG settings are currently supported: @@ -98,7 +98,7 @@ currently supported: Please report any issues before disabling HTTP/2 support: https://golang.org/s/http2bug -The http package's Transport and Server both automatically enable +The http package's [Transport] and [Server] both automatically enable HTTP/2 support for simple configurations. To enable HTTP/2 for more complex configurations, to use lower-level HTTP/2 features, or to use a newer version of Go's http2 package, import "golang.org/x/net/http2" diff --git a/fcgi/child.go b/fcgi/child.go index dc82bf7c..7665e7d2 100644 --- a/fcgi/child.go +++ b/fcgi/child.go @@ -335,7 +335,7 @@ func (c *child) cleanUp() { // goroutine for each. The goroutine reads requests and then calls handler // to reply to them. // If l is nil, Serve accepts connections from os.Stdin. -// If handler is nil, http.DefaultServeMux is used. +// If handler is nil, [http.DefaultServeMux] is used. func Serve(l net.Listener, handler http.Handler) error { if l == nil { var err error diff --git a/filetransport.go b/filetransport.go index 2a9e9b02..7384b22f 100644 --- a/filetransport.go +++ b/filetransport.go @@ -15,13 +15,13 @@ type fileTransport struct { fh fileHandler } -// NewFileTransport returns a new RoundTripper, serving the provided -// FileSystem. The returned RoundTripper ignores the URL host in its +// NewFileTransport returns a new [RoundTripper], serving the provided +// [FileSystem]. The returned RoundTripper ignores the URL host in its // incoming requests, as well as most other properties of the // request. // // The typical use case for NewFileTransport is to register the "file" -// protocol with a Transport, as in: +// protocol with a [Transport], as in: // // t := &http.Transport{} // t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/"))) @@ -32,13 +32,13 @@ func NewFileTransport(fs FileSystem) RoundTripper { return fileTransport{fileHandler{fs}} } -// NewFileTransportFS returns a new RoundTripper, serving the provided +// NewFileTransportFS returns a new [RoundTripper], serving the provided // file system fsys. The returned RoundTripper ignores the URL host in its // incoming requests, as well as most other properties of the // request. // // The typical use case for NewFileTransportFS is to register the "file" -// protocol with a Transport, as in: +// protocol with a [Transport], as in: // // fsys := os.DirFS("/") // t := &http.Transport{} diff --git a/fs.go b/fs.go index ace74a7b..af7511a7 100644 --- a/fs.go +++ b/fs.go @@ -25,12 +25,12 @@ import ( "time" ) -// A Dir implements FileSystem using the native file system restricted to a +// A Dir implements [FileSystem] using the native file system restricted to a // specific directory tree. // -// While the FileSystem.Open method takes '/'-separated paths, a Dir's string +// While the [FileSystem.Open] method takes '/'-separated paths, a Dir's string // value is a filename on the native file system, not a URL, so it is separated -// by filepath.Separator, which isn't necessarily '/'. +// by [filepath.Separator], which isn't necessarily '/'. // // Note that Dir could expose sensitive files and directories. Dir will follow // symlinks pointing out of the directory tree, which can be especially dangerous @@ -67,7 +67,7 @@ func mapOpenError(originalErr error, name string, sep rune, stat func(string) (f return originalErr } -// Open implements FileSystem using os.Open, opening files for reading rooted +// Open implements [FileSystem] using [os.Open], opening files for reading rooted // and relative to the directory d. func (d Dir) Open(name string) (File, error) { path, err := safefilepath.FromFS(path.Clean("/" + name)) @@ -89,18 +89,18 @@ func (d Dir) Open(name string) (File, error) { // A FileSystem implements access to a collection of named files. // The elements in a file path are separated by slash ('/', U+002F) // characters, regardless of host operating system convention. -// See the FileServer function to convert a FileSystem to a Handler. +// See the [FileServer] function to convert a FileSystem to a [Handler]. // -// This interface predates the fs.FS interface, which can be used instead: -// the FS adapter function converts an fs.FS to a FileSystem. +// This interface predates the [fs.FS] interface, which can be used instead: +// the [FS] adapter function converts an fs.FS to a FileSystem. type FileSystem interface { Open(name string) (File, error) } -// A File is returned by a FileSystem's Open method and can be -// served by the FileServer implementation. +// A File is returned by a [FileSystem]'s Open method and can be +// served by the [FileServer] implementation. // -// The methods should behave the same as those on an *os.File. +// The methods should behave the same as those on an [*os.File]. type File interface { io.Closer io.Reader @@ -167,7 +167,7 @@ func dirList(w ResponseWriter, r *Request, f File) { } // ServeContent replies to the request using the content in the -// provided ReadSeeker. The main benefit of ServeContent over io.Copy +// provided ReadSeeker. The main benefit of ServeContent over [io.Copy] // is that it handles Range requests properly, sets the MIME type, and // handles If-Match, If-Unmodified-Since, If-None-Match, If-Modified-Since, // and If-Range requests. @@ -175,7 +175,7 @@ func dirList(w ResponseWriter, r *Request, f File) { // If the response's Content-Type header is not set, ServeContent // first tries to deduce the type from name's file extension and, // if that fails, falls back to reading the first block of the content -// and passing it to DetectContentType. +// and passing it to [DetectContentType]. // The name is otherwise unused; in particular it can be empty and is // never sent in the response. // @@ -190,7 +190,7 @@ func dirList(w ResponseWriter, r *Request, f File) { // If the caller has set w's ETag header formatted per RFC 7232, section 2.3, // ServeContent uses it to handle requests using If-Match, If-None-Match, or If-Range. // -// Note that *os.File implements the io.ReadSeeker interface. +// Note that [*os.File] implements the [io.ReadSeeker] interface. func ServeContent(w ResponseWriter, req *Request, name string, modtime time.Time, content io.ReadSeeker) { sizeFunc := func() (int64, error) { size, err := content.Seek(0, io.SeekEnd) @@ -741,13 +741,13 @@ func localRedirect(w ResponseWriter, r *Request, newPath string) { // // As a precaution, ServeFile will reject requests where r.URL.Path // contains a ".." path element; this protects against callers who -// might unsafely use filepath.Join on r.URL.Path without sanitizing +// might unsafely use [filepath.Join] on r.URL.Path without sanitizing // it and then use that filepath.Join result as the name argument. // // As another special case, ServeFile redirects any request where r.URL.Path // ends in "/index.html" to the same path, without the final // "index.html". To avoid such redirects either modify the path or -// use ServeContent. +// use [ServeContent]. // // Outside of those two special cases, ServeFile does not use // r.URL.Path for selecting the file or directory to serve; only the @@ -772,11 +772,11 @@ func ServeFile(w ResponseWriter, r *Request, name string) { // If the provided file or directory name is a relative path, it is // interpreted relative to the current directory and may ascend to // parent directories. If the provided name is constructed from user -// input, it should be sanitized before calling ServeFile. +// input, it should be sanitized before calling [ServeFile]. // // As a precaution, ServeFile will reject requests where r.URL.Path // contains a ".." path element; this protects against callers who -// might unsafely use filepath.Join on r.URL.Path without sanitizing +// might unsafely use [filepath.Join] on r.URL.Path without sanitizing // it and then use that filepath.Join result as the name argument. // // As another special case, ServeFile redirects any request where r.URL.Path @@ -890,9 +890,9 @@ func (f ioFile) Readdir(count int) ([]fs.FileInfo, error) { return list, nil } -// FS converts fsys to a FileSystem implementation, -// for use with FileServer and NewFileTransport. -// The files provided by fsys must implement io.Seeker. +// FS converts fsys to a [FileSystem] implementation, +// for use with [FileServer] and [NewFileTransport]. +// The files provided by fsys must implement [io.Seeker]. func FS(fsys fs.FS) FileSystem { return ioFS{fsys} } @@ -905,11 +905,11 @@ func FS(fsys fs.FS) FileSystem { // "index.html". // // To use the operating system's file system implementation, -// use http.Dir: +// use [http.Dir]: // // http.Handle("/", http.FileServer(http.Dir("/tmp"))) // -// To use an fs.FS implementation, use http.FileServerFS instead. +// To use an [fs.FS] implementation, use [http.FileServerFS] instead. func FileServer(root FileSystem) Handler { return &fileHandler{root} } diff --git a/header.go b/header.go index e0b342c6..9d0f3a12 100644 --- a/header.go +++ b/header.go @@ -20,13 +20,13 @@ import ( // A Header represents the key-value pairs in an HTTP header. // // The keys should be in canonical form, as returned by -// CanonicalHeaderKey. +// [CanonicalHeaderKey]. type Header map[string][]string // Add adds the key, value pair to the header. // It appends to any existing values associated with key. // The key is case insensitive; it is canonicalized by -// CanonicalHeaderKey. +// [CanonicalHeaderKey]. func (h Header) Add(key, value string) { textproto.MIMEHeader(h).Add(key, value) } @@ -34,7 +34,7 @@ func (h Header) Add(key, value string) { // Set sets the header entries associated with key to the // single element value. It replaces any existing values // associated with key. The key is case insensitive; it is -// canonicalized by textproto.CanonicalMIMEHeaderKey. +// canonicalized by [textproto.CanonicalMIMEHeaderKey]. // To use non-canonical keys, assign to the map directly. func (h Header) Set(key, value string) { textproto.MIMEHeader(h).Set(key, value) @@ -42,7 +42,7 @@ func (h Header) Set(key, value string) { // Get gets the first value associated with the given key. If // there are no values associated with the key, Get returns "". -// It is case insensitive; textproto.CanonicalMIMEHeaderKey is +// It is case insensitive; [textproto.CanonicalMIMEHeaderKey] is // used to canonicalize the provided key. Get assumes that all // keys are stored in canonical form. To use non-canonical keys, // access the map directly. @@ -51,7 +51,7 @@ func (h Header) Get(key string) string { } // Values returns all values associated with the given key. -// It is case insensitive; textproto.CanonicalMIMEHeaderKey is +// It is case insensitive; [textproto.CanonicalMIMEHeaderKey] is // used to canonicalize the provided key. To use non-canonical // keys, access the map directly. // The returned slice is not a copy. @@ -76,7 +76,7 @@ func (h Header) has(key string) bool { // Del deletes the values associated with key. // The key is case insensitive; it is canonicalized by -// CanonicalHeaderKey. +// [CanonicalHeaderKey]. func (h Header) Del(key string) { textproto.MIMEHeader(h).Del(key) } @@ -125,7 +125,7 @@ var timeFormats = []string{ // ParseTime parses a time header (such as the Date: header), // trying each of the three formats allowed by HTTP/1.1: -// TimeFormat, time.RFC850, and time.ANSIC. +// [TimeFormat], [time.RFC850], and [time.ANSIC]. func ParseTime(text string) (t time.Time, err error) { for _, layout := range timeFormats { t, err = time.Parse(layout, text) diff --git a/http.go b/http.go index 9b81654f..6e2259ad 100644 --- a/http.go +++ b/http.go @@ -103,10 +103,10 @@ func hexEscapeNonASCII(s string) string { return string(b) } -// NoBody is an io.ReadCloser with no bytes. Read always returns EOF +// NoBody is an [io.ReadCloser] with no bytes. Read always returns EOF // and Close always returns nil. It can be used in an outgoing client // request to explicitly signal that a request has zero bytes. -// An alternative, however, is to simply set Request.Body to nil. +// An alternative, however, is to simply set [Request.Body] to nil. var NoBody = noBody{} type noBody struct{} @@ -121,7 +121,7 @@ var ( _ io.ReadCloser = NoBody ) -// PushOptions describes options for Pusher.Push. +// PushOptions describes options for [Pusher.Push]. type PushOptions struct { // Method specifies the HTTP method for the promised request. // If set, it must be "GET" or "HEAD". Empty means "GET". diff --git a/httptest/httptest.go b/httptest/httptest.go index 9bedefd2..f0ca6436 100644 --- a/httptest/httptest.go +++ b/httptest/httptest.go @@ -15,7 +15,7 @@ import ( ) // NewRequest returns a new incoming server Request, suitable -// for passing to an http.Handler for testing. +// for passing to an [http.Handler] for testing. // // The target is the RFC 7230 "request-target": it may be either a // path or an absolute URL. If target is an absolute URL, the host name diff --git a/httptest/recorder.go b/httptest/recorder.go index 1c1d8801..dd51901b 100644 --- a/httptest/recorder.go +++ b/httptest/recorder.go @@ -16,7 +16,7 @@ import ( "golang.org/x/net/http/httpguts" ) -// ResponseRecorder is an implementation of http.ResponseWriter that +// ResponseRecorder is an implementation of [http.ResponseWriter] that // records its mutations for later inspection in tests. type ResponseRecorder struct { // Code is the HTTP response code set by WriteHeader. @@ -47,7 +47,7 @@ type ResponseRecorder struct { wroteHeader bool } -// NewRecorder returns an initialized ResponseRecorder. +// NewRecorder returns an initialized [ResponseRecorder]. func NewRecorder() *ResponseRecorder { return &ResponseRecorder{ HeaderMap: make(http.Header), @@ -57,12 +57,12 @@ func NewRecorder() *ResponseRecorder { } // DefaultRemoteAddr is the default remote address to return in RemoteAddr if -// an explicit DefaultRemoteAddr isn't set on ResponseRecorder. +// an explicit DefaultRemoteAddr isn't set on [ResponseRecorder]. const DefaultRemoteAddr = "1.2.3.4" -// Header implements http.ResponseWriter. It returns the response +// Header implements [http.ResponseWriter]. It returns the response // headers to mutate within a handler. To test the headers that were -// written after a handler completes, use the Result method and see +// written after a handler completes, use the [ResponseRecorder.Result] method and see // the returned Response value's Header. func (rw *ResponseRecorder) Header() http.Header { m := rw.HeaderMap @@ -112,7 +112,7 @@ func (rw *ResponseRecorder) Write(buf []byte) (int, error) { return len(buf), nil } -// WriteString implements io.StringWriter. The data in str is written +// WriteString implements [io.StringWriter]. The data in str is written // to rw.Body, if not nil. func (rw *ResponseRecorder) WriteString(str string) (int, error) { rw.writeHeader(nil, str) @@ -139,7 +139,7 @@ func checkWriteHeaderCode(code int) { } } -// WriteHeader implements http.ResponseWriter. +// WriteHeader implements [http.ResponseWriter]. func (rw *ResponseRecorder) WriteHeader(code int) { if rw.wroteHeader { return @@ -154,7 +154,7 @@ func (rw *ResponseRecorder) WriteHeader(code int) { rw.snapHeader = rw.HeaderMap.Clone() } -// Flush implements http.Flusher. To test whether Flush was +// Flush implements [http.Flusher]. To test whether Flush was // called, see rw.Flushed. func (rw *ResponseRecorder) Flush() { if !rw.wroteHeader { @@ -175,7 +175,7 @@ func (rw *ResponseRecorder) Flush() { // did a write. // // The Response.Body is guaranteed to be non-nil and Body.Read call is -// guaranteed to not return any error other than io.EOF. +// guaranteed to not return any error other than [io.EOF]. // // Result must only be called after the handler has finished running. func (rw *ResponseRecorder) Result() *http.Response { diff --git a/httptest/server.go b/httptest/server.go index c962749e..5095b438 100644 --- a/httptest/server.go +++ b/httptest/server.go @@ -100,7 +100,7 @@ func strSliceContainsPrefix(v []string, pre string) bool { return false } -// NewServer starts and returns a new Server. +// NewServer starts and returns a new [Server]. // The caller should call Close when finished, to shut it down. func NewServer(handler http.Handler) *Server { ts := NewUnstartedServer(handler) @@ -108,7 +108,7 @@ func NewServer(handler http.Handler) *Server { return ts } -// NewUnstartedServer returns a new Server but doesn't start it. +// NewUnstartedServer returns a new [Server] but doesn't start it. // // After changing its configuration, the caller should call Start or // StartTLS. @@ -185,7 +185,7 @@ func (s *Server) StartTLS() { s.goServe() } -// NewTLSServer starts and returns a new Server using TLS. +// NewTLSServer starts and returns a new [Server] using TLS. // The caller should call Close when finished, to shut it down. func NewTLSServer(handler http.Handler) *Server { ts := NewUnstartedServer(handler) @@ -298,7 +298,7 @@ func (s *Server) Certificate() *x509.Certificate { // Client returns an HTTP client configured for making requests to the server. // It is configured to trust the server's TLS test certificate and will -// close its idle connections on Server.Close. +// close its idle connections on [Server.Close]. func (s *Server) Client() *http.Client { return s.client } diff --git a/httptrace/trace.go b/httptrace/trace.go index 6af30f78..706a4329 100644 --- a/httptrace/trace.go +++ b/httptrace/trace.go @@ -19,7 +19,7 @@ import ( // unique type to prevent assignment. type clientEventContextKey struct{} -// ContextClientTrace returns the ClientTrace associated with the +// ContextClientTrace returns the [ClientTrace] associated with the // provided context. If none, it returns nil. func ContextClientTrace(ctx context.Context) *ClientTrace { trace, _ := ctx.Value(clientEventContextKey{}).(*ClientTrace) @@ -233,7 +233,7 @@ func (t *ClientTrace) hasNetHooks() bool { return t.DNSStart != nil || t.DNSDone != nil || t.ConnectStart != nil || t.ConnectDone != nil } -// GotConnInfo is the argument to the ClientTrace.GotConn function and +// GotConnInfo is the argument to the [ClientTrace.GotConn] function and // contains information about the obtained connection. type GotConnInfo struct { // Conn is the connection that was obtained. It is owned by diff --git a/httputil/dump.go b/httputil/dump.go index 7affe5e6..2edb9bc9 100644 --- a/httputil/dump.go +++ b/httputil/dump.go @@ -71,8 +71,8 @@ func outgoingLength(req *http.Request) int64 { return -1 } -// DumpRequestOut is like DumpRequest but for outgoing client requests. It -// includes any headers that the standard http.Transport adds, such as +// DumpRequestOut is like [DumpRequest] but for outgoing client requests. It +// includes any headers that the standard [http.Transport] adds, such as // User-Agent. func DumpRequestOut(req *http.Request, body bool) ([]byte, error) { save := req.Body @@ -203,17 +203,17 @@ var reqWriteExcludeHeaderDump = map[string]bool{ // representation. It should only be used by servers to debug client // requests. The returned representation is an approximation only; // some details of the initial request are lost while parsing it into -// an http.Request. In particular, the order and case of header field +// an [http.Request]. In particular, the order and case of header field // names are lost. The order of values in multi-valued headers is kept // intact. HTTP/2 requests are dumped in HTTP/1.x form, not in their // original binary representations. // // If body is true, DumpRequest also returns the body. To do so, it -// consumes req.Body and then replaces it with a new io.ReadCloser +// consumes req.Body and then replaces it with a new [io.ReadCloser] // that yields the same bytes. If DumpRequest returns an error, // the state of req is undefined. // -// The documentation for http.Request.Write details which fields +// The documentation for [http.Request.Write] details which fields // of req are included in the dump. func DumpRequest(req *http.Request, body bool) ([]byte, error) { var err error diff --git a/httputil/httputil.go b/httputil/httputil.go index 09ea74d6..431930ea 100644 --- a/httputil/httputil.go +++ b/httputil/httputil.go @@ -13,7 +13,7 @@ import ( // NewChunkedReader returns a new chunkedReader that translates the data read from r // out of HTTP "chunked" format before returning it. -// The chunkedReader returns io.EOF when the final 0-length chunk is read. +// The chunkedReader returns [io.EOF] when the final 0-length chunk is read. // // NewChunkedReader is not needed by normal applications. The http package // automatically decodes chunking when reading response bodies. diff --git a/httputil/persist.go b/httputil/persist.go index 84b116df..0cbe3ebf 100644 --- a/httputil/persist.go +++ b/httputil/persist.go @@ -33,7 +33,7 @@ var errClosed = errors.New("i/o operation on closed connection") // It is low-level, old, and unused by Go's current HTTP stack. // We should have deleted it before Go 1. // -// Deprecated: Use the Server in package net/http instead. +// Deprecated: Use the Server in package [net/http] instead. type ServerConn struct { mu sync.Mutex // read-write protects the following fields c net.Conn @@ -50,7 +50,7 @@ type ServerConn struct { // It is low-level, old, and unused by Go's current HTTP stack. // We should have deleted it before Go 1. // -// Deprecated: Use the Server in package net/http instead. +// Deprecated: Use the Server in package [net/http] instead. func NewServerConn(c net.Conn, r *bufio.Reader) *ServerConn { if r == nil { r = bufio.NewReader(c) @@ -58,10 +58,10 @@ func NewServerConn(c net.Conn, r *bufio.Reader) *ServerConn { return &ServerConn{c: c, r: r, pipereq: make(map[*http.Request]uint)} } -// Hijack detaches the ServerConn and returns the underlying connection as well +// Hijack detaches the [ServerConn] and returns the underlying connection as well // as the read-side bufio which may have some left over data. Hijack may be // called before Read has signaled the end of the keep-alive logic. The user -// should not call Hijack while Read or Write is in progress. +// should not call Hijack while [ServerConn.Read] or [ServerConn.Write] is in progress. func (sc *ServerConn) Hijack() (net.Conn, *bufio.Reader) { sc.mu.Lock() defer sc.mu.Unlock() @@ -72,7 +72,7 @@ func (sc *ServerConn) Hijack() (net.Conn, *bufio.Reader) { return c, r } -// Close calls Hijack and then also closes the underlying connection. +// Close calls [ServerConn.Hijack] and then also closes the underlying connection. func (sc *ServerConn) Close() error { c, _ := sc.Hijack() if c != nil { @@ -81,7 +81,7 @@ func (sc *ServerConn) Close() error { return nil } -// Read returns the next request on the wire. An ErrPersistEOF is returned if +// Read returns the next request on the wire. An [ErrPersistEOF] is returned if // it is gracefully determined that there are no more requests (e.g. after the // first request on an HTTP/1.0 connection, or after a Connection:close on a // HTTP/1.1 connection). @@ -171,7 +171,7 @@ func (sc *ServerConn) Pending() int { // Write writes resp in response to req. To close the connection gracefully, set the // Response.Close field to true. Write should be considered operational until -// it returns an error, regardless of any errors returned on the Read side. +// it returns an error, regardless of any errors returned on the [ServerConn.Read] side. func (sc *ServerConn) Write(req *http.Request, resp *http.Response) error { // Retrieve the pipeline ID of this request/response pair @@ -226,7 +226,7 @@ func (sc *ServerConn) Write(req *http.Request, resp *http.Response) error { // It is low-level, old, and unused by Go's current HTTP stack. // We should have deleted it before Go 1. // -// Deprecated: Use Client or Transport in package net/http instead. +// Deprecated: Use Client or Transport in package [net/http] instead. type ClientConn struct { mu sync.Mutex // read-write protects the following fields c net.Conn @@ -244,7 +244,7 @@ type ClientConn struct { // It is low-level, old, and unused by Go's current HTTP stack. // We should have deleted it before Go 1. // -// Deprecated: Use the Client or Transport in package net/http instead. +// Deprecated: Use the Client or Transport in package [net/http] instead. func NewClientConn(c net.Conn, r *bufio.Reader) *ClientConn { if r == nil { r = bufio.NewReader(c) @@ -261,17 +261,17 @@ func NewClientConn(c net.Conn, r *bufio.Reader) *ClientConn { // It is low-level, old, and unused by Go's current HTTP stack. // We should have deleted it before Go 1. // -// Deprecated: Use the Client or Transport in package net/http instead. +// Deprecated: Use the Client or Transport in package [net/http] instead. func NewProxyClientConn(c net.Conn, r *bufio.Reader) *ClientConn { cc := NewClientConn(c, r) cc.writeReq = (*http.Request).WriteProxy return cc } -// Hijack detaches the ClientConn and returns the underlying connection as well +// Hijack detaches the [ClientConn] and returns the underlying connection as well // as the read-side bufio which may have some left over data. Hijack may be // called before the user or Read have signaled the end of the keep-alive -// logic. The user should not call Hijack while Read or Write is in progress. +// logic. The user should not call Hijack while [ClientConn.Read] or ClientConn.Write is in progress. func (cc *ClientConn) Hijack() (c net.Conn, r *bufio.Reader) { cc.mu.Lock() defer cc.mu.Unlock() @@ -282,7 +282,7 @@ func (cc *ClientConn) Hijack() (c net.Conn, r *bufio.Reader) { return } -// Close calls Hijack and then also closes the underlying connection. +// Close calls [ClientConn.Hijack] and then also closes the underlying connection. func (cc *ClientConn) Close() error { c, _ := cc.Hijack() if c != nil { @@ -291,7 +291,7 @@ func (cc *ClientConn) Close() error { return nil } -// Write writes a request. An ErrPersistEOF error is returned if the connection +// Write writes a request. An [ErrPersistEOF] error is returned if the connection // has been closed in an HTTP keep-alive sense. If req.Close equals true, the // keep-alive connection is logically closed after this request and the opposing // server is informed. An ErrUnexpectedEOF indicates the remote closed the @@ -357,9 +357,9 @@ func (cc *ClientConn) Pending() int { } // Read reads the next response from the wire. A valid response might be -// returned together with an ErrPersistEOF, which means that the remote +// returned together with an [ErrPersistEOF], which means that the remote // requested that this be the last request serviced. Read can be called -// concurrently with Write, but not with another Read. +// concurrently with [ClientConn.Write], but not with another Read. func (cc *ClientConn) Read(req *http.Request) (resp *http.Response, err error) { // Retrieve the pipeline ID of this request/response pair cc.mu.Lock() diff --git a/httputil/reverseproxy.go b/httputil/reverseproxy.go index 719ab62d..5c70f0d2 100644 --- a/httputil/reverseproxy.go +++ b/httputil/reverseproxy.go @@ -26,7 +26,7 @@ import ( "golang.org/x/net/http/httpguts" ) -// A ProxyRequest contains a request to be rewritten by a ReverseProxy. +// A ProxyRequest contains a request to be rewritten by a [ReverseProxy]. type ProxyRequest struct { // In is the request received by the proxy. // The Rewrite function must not modify In. @@ -45,7 +45,7 @@ type ProxyRequest struct { // // SetURL rewrites the outbound Host header to match the target's host. // To preserve the inbound request's Host header (the default behavior -// of NewSingleHostReverseProxy): +// of [NewSingleHostReverseProxy]): // // rewriteFunc := func(r *httputil.ProxyRequest) { // r.SetURL(url) @@ -68,7 +68,7 @@ func (r *ProxyRequest) SetURL(target *url.URL) { // If the outbound request contains an existing X-Forwarded-For header, // SetXForwarded appends the client IP address to it. To append to the // inbound request's X-Forwarded-For header (the default behavior of -// ReverseProxy when using a Director function), copy the header +// [ReverseProxy] when using a Director function), copy the header // from the inbound request before calling SetXForwarded: // // rewriteFunc := func(r *httputil.ProxyRequest) { @@ -200,7 +200,7 @@ type ReverseProxy struct { } // A BufferPool is an interface for getting and returning temporary -// byte slices for use by io.CopyBuffer. +// byte slices for use by [io.CopyBuffer]. type BufferPool interface { Get() []byte Put([]byte) @@ -239,7 +239,7 @@ func joinURLPath(a, b *url.URL) (path, rawpath string) { return a.Path + b.Path, apath + bpath } -// NewSingleHostReverseProxy returns a new ReverseProxy that routes +// NewSingleHostReverseProxy returns a new [ReverseProxy] that routes // URLs to the scheme, host, and base path provided in target. If the // target's path is "/base" and the incoming request was for "/dir", // the target request will be for /base/dir. diff --git a/internal/ascii/print.go b/internal/ascii/print.go index 585e5bab..98dbf4e3 100644 --- a/internal/ascii/print.go +++ b/internal/ascii/print.go @@ -9,7 +9,7 @@ import ( "unicode" ) -// EqualFold is strings.EqualFold, ASCII only. It reports whether s and t +// EqualFold is [strings.EqualFold], ASCII only. It reports whether s and t // are equal, ASCII-case-insensitively. func EqualFold(s, t string) bool { if len(s) != len(t) { diff --git a/internal/chunked.go b/internal/chunked.go index c0fa4cca..196b5d89 100644 --- a/internal/chunked.go +++ b/internal/chunked.go @@ -22,7 +22,7 @@ var ErrLineTooLong = errors.New("header line too long") // NewChunkedReader returns a new chunkedReader that translates the data read from r // out of HTTP "chunked" format before returning it. -// The chunkedReader returns io.EOF when the final 0-length chunk is read. +// The chunkedReader returns [io.EOF] when the final 0-length chunk is read. // // NewChunkedReader is not needed by normal applications. The http package // automatically decodes chunking when reading response bodies. @@ -221,7 +221,7 @@ type chunkedWriter struct { // Write the contents of data as one chunk to Wire. // NOTE: Note that the corresponding chunk-writing procedure in Conn.Write has -// a bug since it does not check for success of io.WriteString +// a bug since it does not check for success of [io.WriteString] func (cw *chunkedWriter) Write(data []byte) (n int, err error) { // Don't send 0-length data. It looks like EOF for chunked encoding. @@ -253,9 +253,9 @@ func (cw *chunkedWriter) Close() error { return err } -// FlushAfterChunkWriter signals from the caller of NewChunkedWriter +// FlushAfterChunkWriter signals from the caller of [NewChunkedWriter] // that each chunk should be followed by a flush. It is used by the -// http.Transport code to keep the buffering behavior for headers and +// [net/http.Transport] code to keep the buffering behavior for headers and // trailers, but flush out chunks aggressively in the middle for // request bodies which may be generated slowly. See Issue 6574. type FlushAfterChunkWriter struct { diff --git a/pprof/pprof.go b/pprof/pprof.go index bc3225da..bc48f118 100644 --- a/pprof/pprof.go +++ b/pprof/pprof.go @@ -47,12 +47,12 @@ // go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30 // // Or to look at the goroutine blocking profile, after calling -// runtime.SetBlockProfileRate in your program: +// [runtime.SetBlockProfileRate] in your program: // // go tool pprof http://localhost:6060/debug/pprof/block // // Or to look at the holders of contended mutexes, after calling -// runtime.SetMutexProfileFraction in your program: +// [runtime.SetMutexProfileFraction] in your program: // // go tool pprof http://localhost:6060/debug/pprof/mutex // diff --git a/request.go b/request.go index 730f5128..99fdebcf 100644 --- a/request.go +++ b/request.go @@ -107,7 +107,7 @@ var reqWriteExcludeHeader = map[string]bool{ // // The field semantics differ slightly between client and server // usage. In addition to the notes on the fields below, see the -// documentation for Request.Write and RoundTripper. +// documentation for [Request.Write] and [RoundTripper]. type Request struct { // Method specifies the HTTP method (GET, POST, PUT, etc.). // For client requests, an empty string means GET. @@ -333,7 +333,7 @@ type Request struct { } // Context returns the request's context. To change the context, use -// Clone or WithContext. +// [Request.Clone] or [Request.WithContext]. // // The returned context is always non-nil; it defaults to the // background context. @@ -357,8 +357,8 @@ func (r *Request) Context() context.Context { // lifetime of a request and its response: obtaining a connection, // sending the request, and reading the response headers and body. // -// To create a new request with a context, use NewRequestWithContext. -// To make a deep copy of a request with a new context, use Request.Clone. +// To create a new request with a context, use [NewRequestWithContext]. +// To make a deep copy of a request with a new context, use [Request.Clone]. func (r *Request) WithContext(ctx context.Context) *Request { if ctx == nil { panic("nil context") @@ -435,7 +435,7 @@ func (r *Request) Cookies() []*Cookie { var ErrNoCookie = errors.New("http: named cookie not present") // Cookie returns the named cookie provided in the request or -// ErrNoCookie if not found. +// [ErrNoCookie] if not found. // If multiple cookies match the given name, only one cookie will // be returned. func (r *Request) Cookie(name string) (*Cookie, error) { @@ -449,7 +449,7 @@ func (r *Request) Cookie(name string) (*Cookie, error) { } // AddCookie adds a cookie to the request. Per RFC 6265 section 5.4, -// AddCookie does not attach more than one Cookie header field. That +// AddCookie does not attach more than one [Cookie] header field. That // means all cookies, if any, are written into the same line, // separated by semicolon. // AddCookie only sanitizes c's name and value, and does not sanitize @@ -467,7 +467,7 @@ func (r *Request) AddCookie(c *Cookie) { // // Referer is misspelled as in the request itself, a mistake from the // earliest days of HTTP. This value can also be fetched from the -// Header map as Header["Referer"]; the benefit of making it available +// [Header] map as Header["Referer"]; the benefit of making it available // as a method is that the compiler can diagnose programs that use the // alternate (correct English) spelling req.Referrer() but cannot // diagnose programs that use Header["Referrer"]. @@ -485,7 +485,7 @@ var multipartByReader = &multipart.Form{ // MultipartReader returns a MIME multipart reader if this is a // multipart/form-data or a multipart/mixed POST request, else returns nil and an error. -// Use this function instead of ParseMultipartForm to +// Use this function instead of [Request.ParseMultipartForm] to // process the request body as a stream. func (r *Request) MultipartReader() (*multipart.Reader, error) { if r.MultipartForm == multipartByReader { @@ -548,15 +548,15 @@ const defaultUserAgent = "Go-http-client/1.1" // TransferEncoding // Body // -// If Body is present, Content-Length is <= 0 and TransferEncoding +// If Body is present, Content-Length is <= 0 and [Request.TransferEncoding] // hasn't been set to "identity", Write adds "Transfer-Encoding: // chunked" to the header. Body is closed after it is sent. func (r *Request) Write(w io.Writer) error { return r.write(w, false, nil, nil) } -// WriteProxy is like Write but writes the request in the form -// expected by an HTTP proxy. In particular, WriteProxy writes the +// WriteProxy is like [Request.Write] but writes the request in the form +// expected by an HTTP proxy. In particular, [Request.WriteProxy] writes the // initial Request-URI line of the request with an absolute URI, per // section 5.3 of RFC 7230, including the scheme and host. // In either case, WriteProxy also writes a Host header, using @@ -851,33 +851,33 @@ func validMethod(method string) bool { return len(method) > 0 && strings.IndexFunc(method, isNotToken) == -1 } -// NewRequest wraps NewRequestWithContext using context.Background. +// NewRequest wraps [NewRequestWithContext] using [context.Background]. func NewRequest(method, url string, body io.Reader) (*Request, error) { return NewRequestWithContext(context.Background(), method, url, body) } -// NewRequestWithContext returns a new Request given a method, URL, and +// NewRequestWithContext returns a new [Request] given a method, URL, and // optional body. // -// If the provided body is also an io.Closer, the returned -// Request.Body is set to body and will be closed (possibly +// If the provided body is also an [io.Closer], the returned +// [Request.Body] is set to body and will be closed (possibly // asynchronously) by the Client methods Do, Post, and PostForm, -// and Transport.RoundTrip. +// and [Transport.RoundTrip]. // // NewRequestWithContext returns a Request suitable for use with -// Client.Do or Transport.RoundTrip. To create a request for use with -// testing a Server Handler, either use the NewRequest function in the -// net/http/httptest package, use ReadRequest, or manually update the +// [Client.Do] or [Transport.RoundTrip]. To create a request for use with +// testing a Server Handler, either use the [NewRequest] function in the +// net/http/httptest package, use [ReadRequest], or manually update the // Request fields. For an outgoing client request, the context // controls the entire lifetime of a request and its response: // obtaining a connection, sending the request, and reading the // response headers and body. See the Request type's documentation for // the difference between inbound and outbound request fields. // -// If body is of type *bytes.Buffer, *bytes.Reader, or -// *strings.Reader, the returned request's ContentLength is set to its +// If body is of type [*bytes.Buffer], [*bytes.Reader], or +// [*strings.Reader], the returned request's ContentLength is set to its // exact value (instead of -1), GetBody is populated (so 307 and 308 -// redirects can replay the body), and Body is set to NoBody if the +// redirects can replay the body), and Body is set to [NoBody] if the // ContentLength is 0. func NewRequestWithContext(ctx context.Context, method, url string, body io.Reader) (*Request, error) { if method == "" { @@ -1001,7 +1001,7 @@ func parseBasicAuth(auth string) (username, password string, ok bool) { // The username may not contain a colon. Some protocols may impose // additional requirements on pre-escaping the username and // password. For instance, when used with OAuth2, both arguments must -// be URL encoded first with url.QueryEscape. +// be URL encoded first with [url.QueryEscape]. func (r *Request) SetBasicAuth(username, password string) { r.Header.Set("Authorization", "Basic "+basicAuth(username, password)) } @@ -1035,8 +1035,8 @@ func putTextprotoReader(r *textproto.Reader) { // ReadRequest reads and parses an incoming request from b. // // ReadRequest is a low-level function and should only be used for -// specialized applications; most code should use the Server to read -// requests and handle them via the Handler interface. ReadRequest +// specialized applications; most code should use the [Server] to read +// requests and handle them via the [Handler] interface. ReadRequest // only supports HTTP/1.x requests. For HTTP/2, use golang.org/x/net/http2. func ReadRequest(b *bufio.Reader) (*Request, error) { req, err := readRequest(b) @@ -1145,15 +1145,15 @@ func readRequest(b *bufio.Reader) (req *Request, err error) { return req, nil } -// MaxBytesReader is similar to io.LimitReader but is intended for +// MaxBytesReader is similar to [io.LimitReader] but is intended for // limiting the size of incoming request bodies. In contrast to // io.LimitReader, MaxBytesReader's result is a ReadCloser, returns a -// non-nil error of type *MaxBytesError for a Read beyond the limit, +// non-nil error of type [*MaxBytesError] for a Read beyond the limit, // and closes the underlying reader when its Close method is called. // // MaxBytesReader prevents clients from accidentally or maliciously // sending a large request and wasting server resources. If possible, -// it tells the ResponseWriter to close the connection after the limit +// it tells the [ResponseWriter] to close the connection after the limit // has been reached. func MaxBytesReader(w ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser { if n < 0 { // Treat negative limits as equivalent to 0. @@ -1162,7 +1162,7 @@ func MaxBytesReader(w ResponseWriter, r io.ReadCloser, n int64) io.ReadCloser { return &maxBytesReader{w: w, r: r, i: n, n: n} } -// MaxBytesError is returned by MaxBytesReader when its read limit is exceeded. +// MaxBytesError is returned by [MaxBytesReader] when its read limit is exceeded. type MaxBytesError struct { Limit int64 } @@ -1287,14 +1287,14 @@ func parsePostForm(r *Request) (vs url.Values, err error) { // as a form and puts the results into both r.PostForm and r.Form. Request body // parameters take precedence over URL query string values in r.Form. // -// If the request Body's size has not already been limited by MaxBytesReader, +// If the request Body's size has not already been limited by [MaxBytesReader], // the size is capped at 10MB. // // For other HTTP methods, or when the Content-Type is not // application/x-www-form-urlencoded, the request Body is not read, and // r.PostForm is initialized to a non-nil, empty value. // -// ParseMultipartForm calls ParseForm automatically. +// [Request.ParseMultipartForm] calls ParseForm automatically. // ParseForm is idempotent. func (r *Request) ParseForm() error { var err error @@ -1335,7 +1335,7 @@ func (r *Request) ParseForm() error { // The whole request body is parsed and up to a total of maxMemory bytes of // its file parts are stored in memory, with the remainder stored on // disk in temporary files. -// ParseMultipartForm calls ParseForm if necessary. +// ParseMultipartForm calls [Request.ParseForm] if necessary. // If ParseForm returns an error, ParseMultipartForm returns it but also // continues parsing the request body. // After one call to ParseMultipartForm, subsequent calls have no effect. @@ -1383,11 +1383,11 @@ func (r *Request) ParseMultipartForm(maxMemory int64) error { // 2. query parameters (always) // 3. multipart/form-data form body (always) // -// FormValue calls ParseMultipartForm and ParseForm if necessary and ignores -// any errors returned by these functions. +// FormValue calls [Request.ParseMultipartForm] and [Request.ParseForm] +// if necessary and ignores any errors returned by these functions. // If key is not present, FormValue returns the empty string. // To access multiple values of the same key, call ParseForm and -// then inspect Request.Form directly. +// then inspect [Request.Form] directly. func (r *Request) FormValue(key string) string { if r.Form == nil { r.ParseMultipartForm(defaultMaxMemory) @@ -1400,7 +1400,7 @@ func (r *Request) FormValue(key string) string { // PostFormValue returns the first value for the named component of the POST, // PUT, or PATCH request body. URL query parameters are ignored. -// PostFormValue calls ParseMultipartForm and ParseForm if necessary and ignores +// PostFormValue calls [Request.ParseMultipartForm] and [Request.ParseForm] if necessary and ignores // any errors returned by these functions. // If key is not present, PostFormValue returns the empty string. func (r *Request) PostFormValue(key string) string { @@ -1414,7 +1414,7 @@ func (r *Request) PostFormValue(key string) string { } // FormFile returns the first file for the provided form key. -// FormFile calls ParseMultipartForm and ParseForm if necessary. +// FormFile calls [Request.ParseMultipartForm] and [Request.ParseForm] if necessary. func (r *Request) FormFile(key string) (multipart.File, *multipart.FileHeader, error) { if r.MultipartForm == multipartByReader { return nil, nil, errors.New("http: multipart handled by MultipartReader") @@ -1434,7 +1434,7 @@ func (r *Request) FormFile(key string) (multipart.File, *multipart.FileHeader, e return nil, nil, ErrMissingFile } -// PathValue returns the value for the named path wildcard in the ServeMux pattern +// PathValue returns the value for the named path wildcard in the [ServeMux] pattern // that matched the request. // It returns the empty string if the request was not matched against a pattern // or there is no such wildcard in the pattern. diff --git a/response.go b/response.go index 755c6965..0c3d7f6d 100644 --- a/response.go +++ b/response.go @@ -29,7 +29,7 @@ var respExcludeHeader = map[string]bool{ // Response represents the response from an HTTP request. // -// The Client and Transport return Responses from servers once +// The [Client] and [Transport] return Responses from servers once // the response headers have been received. The response body // is streamed on demand as the Body field is read. type Response struct { @@ -126,13 +126,13 @@ func (r *Response) Cookies() []*Cookie { return readSetCookies(r.Header) } -// ErrNoLocation is returned by Response's Location method +// ErrNoLocation is returned by the [Response.Location] method // when no Location header is present. var ErrNoLocation = errors.New("http: no Location header in response") // Location returns the URL of the response's "Location" header, // if present. Relative redirects are resolved relative to -// the Response's Request. ErrNoLocation is returned if no +// [Response.Request]. [ErrNoLocation] is returned if no // Location header is present. func (r *Response) Location() (*url.URL, error) { lv := r.Header.Get("Location") @@ -146,8 +146,8 @@ func (r *Response) Location() (*url.URL, error) { } // ReadResponse reads and returns an HTTP response from r. -// The req parameter optionally specifies the Request that corresponds -// to this Response. If nil, a GET request is assumed. +// The req parameter optionally specifies the [Request] that corresponds +// to this [Response]. If nil, a GET request is assumed. // Clients must call resp.Body.Close when finished reading resp.Body. // After that call, clients can inspect resp.Trailer to find key/value // pairs included in the response trailer. diff --git a/responsecontroller.go b/responsecontroller.go index 92276ffa..f3f24c12 100644 --- a/responsecontroller.go +++ b/responsecontroller.go @@ -13,14 +13,14 @@ import ( // A ResponseController is used by an HTTP handler to control the response. // -// A ResponseController may not be used after the Handler.ServeHTTP method has returned. +// A ResponseController may not be used after the [Handler.ServeHTTP] method has returned. type ResponseController struct { rw ResponseWriter } -// NewResponseController creates a ResponseController for a request. +// NewResponseController creates a [ResponseController] for a request. // -// The ResponseWriter should be the original value passed to the Handler.ServeHTTP method, +// The ResponseWriter should be the original value passed to the [Handler.ServeHTTP] method, // or have an Unwrap method returning the original ResponseWriter. // // If the ResponseWriter implements any of the following methods, the ResponseController @@ -34,7 +34,7 @@ type ResponseController struct { // EnableFullDuplex() error // // If the ResponseWriter does not support a method, ResponseController returns -// an error matching ErrNotSupported. +// an error matching [ErrNotSupported]. func NewResponseController(rw ResponseWriter) *ResponseController { return &ResponseController{rw} } @@ -116,8 +116,8 @@ func (c *ResponseController) SetWriteDeadline(deadline time.Time) error { } } -// EnableFullDuplex indicates that the request handler will interleave reads from Request.Body -// with writes to the ResponseWriter. +// EnableFullDuplex indicates that the request handler will interleave reads from [Request.Body] +// with writes to the [ResponseWriter]. // // For HTTP/1 requests, the Go HTTP server by default consumes any unread portion of // the request body before beginning to write the response, preventing handlers from diff --git a/roundtrip.go b/roundtrip.go index 49ea1a71..08c27017 100644 --- a/roundtrip.go +++ b/roundtrip.go @@ -6,10 +6,10 @@ package http -// RoundTrip implements the RoundTripper interface. +// RoundTrip implements the [RoundTripper] interface. // // For higher-level HTTP client support (such as handling of cookies -// and redirects), see Get, Post, and the Client type. +// and redirects), see [Get], [Post], and the [Client] type. // // Like the RoundTripper interface, the error types returned // by RoundTrip are unspecified. diff --git a/roundtrip_js.go b/roundtrip_js.go index cbf978af..04c241eb 100644 --- a/roundtrip_js.go +++ b/roundtrip_js.go @@ -56,7 +56,7 @@ var jsFetchMissing = js.Global().Get("fetch").IsUndefined() var jsFetchDisabled = js.Global().Get("process").Type() == js.TypeObject && strings.HasPrefix(js.Global().Get("process").Get("argv0").String(), "node") -// RoundTrip implements the RoundTripper interface using the WHATWG Fetch API. +// RoundTrip implements the [RoundTripper] interface using the WHATWG Fetch API. func (t *Transport) RoundTrip(req *Request) (*Response, error) { // The Transport has a documented contract that states that if the DialContext or // DialTLSContext functions are set, they will be used to set up the connections. diff --git a/server.go b/server.go index 92457785..acac78bc 100644 --- a/server.go +++ b/server.go @@ -61,7 +61,7 @@ var ( // A Handler responds to an HTTP request. // -// ServeHTTP should write reply headers and data to the [ResponseWriter] +// [Handler.ServeHTTP] should write reply headers and data to the [ResponseWriter] // and then return. Returning signals that the request is finished; it // is not valid to use the [ResponseWriter] or read from the // [Request.Body] after or concurrently with the completion of the @@ -161,8 +161,8 @@ type ResponseWriter interface { // The Flusher interface is implemented by ResponseWriters that allow // an HTTP handler to flush buffered data to the client. // -// The default HTTP/1.x and HTTP/2 ResponseWriter implementations -// support Flusher, but ResponseWriter wrappers may not. Handlers +// The default HTTP/1.x and HTTP/2 [ResponseWriter] implementations +// support [Flusher], but ResponseWriter wrappers may not. Handlers // should always test for this ability at runtime. // // Note that even for ResponseWriters that support Flush, @@ -177,7 +177,7 @@ type Flusher interface { // The Hijacker interface is implemented by ResponseWriters that allow // an HTTP handler to take over the connection. // -// The default ResponseWriter for HTTP/1.x connections supports +// The default [ResponseWriter] for HTTP/1.x connections supports // Hijacker, but HTTP/2 connections intentionally do not. // ResponseWriter wrappers may also not support Hijacker. Handlers // should always test for this ability at runtime. @@ -211,7 +211,7 @@ type Hijacker interface { // if the client has disconnected before the response is ready. // // Deprecated: the CloseNotifier interface predates Go's context package. -// New code should use Request.Context instead. +// New code should use [Request.Context] instead. type CloseNotifier interface { // CloseNotify returns a channel that receives at most a // single value (true) when the client connection has gone @@ -505,7 +505,7 @@ func (c *response) EnableFullDuplex() error { return nil } -// TrailerPrefix is a magic prefix for ResponseWriter.Header map keys +// TrailerPrefix is a magic prefix for [ResponseWriter.Header] map keys // that, if present, signals that the map entry is actually for // the response trailers, and not the response headers. The prefix // is stripped after the ServeHTTP call finishes and the values are @@ -571,8 +571,8 @@ type writerOnly struct { io.Writer } -// ReadFrom is here to optimize copying from an *os.File regular file -// to a *net.TCPConn with sendfile, or from a supported src type such +// ReadFrom is here to optimize copying from an [*os.File] regular file +// to a [*net.TCPConn] with sendfile, or from a supported src type such // as a *net.TCPConn on Linux with splice. func (w *response) ReadFrom(src io.Reader) (n int64, err error) { buf := getCopyBuf() @@ -867,7 +867,7 @@ func putBufioWriter(bw *bufio.Writer) { // DefaultMaxHeaderBytes is the maximum permitted size of the headers // in an HTTP request. -// This can be overridden by setting Server.MaxHeaderBytes. +// This can be overridden by setting [Server.MaxHeaderBytes]. const DefaultMaxHeaderBytes = 1 << 20 // 1 MB func (srv *Server) maxHeaderBytes() int { @@ -940,11 +940,11 @@ func (ecr *expectContinueReader) Close() error { } // TimeFormat is the time format to use when generating times in HTTP -// headers. It is like time.RFC1123 but hard-codes GMT as the time +// headers. It is like [time.RFC1123] but hard-codes GMT as the time // zone. The time being formatted must be in UTC for Format to // generate the correct format. // -// For parsing this time format, see ParseTime. +// For parsing this time format, see [ParseTime]. const TimeFormat = "Mon, 02 Jan 2006 15:04:05 GMT" // appendTime is a non-allocating version of []byte(t.UTC().Format(TimeFormat)) @@ -1590,13 +1590,13 @@ func (w *response) bodyAllowed() bool { // The Writers are wired together like: // // 1. *response (the ResponseWriter) -> -// 2. (*response).w, a *bufio.Writer of bufferBeforeChunkingSize bytes -> +// 2. (*response).w, a [*bufio.Writer] of bufferBeforeChunkingSize bytes -> // 3. chunkWriter.Writer (whose writeHeader finalizes Content-Length/Type) // and which writes the chunk headers, if needed -> // 4. conn.bufw, a *bufio.Writer of default (4kB) bytes, writing to -> // 5. checkConnErrorWriter{c}, which notes any non-nil error on Write // and populates c.werr with it if so, but otherwise writes to -> -// 6. the rwc, the net.Conn. +// 6. the rwc, the [net.Conn]. // // TODO(bradfitz): short-circuit some of the buffering when the // initial header contains both a Content-Type and Content-Length. @@ -2097,8 +2097,8 @@ func (w *response) sendExpectationFailed() { w.finishRequest() } -// Hijack implements the Hijacker.Hijack method. Our response is both a ResponseWriter -// and a Hijacker. +// Hijack implements the [Hijacker.Hijack] method. Our response is both a [ResponseWriter] +// and a [Hijacker]. func (w *response) Hijack() (rwc net.Conn, buf *bufio.ReadWriter, err error) { if w.handlerDone.Load() { panic("net/http: Hijack called after ServeHTTP finished") @@ -2158,7 +2158,7 @@ func requestBodyRemains(rc io.ReadCloser) bool { // The HandlerFunc type is an adapter to allow the use of // ordinary functions as HTTP handlers. If f is a function // with the appropriate signature, HandlerFunc(f) is a -// Handler that calls f. +// [Handler] that calls f. type HandlerFunc func(ResponseWriter, *Request) // ServeHTTP calls f(w, r). @@ -2217,9 +2217,9 @@ func StripPrefix(prefix string, h Handler) Handler { // which may be a path relative to the request path. // // The provided code should be in the 3xx range and is usually -// StatusMovedPermanently, StatusFound or StatusSeeOther. +// [StatusMovedPermanently], [StatusFound] or [StatusSeeOther]. // -// If the Content-Type header has not been set, Redirect sets it +// If the Content-Type header has not been set, [Redirect] sets it // to "text/html; charset=utf-8" and writes a small HTML body. // Setting the Content-Type header to any value, including nil, // disables that behavior. @@ -2307,7 +2307,7 @@ func (rh *redirectHandler) ServeHTTP(w ResponseWriter, r *Request) { // status code. // // The provided code should be in the 3xx range and is usually -// StatusMovedPermanently, StatusFound or StatusSeeOther. +// [StatusMovedPermanently], [StatusFound] or [StatusSeeOther]. func RedirectHandler(url string, code int) Handler { return &redirectHandler{url, code} } @@ -2394,7 +2394,7 @@ func RedirectHandler(url string, code int) Handler { // // # Trailing-slash redirection // -// Consider a ServeMux with a handler for a subtree, registered using a trailing slash or "..." wildcard. +// Consider a [ServeMux] with a handler for a subtree, registered using a trailing slash or "..." wildcard. // If the ServeMux receives a request for the subtree root without a trailing slash, // it redirects the request by adding the trailing slash. // This behavior can be overridden with a separate registration for the path without @@ -2437,12 +2437,12 @@ type ServeMux struct { mux121 serveMux121 // used only when GODEBUG=httpmuxgo121=1 } -// NewServeMux allocates and returns a new ServeMux. +// NewServeMux allocates and returns a new [ServeMux]. func NewServeMux() *ServeMux { return &ServeMux{} } -// DefaultServeMux is the default ServeMux used by Serve. +// DefaultServeMux is the default [ServeMux] used by [Serve]. var DefaultServeMux = &defaultServeMux var defaultServeMux ServeMux @@ -2784,7 +2784,7 @@ func (mux *ServeMux) registerErr(patstr string, handler Handler) error { // // The handler is typically nil, in which case [DefaultServeMux] is used. // -// HTTP/2 support is only enabled if the Listener returns *tls.Conn +// HTTP/2 support is only enabled if the Listener returns [*tls.Conn] // connections and they were configured with "h2" in the TLS // Config.NextProtos. // @@ -2924,13 +2924,13 @@ type Server struct { } // Close immediately closes all active net.Listeners and any -// connections in state StateNew, StateActive, or StateIdle. For a -// graceful shutdown, use Shutdown. +// connections in state [StateNew], [StateActive], or [StateIdle]. For a +// graceful shutdown, use [Server.Shutdown]. // // Close does not attempt to close (and does not even know about) // any hijacked connections, such as WebSockets. // -// Close returns any error returned from closing the Server's +// Close returns any error returned from closing the [Server]'s // underlying Listener(s). func (srv *Server) Close() error { srv.inShutdown.Store(true) @@ -2968,16 +2968,16 @@ const shutdownPollIntervalMax = 500 * time.Millisecond // indefinitely for connections to return to idle and then shut down. // If the provided context expires before the shutdown is complete, // Shutdown returns the context's error, otherwise it returns any -// error returned from closing the Server's underlying Listener(s). +// error returned from closing the [Server]'s underlying Listener(s). // -// When Shutdown is called, Serve, ListenAndServe, and -// ListenAndServeTLS immediately return ErrServerClosed. Make sure the +// When Shutdown is called, [Serve], [ListenAndServe], and +// [ListenAndServeTLS] immediately return [ErrServerClosed]. Make sure the // program doesn't exit and waits instead for Shutdown to return. // // Shutdown does not attempt to close nor wait for hijacked // connections such as WebSockets. The caller of Shutdown should // separately notify such long-lived connections of shutdown and wait -// for them to close, if desired. See RegisterOnShutdown for a way to +// for them to close, if desired. See [Server.RegisterOnShutdown] for a way to // register shutdown notification functions. // // Once Shutdown has been called on a server, it may not be reused; @@ -3020,7 +3020,7 @@ func (srv *Server) Shutdown(ctx context.Context) error { } } -// RegisterOnShutdown registers a function to call on Shutdown. +// RegisterOnShutdown registers a function to call on [Server.Shutdown]. // This can be used to gracefully shutdown connections that have // undergone ALPN protocol upgrade or that have been hijacked. // This function should start protocol-specific graceful shutdown, @@ -3068,7 +3068,7 @@ func (s *Server) closeListenersLocked() error { } // A ConnState represents the state of a client connection to a server. -// It's used by the optional Server.ConnState hook. +// It's used by the optional [Server.ConnState] hook. type ConnState int const ( @@ -3145,7 +3145,7 @@ func (sh serverHandler) ServeHTTP(rw ResponseWriter, req *Request) { // behavior doesn't match that of many proxies, and the mismatch can lead to // security issues. // -// AllowQuerySemicolons should be invoked before Request.ParseForm is called. +// AllowQuerySemicolons should be invoked before [Request.ParseForm] is called. func AllowQuerySemicolons(h Handler) Handler { return HandlerFunc(func(w ResponseWriter, r *Request) { if strings.Contains(r.URL.RawQuery, ";") { @@ -3162,13 +3162,13 @@ func AllowQuerySemicolons(h Handler) Handler { } // ListenAndServe listens on the TCP network address srv.Addr and then -// calls Serve to handle requests on incoming connections. +// calls [Serve] to handle requests on incoming connections. // Accepted connections are configured to enable TCP keep-alives. // // If srv.Addr is blank, ":http" is used. // -// ListenAndServe always returns a non-nil error. After Shutdown or Close, -// the returned error is ErrServerClosed. +// ListenAndServe always returns a non-nil error. After [Server.Shutdown] or [Server.Close], +// the returned error is [ErrServerClosed]. func (srv *Server) ListenAndServe() error { if srv.shuttingDown() { return ErrServerClosed @@ -3208,20 +3208,20 @@ func (srv *Server) shouldConfigureHTTP2ForServe() bool { return strSliceContains(srv.TLSConfig.NextProtos, http2NextProtoTLS) } -// ErrServerClosed is returned by the Server's Serve, ServeTLS, ListenAndServe, -// and ListenAndServeTLS methods after a call to Shutdown or Close. +// ErrServerClosed is returned by the [Server.Serve], [ServeTLS], [ListenAndServe], +// and [ListenAndServeTLS] methods after a call to [Server.Shutdown] or [Server.Close]. var ErrServerClosed = errors.New("http: Server closed") // Serve accepts incoming connections on the Listener l, creating a // new service goroutine for each. The service goroutines read requests and // then call srv.Handler to reply to them. // -// HTTP/2 support is only enabled if the Listener returns *tls.Conn +// HTTP/2 support is only enabled if the Listener returns [*tls.Conn] // connections and they were configured with "h2" in the TLS // Config.NextProtos. // // Serve always returns a non-nil error and closes l. -// After Shutdown or Close, the returned error is ErrServerClosed. +// After [Server.Shutdown] or [Server.Close], the returned error is [ErrServerClosed]. func (srv *Server) Serve(l net.Listener) error { if fn := testHookServerServe; fn != nil { fn(srv, l) // call hook with unwrapped listener @@ -3291,14 +3291,14 @@ func (srv *Server) Serve(l net.Listener) error { // setup and then read requests, calling srv.Handler to reply to them. // // Files containing a certificate and matching private key for the -// server must be provided if neither the Server's +// server must be provided if neither the [Server]'s // TLSConfig.Certificates nor TLSConfig.GetCertificate are populated. // If the certificate is signed by a certificate authority, the // certFile should be the concatenation of the server's certificate, // any intermediates, and the CA's certificate. // -// ServeTLS always returns a non-nil error. After Shutdown or Close, the -// returned error is ErrServerClosed. +// ServeTLS always returns a non-nil error. After [Server.Shutdown] or [Server.Close], the +// returned error is [ErrServerClosed]. func (srv *Server) ServeTLS(l net.Listener, certFile, keyFile string) error { // Setup HTTP/2 before srv.Serve, to initialize srv.TLSConfig // before we clone it and create the TLS Listener. @@ -3427,7 +3427,7 @@ func logf(r *Request, format string, args ...any) { } // ListenAndServe listens on the TCP network address addr and then calls -// Serve with handler to handle requests on incoming connections. +// [Serve] with handler to handle requests on incoming connections. // Accepted connections are configured to enable TCP keep-alives. // // The handler is typically nil, in which case [DefaultServeMux] is used. @@ -3449,11 +3449,11 @@ func ListenAndServeTLS(addr, certFile, keyFile string, handler Handler) error { } // ListenAndServeTLS listens on the TCP network address srv.Addr and -// then calls ServeTLS to handle requests on incoming TLS connections. +// then calls [ServeTLS] to handle requests on incoming TLS connections. // Accepted connections are configured to enable TCP keep-alives. // // Filenames containing a certificate and matching private key for the -// server must be provided if neither the Server's TLSConfig.Certificates +// server must be provided if neither the [Server]'s TLSConfig.Certificates // nor TLSConfig.GetCertificate are populated. If the certificate is // signed by a certificate authority, the certFile should be the // concatenation of the server's certificate, any intermediates, and @@ -3461,8 +3461,8 @@ func ListenAndServeTLS(addr, certFile, keyFile string, handler Handler) error { // // If srv.Addr is blank, ":https" is used. // -// ListenAndServeTLS always returns a non-nil error. After Shutdown or -// Close, the returned error is ErrServerClosed. +// ListenAndServeTLS always returns a non-nil error. After [Server.Shutdown] or +// [Server.Close], the returned error is [ErrServerClosed]. func (srv *Server) ListenAndServeTLS(certFile, keyFile string) error { if srv.shuttingDown() { return ErrServerClosed @@ -3532,17 +3532,17 @@ func (srv *Server) onceSetNextProtoDefaults() { } } -// TimeoutHandler returns a Handler that runs h with the given time limit. +// TimeoutHandler returns a [Handler] that runs h with the given time limit. // // The new Handler calls h.ServeHTTP to handle each request, but if a // call runs for longer than its time limit, the handler responds with // a 503 Service Unavailable error and the given message in its body. // (If msg is empty, a suitable default message will be sent.) -// After such a timeout, writes by h to its ResponseWriter will return -// ErrHandlerTimeout. +// After such a timeout, writes by h to its [ResponseWriter] will return +// [ErrHandlerTimeout]. // -// TimeoutHandler supports the Pusher interface but does not support -// the Hijacker or Flusher interfaces. +// TimeoutHandler supports the [Pusher] interface but does not support +// the [Hijacker] or [Flusher] interfaces. func TimeoutHandler(h Handler, dt time.Duration, msg string) Handler { return &timeoutHandler{ handler: h, @@ -3551,7 +3551,7 @@ func TimeoutHandler(h Handler, dt time.Duration, msg string) Handler { } } -// ErrHandlerTimeout is returned on ResponseWriter Write calls +// ErrHandlerTimeout is returned on [ResponseWriter] Write calls // in handlers which have timed out. var ErrHandlerTimeout = errors.New("http: Handler timeout") @@ -3640,7 +3640,7 @@ type timeoutWriter struct { var _ Pusher = (*timeoutWriter)(nil) -// Push implements the Pusher interface. +// Push implements the [Pusher] interface. func (tw *timeoutWriter) Push(target string, opts *PushOptions) error { if pusher, ok := tw.w.(Pusher); ok { return pusher.Push(target, opts) @@ -3725,7 +3725,7 @@ type initALPNRequest struct { h serverHandler } -// BaseContext is an exported but unadvertised http.Handler method +// BaseContext is an exported but unadvertised [http.Handler] method // recognized by x/net/http2 to pass down a context; the TLSNextProto // API predates context support so we shoehorn through the only // interface we have available. @@ -3833,7 +3833,7 @@ func tlsRecordHeaderLooksLikeHTTP(hdr [5]byte) bool { return false } -// MaxBytesHandler returns a Handler that runs h with its ResponseWriter and Request.Body wrapped by a MaxBytesReader. +// MaxBytesHandler returns a [Handler] that runs h with its [ResponseWriter] and [Request.Body] wrapped by a MaxBytesReader. func MaxBytesHandler(h Handler, n int64) Handler { return HandlerFunc(func(w ResponseWriter, r *Request) { r2 := *r diff --git a/transfer.go b/transfer.go index d7872584..315c6e27 100644 --- a/transfer.go +++ b/transfer.go @@ -817,10 +817,10 @@ type body struct { onHitEOF func() // if non-nil, func to call when EOF is Read } -// ErrBodyReadAfterClose is returned when reading a Request or Response +// ErrBodyReadAfterClose is returned when reading a [Request] or [Response] // Body after the body has been closed. This typically happens when the body is -// read after an HTTP Handler calls WriteHeader or Write on its -// ResponseWriter. +// read after an HTTP [Handler] calls WriteHeader or Write on its +// [ResponseWriter]. var ErrBodyReadAfterClose = errors.New("http: invalid Read on closed Body") func (b *body) Read(p []byte) (n int, err error) { diff --git a/transport.go b/transport.go index 170ba86c..57c70e72 100644 --- a/transport.go +++ b/transport.go @@ -35,8 +35,8 @@ import ( "golang.org/x/net/http/httpproxy" ) -// DefaultTransport is the default implementation of Transport and is -// used by DefaultClient. It establishes network connections as needed +// DefaultTransport is the default implementation of [Transport] and is +// used by [DefaultClient]. It establishes network connections as needed // and caches them for reuse by subsequent calls. It uses HTTP proxies // as directed by the environment variables HTTP_PROXY, HTTPS_PROXY // and NO_PROXY (or the lowercase versions thereof). @@ -53,42 +53,42 @@ var DefaultTransport RoundTripper = &Transport{ ExpectContinueTimeout: 1 * time.Second, } -// DefaultMaxIdleConnsPerHost is the default value of Transport's +// DefaultMaxIdleConnsPerHost is the default value of [Transport]'s // MaxIdleConnsPerHost. const DefaultMaxIdleConnsPerHost = 2 -// Transport is an implementation of RoundTripper that supports HTTP, +// Transport is an implementation of [RoundTripper] that supports HTTP, // HTTPS, and HTTP proxies (for either HTTP or HTTPS with CONNECT). // // By default, Transport caches connections for future re-use. // This may leave many open connections when accessing many hosts. -// This behavior can be managed using Transport's CloseIdleConnections method -// and the MaxIdleConnsPerHost and DisableKeepAlives fields. +// This behavior can be managed using [Transport.CloseIdleConnections] method +// and the [Transport.MaxIdleConnsPerHost] and [Transport.DisableKeepAlives] fields. // // Transports should be reused instead of created as needed. // Transports are safe for concurrent use by multiple goroutines. // // A Transport is a low-level primitive for making HTTP and HTTPS requests. -// For high-level functionality, such as cookies and redirects, see Client. +// For high-level functionality, such as cookies and redirects, see [Client]. // // Transport uses HTTP/1.1 for HTTP URLs and either HTTP/1.1 or HTTP/2 // for HTTPS URLs, depending on whether the server supports HTTP/2, -// and how the Transport is configured. The DefaultTransport supports HTTP/2. +// and how the Transport is configured. The [DefaultTransport] supports HTTP/2. // To explicitly enable HTTP/2 on a transport, use golang.org/x/net/http2 // and call ConfigureTransport. See the package docs for more about HTTP/2. // // Responses with status codes in the 1xx range are either handled // automatically (100 expect-continue) or ignored. The one // exception is HTTP status code 101 (Switching Protocols), which is -// considered a terminal status and returned by RoundTrip. To see the +// considered a terminal status and returned by [Transport.RoundTrip]. To see the // ignored 1xx responses, use the httptrace trace package's // ClientTrace.Got1xxResponse. // // Transport only retries a request upon encountering a network error // if the connection has been already been used successfully and if the -// request is idempotent and either has no body or has its Request.GetBody +// request is idempotent and either has no body or has its [Request.GetBody] // defined. HTTP requests are considered idempotent if they have HTTP methods -// GET, HEAD, OPTIONS, or TRACE; or if their Header map contains an +// GET, HEAD, OPTIONS, or TRACE; or if their [Header] map contains an // "Idempotency-Key" or "X-Idempotency-Key" entry. If the idempotency key // value is a zero-length slice, the request is treated as idempotent but the // header is not sent on the wire. @@ -453,7 +453,7 @@ func ProxyFromEnvironment(req *Request) (*url.URL, error) { return envProxyFunc()(req.URL) } -// ProxyURL returns a proxy function (for use in a Transport) +// ProxyURL returns a proxy function (for use in a [Transport]) // that always returns the same URL. func ProxyURL(fixedURL *url.URL) func(*Request) (*url.URL, error) { return func(*Request) (*url.URL, error) { @@ -752,14 +752,14 @@ func (pc *persistConn) shouldRetryRequest(req *Request, err error) bool { var ErrSkipAltProtocol = errors.New("net/http: skip alternate protocol") // RegisterProtocol registers a new protocol with scheme. -// The Transport will pass requests using the given scheme to rt. +// The [Transport] will pass requests using the given scheme to rt. // It is rt's responsibility to simulate HTTP request semantics. // // RegisterProtocol can be used by other packages to provide // implementations of protocol schemes like "ftp" or "file". // -// If rt.RoundTrip returns ErrSkipAltProtocol, the Transport will -// handle the RoundTrip itself for that one request, as if the +// If rt.RoundTrip returns [ErrSkipAltProtocol], the Transport will +// handle the [Transport.RoundTrip] itself for that one request, as if the // protocol were not registered. func (t *Transport) RegisterProtocol(scheme string, rt RoundTripper) { t.altMu.Lock() @@ -799,9 +799,9 @@ func (t *Transport) CloseIdleConnections() { } // CancelRequest cancels an in-flight request by closing its connection. -// CancelRequest should only be called after RoundTrip has returned. +// CancelRequest should only be called after [Transport.RoundTrip] has returned. // -// Deprecated: Use Request.WithContext to create a request with a +// Deprecated: Use [Request.WithContext] to create a request with a // cancelable context instead. CancelRequest cannot cancel HTTP/2 // requests. func (t *Transport) CancelRequest(req *Request) { diff --git a/triv.go b/triv.go index f614922c..c1696425 100644 --- a/triv.go +++ b/triv.go @@ -34,7 +34,7 @@ type Counter struct { n int } -// This makes Counter satisfy the expvar.Var interface, so we can export +// This makes Counter satisfy the [expvar.Var] interface, so we can export // it directly. func (ctr *Counter) String() string { ctr.mu.Lock() From 66b1a31b92268324f8e45650a5404d9fda24aa80 Mon Sep 17 00:00:00 2001 From: Julian Tibble Date: Wed, 14 Feb 2024 13:24:52 +0000 Subject: [PATCH 87/93] [release-branch.go1.22] net/http: add missing call to decConnsPerHost A recent change to Transport.dialConnFor introduced an early return that skipped dialing. This path did not call decConnsPerHost, which can cause subsequent HTTP calls to hang if Transport.MaxConnsPerHost is set. For #65705 Fixes #65759 Change-Id: I157591114b02a3a66488d3ead7f1e6dbd374a41c Reviewed-on: https://go-review.googlesource.com/c/go/+/564036 Reviewed-by: Damien Neil LUCI-TryBot-Result: Go LUCI Auto-Submit: Damien Neil Reviewed-by: Than McIntosh (cherry picked from commit 098a87fb1930b9ef99d394fe1bca75f1bd74ce8d) Reviewed-on: https://go-review.googlesource.com/c/go/+/566536 Reviewed-by: Carlos Amedee --- transport.go | 1 + transport_test.go | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/transport.go b/transport.go index 57c70e72..17067ac0 100644 --- a/transport.go +++ b/transport.go @@ -1478,6 +1478,7 @@ func (t *Transport) dialConnFor(w *wantConn) { defer w.afterDial() ctx := w.getCtxForDial() if ctx == nil { + t.decConnsPerHost(w.key) return } diff --git a/transport_test.go b/transport_test.go index 3057024b..3fb56246 100644 --- a/transport_test.go +++ b/transport_test.go @@ -730,6 +730,56 @@ func testTransportMaxConnsPerHost(t *testing.T, mode testMode) { } } +func TestTransportMaxConnsPerHostDialCancellation(t *testing.T) { + run(t, testTransportMaxConnsPerHostDialCancellation, + testNotParallel, // because test uses SetPendingDialHooks + []testMode{http1Mode, https1Mode, http2Mode}, + ) +} + +func testTransportMaxConnsPerHostDialCancellation(t *testing.T, mode testMode) { + CondSkipHTTP2(t) + + h := HandlerFunc(func(w ResponseWriter, r *Request) { + _, err := w.Write([]byte("foo")) + if err != nil { + t.Fatalf("Write: %v", err) + } + }) + + cst := newClientServerTest(t, mode, h) + defer cst.close() + ts := cst.ts + c := ts.Client() + tr := c.Transport.(*Transport) + tr.MaxConnsPerHost = 1 + + // This request is cancelled when dial is queued, which preempts dialing. + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + SetPendingDialHooks(cancel, nil) + defer SetPendingDialHooks(nil, nil) + + req, _ := NewRequestWithContext(ctx, "GET", ts.URL, nil) + _, err := c.Do(req) + if !errors.Is(err, context.Canceled) { + t.Errorf("expected error %v, got %v", context.Canceled, err) + } + + // This request should succeed. + SetPendingDialHooks(nil, nil) + req, _ = NewRequest("GET", ts.URL, nil) + resp, err := c.Do(req) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + _, err = io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body failed: %v", err) + } +} + func TestTransportRemovesDeadIdleConnections(t *testing.T) { run(t, testTransportRemovesDeadIdleConnections, []testMode{http1Mode}) } From 1008e41fe52ffc8baefdf5d5858523f64dbf8d08 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Thu, 11 Jan 2024 11:31:57 -0800 Subject: [PATCH 88/93] [release-branch.go1.22] net/http, net/http/cookiejar: avoid subdomain matches on IPv6 zones When deciding whether to forward cookies or sensitive headers across a redirect, do not attempt to interpret an IPv6 address as a domain name. Avoids a case where a maliciously-crafted redirect to an IPv6 address with a scoped addressing zone could be misinterpreted as a within-domain redirect. For example, we could interpret "::1%.www.example.com" as a subdomain of "www.example.com". Thanks to Juho Nurminen of Mattermost for reporting this issue. Fixes CVE-2023-45289 Fixes #65859 For #65065 Change-Id: I8f463f59f0e700c8a18733d2b264a8bcb3a19599 Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/2131938 Reviewed-by: Tatiana Bradley Reviewed-by: Roland Shoemaker Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/2174344 Reviewed-by: Carlos Amedee Reviewed-on: https://go-review.googlesource.com/c/go/+/569236 Reviewed-by: Carlos Amedee LUCI-TryBot-Result: Go LUCI Auto-Submit: Michael Knyszek --- client.go | 6 ++++++ client_test.go | 1 + cookiejar/jar.go | 7 +++++++ cookiejar/jar_test.go | 10 ++++++++++ 4 files changed, 24 insertions(+) diff --git a/client.go b/client.go index ee6de24f..8fc348fe 100644 --- a/client.go +++ b/client.go @@ -1014,6 +1014,12 @@ func isDomainOrSubdomain(sub, parent string) bool { if sub == parent { return true } + // If sub contains a :, it's probably an IPv6 address (and is definitely not a hostname). + // Don't check the suffix in this case, to avoid matching the contents of a IPv6 zone. + // For example, "::1%.www.example.com" is not a subdomain of "www.example.com". + if strings.ContainsAny(sub, ":%") { + return false + } // If sub is "foo.example.com" and parent is "example.com", // that means sub must end in "."+parent. // Do it without allocating. diff --git a/client_test.go b/client_test.go index 7459b9cb..e2a1cbbd 100644 --- a/client_test.go +++ b/client_test.go @@ -1711,6 +1711,7 @@ func TestShouldCopyHeaderOnRedirect(t *testing.T) { {"authorization", "http://foo.com/", "https://foo.com/", true}, {"authorization", "http://foo.com:1234/", "http://foo.com:4321/", true}, {"www-authenticate", "http://foo.com/", "http://bar.com/", false}, + {"authorization", "http://foo.com/", "http://[::1%25.foo.com]/", false}, // But subdomains should work: {"www-authenticate", "http://foo.com/", "http://foo.com/", true}, diff --git a/cookiejar/jar.go b/cookiejar/jar.go index 59cde82c..e7f5ddd4 100644 --- a/cookiejar/jar.go +++ b/cookiejar/jar.go @@ -362,6 +362,13 @@ func jarKey(host string, psl PublicSuffixList) string { // isIP reports whether host is an IP address. func isIP(host string) bool { + if strings.ContainsAny(host, ":%") { + // Probable IPv6 address. + // Hostnames can't contain : or %, so this is definitely not a valid host. + // Treating it as an IP is the more conservative option, and avoids the risk + // of interpeting ::1%.www.example.com as a subtomain of www.example.com. + return true + } return net.ParseIP(host) != nil } diff --git a/cookiejar/jar_test.go b/cookiejar/jar_test.go index 56d0695a..251f7c16 100644 --- a/cookiejar/jar_test.go +++ b/cookiejar/jar_test.go @@ -252,6 +252,7 @@ var isIPTests = map[string]bool{ "127.0.0.1": true, "1.2.3.4": true, "2001:4860:0:2001::68": true, + "::1%zone": true, "example.com": false, "1.1.1.300": false, "www.foo.bar.net": false, @@ -629,6 +630,15 @@ var basicsTests = [...]jarTest{ {"http://www.host.test:1234/", "a=1"}, }, }, + { + "IPv6 zone is not treated as a host.", + "https://example.com/", + []string{"a=1"}, + "a=1", + []query{ + {"https://[::1%25.example.com]:80/", ""}, + }, + }, } func TestBasics(t *testing.T) { From 8715feeac951fa7c426ab34965974e93bccff089 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Wed, 27 Mar 2024 15:01:59 -0700 Subject: [PATCH 89/93] [release-branch.go1.22] all: update golang.org/x/net Pulls in one HTTP/2 fix: ae3c50b55f http2: reject DATA frames after 1xx and before final headers For golang/go#65927 Fixes golang/go#66255 Change-Id: Ib810455297083fc0722a997d0aa675132c38393c Reviewed-on: https://go-review.googlesource.com/c/go/+/574935 Reviewed-by: Dmitri Shuralyov TryBot-Bypass: Dmitri Shuralyov Reviewed-by: Dmitri Shuralyov --- h2_bundle.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/h2_bundle.go b/h2_bundle.go index ac41144d..969c3ffd 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -9712,7 +9712,7 @@ func (rl *http2clientConnReadLoop) processData(f *http2DataFrame) error { }) return nil } - if !cs.firstByte { + if !cs.pastHeaders { cc.logf("protocol error: received DATA before a HEADERS frame") rl.endStreamError(cs, http2StreamError{ StreamID: f.StreamID, From efc54b3e16de765e3e30e865b616163c81a0aa2d Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Thu, 28 Mar 2024 16:57:51 -0700 Subject: [PATCH 90/93] [release-branch.go1.22] net/http: update bundled golang.org/x/net/http2 Disable cmd/internal/moddeps test, since this update includes PRIVATE track fixes. Fixes CVE-2023-45288 For #65051 Fixes #66298 Change-Id: I5bbf774ebe7651e4bb7e55139d3794bd2b8e8fa8 Reviewed-on: https://team-review.git.corp.google.com/c/golang/go-private/+/2197227 Reviewed-by: Tatiana Bradley Run-TryBot: Damien Neil Reviewed-by: Dmitri Shuralyov Reviewed-on: https://go-review.googlesource.com/c/go/+/576076 Auto-Submit: Dmitri Shuralyov TryBot-Bypass: Dmitri Shuralyov Reviewed-by: Than McIntosh --- h2_bundle.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/h2_bundle.go b/h2_bundle.go index 969c3ffd..75454dba 100644 --- a/h2_bundle.go +++ b/h2_bundle.go @@ -2969,6 +2969,7 @@ func (fr *http2Framer) readMetaFrame(hf *http2HeadersFrame) (*http2MetaHeadersFr if size > remainSize { hdec.SetEmitEnabled(false) mh.Truncated = true + remainSize = 0 return } remainSize -= size @@ -2981,6 +2982,36 @@ func (fr *http2Framer) readMetaFrame(hf *http2HeadersFrame) (*http2MetaHeadersFr var hc http2headersOrContinuation = hf for { frag := hc.HeaderBlockFragment() + + // Avoid parsing large amounts of headers that we will then discard. + // If the sender exceeds the max header list size by too much, + // skip parsing the fragment and close the connection. + // + // "Too much" is either any CONTINUATION frame after we've already + // exceeded the max header list size (in which case remainSize is 0), + // or a frame whose encoded size is more than twice the remaining + // header list bytes we're willing to accept. + if int64(len(frag)) > int64(2*remainSize) { + if http2VerboseLogs { + log.Printf("http2: header list too large") + } + // It would be nice to send a RST_STREAM before sending the GOAWAY, + // but the struture of the server's frame writer makes this difficult. + return nil, http2ConnectionError(http2ErrCodeProtocol) + } + + // Also close the connection after any CONTINUATION frame following an + // invalid header, since we stop tracking the size of the headers after + // an invalid one. + if invalid != nil { + if http2VerboseLogs { + log.Printf("http2: invalid header: %v", invalid) + } + // It would be nice to send a RST_STREAM before sending the GOAWAY, + // but the struture of the server's frame writer makes this difficult. + return nil, http2ConnectionError(http2ErrCodeProtocol) + } + if _, err := hdec.Write(frag); err != nil { return nil, http2ConnectionError(http2ErrCodeCompression) } From 87b4571664d279528a8deb57b4724d642e358df8 Mon Sep 17 00:00:00 2001 From: decfox Date: Fri, 22 Nov 2024 17:23:47 +0530 Subject: [PATCH 91/93] chore: upgrade UPSTREAM to go1.22.2 --- UPSTREAM | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/UPSTREAM b/UPSTREAM index e1e1d980..e8171842 100644 --- a/UPSTREAM +++ b/UPSTREAM @@ -1 +1 @@ -go1.21.11 +go1.22.2 From b933b81d5d907a4dbbda5e9be31570b0f714047d Mon Sep 17 00:00:00 2001 From: decfox Date: Sun, 24 Nov 2024 12:40:29 +0530 Subject: [PATCH 92/93] chore: sync with upstream --- .github/workflows/go.yml | 2 +- cgi/cgi_main.go | 3 +- cgi/host_test.go | 2 +- internal/bisect/bisect.go | 794 ++++++++++++++++++++++++++++++++++++ internal/godebug/godebug.go | 305 ++++++++++++++ internal/godebugs/table.go | 77 ++++ servemux121.go | 3 +- transfer.go | 2 +- 8 files changed, 1183 insertions(+), 5 deletions(-) create mode 100644 internal/bisect/bisect.go create mode 100644 internal/godebug/godebug.go create mode 100644 internal/godebugs/table.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index fc12031d..3e0e9fff 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: "1.21.8" + go-version: "1.22.2" - name: Build run: go build -v ./... diff --git a/cgi/cgi_main.go b/cgi/cgi_main.go index 8997d66a..26ab53a6 100644 --- a/cgi/cgi_main.go +++ b/cgi/cgi_main.go @@ -7,12 +7,13 @@ package cgi import ( "fmt" "io" - "net/http" "os" "path" "sort" "strings" "time" + + http "github.com/ooni/oohttp" ) func cgiMain() { diff --git a/cgi/host_test.go b/cgi/host_test.go index 80500363..76d8566c 100644 --- a/cgi/host_test.go +++ b/cgi/host_test.go @@ -9,7 +9,6 @@ package cgi import ( "bufio" "fmt" - "internal/testenv" "io" "net" "os" @@ -23,6 +22,7 @@ import ( http "github.com/ooni/oohttp" httptest "github.com/ooni/oohttp/httptest" + testenv "github.com/ooni/oohttp/internal/testenv" ) // TestMain executes the test binary as the cgi server if diff --git a/internal/bisect/bisect.go b/internal/bisect/bisect.go new file mode 100644 index 00000000..3e5a6849 --- /dev/null +++ b/internal/bisect/bisect.go @@ -0,0 +1,794 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package bisect can be used by compilers and other programs +// to serve as a target for the bisect debugging tool. +// See [golang.org/x/tools/cmd/bisect] for details about using the tool. +// +// To be a bisect target, allowing bisect to help determine which of a set of independent +// changes provokes a failure, a program needs to: +// +// 1. Define a way to accept a change pattern on its command line or in its environment. +// The most common mechanism is a command-line flag. +// The pattern can be passed to [New] to create a [Matcher], the compiled form of a pattern. +// +// 2. Assign each change a unique ID. One possibility is to use a sequence number, +// but the most common mechanism is to hash some kind of identifying information +// like the file and line number where the change might be applied. +// [Hash] hashes its arguments to compute an ID. +// +// 3. Enable each change that the pattern says should be enabled. +// The [Matcher.ShouldEnable] method answers this question for a given change ID. +// +// 4. Print a report identifying each change that the pattern says should be printed. +// The [Matcher.ShouldPrint] method answers this question for a given change ID. +// The report consists of one more lines on standard error or standard output +// that contain a “match marker”. [Marker] returns the match marker for a given ID. +// When bisect reports a change as causing the failure, it identifies the change +// by printing the report lines with the match marker removed. +// +// # Example Usage +// +// A program starts by defining how it receives the pattern. In this example, we will assume a flag. +// The next step is to compile the pattern: +// +// m, err := bisect.New(patternFlag) +// if err != nil { +// log.Fatal(err) +// } +// +// Then, each time a potential change is considered, the program computes +// a change ID by hashing identifying information (source file and line, in this case) +// and then calls m.ShouldPrint and m.ShouldEnable to decide whether to +// print and enable the change, respectively. The two can return different values +// depending on whether bisect is trying to find a minimal set of changes to +// disable or to enable to provoke the failure. +// +// It is usually helpful to write a helper function that accepts the identifying information +// and then takes care of hashing, printing, and reporting whether the identified change +// should be enabled. For example, a helper for changes identified by a file and line number +// would be: +// +// func ShouldEnable(file string, line int) { +// h := bisect.Hash(file, line) +// if m.ShouldPrint(h) { +// fmt.Fprintf(os.Stderr, "%v %s:%d\n", bisect.Marker(h), file, line) +// } +// return m.ShouldEnable(h) +// } +// +// Finally, note that New returns a nil Matcher when there is no pattern, +// meaning that the target is not running under bisect at all, +// so all changes should be enabled and none should be printed. +// In that common case, the computation of the hash can be avoided entirely +// by checking for m == nil first: +// +// func ShouldEnable(file string, line int) bool { +// if m == nil { +// return true +// } +// h := bisect.Hash(file, line) +// if m.ShouldPrint(h) { +// fmt.Fprintf(os.Stderr, "%v %s:%d\n", bisect.Marker(h), file, line) +// } +// return m.ShouldEnable(h) +// } +// +// When the identifying information is expensive to format, this code can call +// [Matcher.MarkerOnly] to find out whether short report lines containing only the +// marker are permitted for a given run. (Bisect permits such lines when it is +// still exploring the space of possible changes and will not be showing the +// output to the user.) If so, the client can choose to print only the marker: +// +// func ShouldEnable(file string, line int) bool { +// if m == nil { +// return true +// } +// h := bisect.Hash(file, line) +// if m.ShouldPrint(h) { +// if m.MarkerOnly() { +// bisect.PrintMarker(os.Stderr, h) +// } else { +// fmt.Fprintf(os.Stderr, "%v %s:%d\n", bisect.Marker(h), file, line) +// } +// } +// return m.ShouldEnable(h) +// } +// +// This specific helper – deciding whether to enable a change identified by +// file and line number and printing about the change when necessary – is +// provided by the [Matcher.FileLine] method. +// +// Another common usage is deciding whether to make a change in a function +// based on the caller's stack, to identify the specific calling contexts that the +// change breaks. The [Matcher.Stack] method takes care of obtaining the stack, +// printing it when necessary, and reporting whether to enable the change +// based on that stack. +// +// # Pattern Syntax +// +// Patterns are generated by the bisect tool and interpreted by [New]. +// Users should not have to understand the patterns except when +// debugging a target's bisect support or debugging the bisect tool itself. +// +// The pattern syntax selecting a change is a sequence of bit strings +// separated by + and - operators. Each bit string denotes the set of +// changes with IDs ending in those bits, + is set addition, - is set subtraction, +// and the expression is evaluated in the usual left-to-right order. +// The special binary number “y” denotes the set of all changes, +// standing in for the empty bit string. +// In the expression, all the + operators must appear before all the - operators. +// A leading + adds to an empty set. A leading - subtracts from the set of all +// possible suffixes. +// +// For example: +// +// - “01+10” and “+01+10” both denote the set of changes +// with IDs ending with the bits 01 or 10. +// +// - “01+10-1001” denotes the set of changes with IDs +// ending with the bits 01 or 10, but excluding those ending in 1001. +// +// - “-01-1000” and “y-01-1000 both denote the set of all changes +// with IDs not ending in 01 nor 1000. +// +// - “0+1-01+001” is not a valid pattern, because all the + operators do not +// appear before all the - operators. +// +// In the syntaxes described so far, the pattern specifies the changes to +// enable and report. If a pattern is prefixed by a “!”, the meaning +// changes: the pattern specifies the changes to DISABLE and report. This +// mode of operation is needed when a program passes with all changes +// enabled but fails with no changes enabled. In this case, bisect +// searches for minimal sets of changes to disable. +// Put another way, the leading “!” inverts the result from [Matcher.ShouldEnable] +// but does not invert the result from [Matcher.ShouldPrint]. +// +// As a convenience for manual debugging, “n” is an alias for “!y”, +// meaning to disable and report all changes. +// +// Finally, a leading “v” in the pattern indicates that the reports will be shown +// to the user of bisect to describe the changes involved in a failure. +// At the API level, the leading “v” causes [Matcher.Visible] to return true. +// See the next section for details. +// +// # Match Reports +// +// The target program must enable only those changed matched +// by the pattern, and it must print a match report for each such change. +// A match report consists of one or more lines of text that will be +// printed by the bisect tool to describe a change implicated in causing +// a failure. Each line in the report for a given change must contain a +// match marker with that change ID, as returned by [Marker]. +// The markers are elided when displaying the lines to the user. +// +// A match marker has the form “[bisect-match 0x1234]” where +// 0x1234 is the change ID in hexadecimal. +// An alternate form is “[bisect-match 010101]”, giving the change ID in binary. +// +// When [Matcher.Visible] returns false, the match reports are only +// being processed by bisect to learn the set of enabled changes, +// not shown to the user, meaning that each report can be a match +// marker on a line by itself, eliding the usual textual description. +// When the textual description is expensive to compute, +// checking [Matcher.Visible] can help the avoid that expense +// in most runs. +package bisect + +import ( + "runtime" + "sync" + "sync/atomic" + "unsafe" +) + +// New creates and returns a new Matcher implementing the given pattern. +// The pattern syntax is defined in the package doc comment. +// +// In addition to the pattern syntax syntax, New("") returns nil, nil. +// The nil *Matcher is valid for use: it returns true from ShouldEnable +// and false from ShouldPrint for all changes. Callers can avoid calling +// [Hash], [Matcher.ShouldEnable], and [Matcher.ShouldPrint] entirely +// when they recognize the nil Matcher. +func New(pattern string) (*Matcher, error) { + if pattern == "" { + return nil, nil + } + + m := new(Matcher) + + p := pattern + // Special case for leading 'q' so that 'qn' quietly disables, e.g. fmahash=qn to disable fma + // Any instance of 'v' disables 'q'. + if len(p) > 0 && p[0] == 'q' { + m.quiet = true + p = p[1:] + if p == "" { + return nil, &parseError{"invalid pattern syntax: " + pattern} + } + } + // Allow multiple v, so that “bisect cmd vPATTERN” can force verbose all the time. + for len(p) > 0 && p[0] == 'v' { + m.verbose = true + m.quiet = false + p = p[1:] + if p == "" { + return nil, &parseError{"invalid pattern syntax: " + pattern} + } + } + + // Allow multiple !, each negating the last, so that “bisect cmd !PATTERN” works + // even when bisect chooses to add its own !. + m.enable = true + for len(p) > 0 && p[0] == '!' { + m.enable = !m.enable + p = p[1:] + if p == "" { + return nil, &parseError{"invalid pattern syntax: " + pattern} + } + } + + if p == "n" { + // n is an alias for !y. + m.enable = !m.enable + p = "y" + } + + // Parse actual pattern syntax. + result := true + bits := uint64(0) + start := 0 + wid := 1 // 1-bit (binary); sometimes 4-bit (hex) + for i := 0; i <= len(p); i++ { + // Imagine a trailing - at the end of the pattern to flush final suffix + c := byte('-') + if i < len(p) { + c = p[i] + } + if i == start && wid == 1 && c == 'x' { // leading x for hex + start = i + 1 + wid = 4 + continue + } + switch c { + default: + return nil, &parseError{"invalid pattern syntax: " + pattern} + case '2', '3', '4', '5', '6', '7', '8', '9': + if wid != 4 { + return nil, &parseError{"invalid pattern syntax: " + pattern} + } + fallthrough + case '0', '1': + bits <<= wid + bits |= uint64(c - '0') + case 'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F': + if wid != 4 { + return nil, &parseError{"invalid pattern syntax: " + pattern} + } + bits <<= 4 + bits |= uint64(c&^0x20 - 'A' + 10) + case 'y': + if i+1 < len(p) && (p[i+1] == '0' || p[i+1] == '1') { + return nil, &parseError{"invalid pattern syntax: " + pattern} + } + bits = 0 + case '+', '-': + if c == '+' && result == false { + // Have already seen a -. Should be - from here on. + return nil, &parseError{"invalid pattern syntax (+ after -): " + pattern} + } + if i > 0 { + n := (i - start) * wid + if n > 64 { + return nil, &parseError{"pattern bits too long: " + pattern} + } + if n <= 0 { + return nil, &parseError{"invalid pattern syntax: " + pattern} + } + if p[start] == 'y' { + n = 0 + } + mask := uint64(1)<= 0; i-- { + c := &m.list[i] + if id&c.mask == c.bits { + return c.result + } + } + return false +} + +// FileLine reports whether the change identified by file and line should be enabled. +// If the change should be printed, FileLine prints a one-line report to w. +func (m *Matcher) FileLine(w Writer, file string, line int) bool { + if m == nil { + return true + } + return m.fileLine(w, file, line) +} + +// fileLine does the real work for FileLine. +// This lets FileLine's body handle m == nil and potentially be inlined. +func (m *Matcher) fileLine(w Writer, file string, line int) bool { + h := Hash(file, line) + if m.ShouldPrint(h) { + if m.MarkerOnly() { + PrintMarker(w, h) + } else { + printFileLine(w, h, file, line) + } + } + return m.ShouldEnable(h) +} + +// printFileLine prints a non-marker-only report for file:line to w. +func printFileLine(w Writer, h uint64, file string, line int) error { + const markerLen = 40 // overestimate + b := make([]byte, 0, markerLen+len(file)+24) + b = AppendMarker(b, h) + b = appendFileLine(b, file, line) + b = append(b, '\n') + _, err := w.Write(b) + return err +} + +// appendFileLine appends file:line to dst, returning the extended slice. +func appendFileLine(dst []byte, file string, line int) []byte { + dst = append(dst, file...) + dst = append(dst, ':') + u := uint(line) + if line < 0 { + dst = append(dst, '-') + u = -u + } + var buf [24]byte + i := len(buf) + for i == len(buf) || u > 0 { + i-- + buf[i] = '0' + byte(u%10) + u /= 10 + } + dst = append(dst, buf[i:]...) + return dst +} + +// MatchStack assigns the current call stack a change ID. +// If the stack should be printed, MatchStack prints it. +// Then MatchStack reports whether a change at the current call stack should be enabled. +func (m *Matcher) Stack(w Writer) bool { + if m == nil { + return true + } + return m.stack(w) +} + +// stack does the real work for Stack. +// This lets stack's body handle m == nil and potentially be inlined. +func (m *Matcher) stack(w Writer) bool { + const maxStack = 16 + var stk [maxStack]uintptr + n := runtime.Callers(2, stk[:]) + // caller #2 is not for printing; need it to normalize PCs if ASLR. + if n <= 1 { + return false + } + + base := stk[0] + // normalize PCs + for i := range stk[:n] { + stk[i] -= base + } + + h := Hash(stk[:n]) + if m.ShouldPrint(h) { + var d *dedup + for { + d = m.dedup.Load() + if d != nil { + break + } + d = new(dedup) + if m.dedup.CompareAndSwap(nil, d) { + break + } + } + + if m.MarkerOnly() { + if !d.seenLossy(h) { + PrintMarker(w, h) + } + } else { + if !d.seen(h) { + // Restore PCs in stack for printing + for i := range stk[:n] { + stk[i] += base + } + printStack(w, h, stk[1:n]) + } + } + } + return m.ShouldEnable(h) +} + +// Writer is the same interface as io.Writer. +// It is duplicated here to avoid importing io. +type Writer interface { + Write([]byte) (int, error) +} + +// PrintMarker prints to w a one-line report containing only the marker for h. +// It is appropriate to use when [Matcher.ShouldPrint] and [Matcher.MarkerOnly] both return true. +func PrintMarker(w Writer, h uint64) error { + var buf [50]byte + b := AppendMarker(buf[:0], h) + b = append(b, '\n') + _, err := w.Write(b) + return err +} + +// printStack prints to w a multi-line report containing a formatting of the call stack stk, +// with each line preceded by the marker for h. +func printStack(w Writer, h uint64, stk []uintptr) error { + buf := make([]byte, 0, 2048) + + var prefixBuf [100]byte + prefix := AppendMarker(prefixBuf[:0], h) + + frames := runtime.CallersFrames(stk) + for { + f, more := frames.Next() + buf = append(buf, prefix...) + buf = append(buf, f.Func.Name()...) + buf = append(buf, "()\n"...) + buf = append(buf, prefix...) + buf = append(buf, '\t') + buf = appendFileLine(buf, f.File, f.Line) + buf = append(buf, '\n') + if !more { + break + } + } + buf = append(buf, prefix...) + buf = append(buf, '\n') + _, err := w.Write(buf) + return err +} + +// Marker returns the match marker text to use on any line reporting details +// about a match of the given ID. +// It always returns the hexadecimal format. +func Marker(id uint64) string { + return string(AppendMarker(nil, id)) +} + +// AppendMarker is like [Marker] but appends the marker to dst. +func AppendMarker(dst []byte, id uint64) []byte { + const prefix = "[bisect-match 0x" + var buf [len(prefix) + 16 + 1]byte + copy(buf[:], prefix) + for i := 0; i < 16; i++ { + buf[len(prefix)+i] = "0123456789abcdef"[id>>60] + id <<= 4 + } + buf[len(prefix)+16] = ']' + return append(dst, buf[:]...) +} + +// CutMarker finds the first match marker in line and removes it, +// returning the shortened line (with the marker removed), +// the ID from the match marker, +// and whether a marker was found at all. +// If there is no marker, CutMarker returns line, 0, false. +func CutMarker(line string) (short string, id uint64, ok bool) { + // Find first instance of prefix. + prefix := "[bisect-match " + i := 0 + for ; ; i++ { + if i >= len(line)-len(prefix) { + return line, 0, false + } + if line[i] == '[' && line[i:i+len(prefix)] == prefix { + break + } + } + + // Scan to ]. + j := i + len(prefix) + for j < len(line) && line[j] != ']' { + j++ + } + if j >= len(line) { + return line, 0, false + } + + // Parse id. + idstr := line[i+len(prefix) : j] + if len(idstr) >= 3 && idstr[:2] == "0x" { + // parse hex + if len(idstr) > 2+16 { // max 0x + 16 digits + return line, 0, false + } + for i := 2; i < len(idstr); i++ { + id <<= 4 + switch c := idstr[i]; { + case '0' <= c && c <= '9': + id |= uint64(c - '0') + case 'a' <= c && c <= 'f': + id |= uint64(c - 'a' + 10) + case 'A' <= c && c <= 'F': + id |= uint64(c - 'A' + 10) + } + } + } else { + if idstr == "" || len(idstr) > 64 { // min 1 digit, max 64 digits + return line, 0, false + } + // parse binary + for i := 0; i < len(idstr); i++ { + id <<= 1 + switch c := idstr[i]; c { + default: + return line, 0, false + case '0', '1': + id |= uint64(c - '0') + } + } + } + + // Construct shortened line. + // Remove at most one space from around the marker, + // so that "foo [marker] bar" shortens to "foo bar". + j++ // skip ] + if i > 0 && line[i-1] == ' ' { + i-- + } else if j < len(line) && line[j] == ' ' { + j++ + } + short = line[:i] + line[j:] + return short, id, true +} + +// Hash computes a hash of the data arguments, +// each of which must be of type string, byte, int, uint, int32, uint32, int64, uint64, uintptr, or a slice of one of those types. +func Hash(data ...any) uint64 { + h := offset64 + for _, v := range data { + switch v := v.(type) { + default: + // Note: Not printing the type, because reflect.ValueOf(v) + // would make the interfaces prepared by the caller escape + // and therefore allocate. This way, Hash(file, line) runs + // without any allocation. It should be clear from the + // source code calling Hash what the bad argument was. + panic("bisect.Hash: unexpected argument type") + case string: + h = fnvString(h, v) + case byte: + h = fnv(h, v) + case int: + h = fnvUint64(h, uint64(v)) + case uint: + h = fnvUint64(h, uint64(v)) + case int32: + h = fnvUint32(h, uint32(v)) + case uint32: + h = fnvUint32(h, v) + case int64: + h = fnvUint64(h, uint64(v)) + case uint64: + h = fnvUint64(h, v) + case uintptr: + h = fnvUint64(h, uint64(v)) + case []string: + for _, x := range v { + h = fnvString(h, x) + } + case []byte: + for _, x := range v { + h = fnv(h, x) + } + case []int: + for _, x := range v { + h = fnvUint64(h, uint64(x)) + } + case []uint: + for _, x := range v { + h = fnvUint64(h, uint64(x)) + } + case []int32: + for _, x := range v { + h = fnvUint32(h, uint32(x)) + } + case []uint32: + for _, x := range v { + h = fnvUint32(h, x) + } + case []int64: + for _, x := range v { + h = fnvUint64(h, uint64(x)) + } + case []uint64: + for _, x := range v { + h = fnvUint64(h, x) + } + case []uintptr: + for _, x := range v { + h = fnvUint64(h, uint64(x)) + } + } + } + return h +} + +// Trivial error implementation, here to avoid importing errors. + +// parseError is a trivial error implementation, +// defined here to avoid importing errors. +type parseError struct{ text string } + +func (e *parseError) Error() string { return e.text } + +// FNV-1a implementation. See Go's hash/fnv/fnv.go. +// Copied here for simplicity (can handle integers more directly) +// and to avoid importing hash/fnv. + +const ( + offset64 uint64 = 14695981039346656037 + prime64 uint64 = 1099511628211 +) + +func fnv(h uint64, x byte) uint64 { + h ^= uint64(x) + h *= prime64 + return h +} + +func fnvString(h uint64, x string) uint64 { + for i := 0; i < len(x); i++ { + h ^= uint64(x[i]) + h *= prime64 + } + return h +} + +func fnvUint64(h uint64, x uint64) uint64 { + for i := 0; i < 8; i++ { + h ^= x & 0xFF + x >>= 8 + h *= prime64 + } + return h +} + +func fnvUint32(h uint64, x uint32) uint64 { + for i := 0; i < 4; i++ { + h ^= uint64(x & 0xFF) + x >>= 8 + h *= prime64 + } + return h +} + +// A dedup is a deduplicator for call stacks, so that we only print +// a report for new call stacks, not for call stacks we've already +// reported. +// +// It has two modes: an approximate but lock-free mode that +// may still emit some duplicates, and a precise mode that uses +// a lock and never emits duplicates. +type dedup struct { + // 128-entry 4-way, lossy cache for seenLossy + recent [128][4]uint64 + + // complete history for seen + mu sync.Mutex + m map[uint64]bool +} + +// seen records that h has now been seen and reports whether it was seen before. +// When seen returns false, the caller is expected to print a report for h. +func (d *dedup) seen(h uint64) bool { + d.mu.Lock() + if d.m == nil { + d.m = make(map[uint64]bool) + } + seen := d.m[h] + d.m[h] = true + d.mu.Unlock() + return seen +} + +// seenLossy is a variant of seen that avoids a lock by using a cache of recently seen hashes. +// Each cache entry is N-way set-associative: h can appear in any of the slots. +// If h does not appear in any of them, then it is inserted into a random slot, +// overwriting whatever was there before. +func (d *dedup) seenLossy(h uint64) bool { + cache := &d.recent[uint(h)%uint(len(d.recent))] + for i := 0; i < len(cache); i++ { + if atomic.LoadUint64(&cache[i]) == h { + return true + } + } + + // Compute index in set to evict as hash of current set. + ch := offset64 + for _, x := range cache { + ch = fnvUint64(ch, x) + } + atomic.StoreUint64(&cache[uint(ch)%uint(len(cache))], h) + return false +} diff --git a/internal/godebug/godebug.go b/internal/godebug/godebug.go new file mode 100644 index 00000000..e93e9f9b --- /dev/null +++ b/internal/godebug/godebug.go @@ -0,0 +1,305 @@ +// Copyright 2021 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package godebug makes the settings in the $GODEBUG environment variable +// available to other packages. These settings are often used for compatibility +// tweaks, when we need to change a default behavior but want to let users +// opt back in to the original. For example GODEBUG=http2server=0 disables +// HTTP/2 support in the net/http server. +// +// In typical usage, code should declare a Setting as a global +// and then call Value each time the current setting value is needed: +// +// var http2server = godebug.New("http2server") +// +// func ServeConn(c net.Conn) { +// if http2server.Value() == "0" { +// disallow HTTP/2 +// ... +// } +// ... +// } +// +// Each time a non-default setting causes a change in program behavior, +// code should call [Setting.IncNonDefault] to increment a counter that can +// be reported by [runtime/metrics.Read]. +// Note that counters used with IncNonDefault must be added to +// various tables in other packages. See the [Setting.IncNonDefault] +// documentation for details. +package godebug + +// Note: Be careful about new imports here. Any package +// that internal/godebug imports cannot itself import internal/godebug, +// meaning it cannot introduce a GODEBUG setting of its own. +// We keep imports to the absolute bare minimum. +import ( + _ "runtime" + "sync" + "sync/atomic" + "unsafe" + _ "unsafe" // go:linkname + + "github.com/ooni/oohttp/internal/bisect" + "github.com/ooni/oohttp/internal/godebugs" +) + +// A Setting is a single setting in the $GODEBUG environment variable. +type Setting struct { + name string + once sync.Once + *setting +} + +type setting struct { + value atomic.Pointer[value] + nonDefaultOnce sync.Once + nonDefault atomic.Uint64 + info *godebugs.Info +} + +type value struct { + text string + bisect *bisect.Matcher +} + +// New returns a new Setting for the $GODEBUG setting with the given name. +// +// GODEBUGs meant for use by end users must be listed in ../godebugs/table.go, +// which is used for generating and checking various documentation. +// If the name is not listed in that table, New will succeed but calling Value +// on the returned Setting will panic. +// To disable that panic for access to an undocumented setting, +// prefix the name with a #, as in godebug.New("#gofsystrace"). +// The # is a signal to New but not part of the key used in $GODEBUG. +func New(name string) *Setting { + return &Setting{name: name} +} + +// Name returns the name of the setting. +func (s *Setting) Name() string { + if s.name != "" && s.name[0] == '#' { + return s.name[1:] + } + return s.name +} + +// Undocumented reports whether this is an undocumented setting. +func (s *Setting) Undocumented() bool { + return s.name != "" && s.name[0] == '#' +} + +// String returns a printable form for the setting: name=value. +func (s *Setting) String() string { + return s.Name() + "=" + s.Value() +} + +// IncNonDefault increments the non-default behavior counter +// associated with the given setting. +// This counter is exposed in the runtime/metrics value +// /godebug/non-default-behavior/:events. +// +// Note that Value must be called at least once before IncNonDefault. +func (s *Setting) IncNonDefault() { + s.nonDefaultOnce.Do(s.register) + s.nonDefault.Add(1) +} + +func (s *Setting) register() { + if s.info == nil || s.info.Opaque { + panic("godebug: unexpected IncNonDefault of " + s.name) + } + registerMetric("/godebug/non-default-behavior/"+s.Name()+":events", s.nonDefault.Load) +} + +// cache is a cache of all the GODEBUG settings, +// a locked map[string]*atomic.Pointer[string]. +// +// All Settings with the same name share a single +// *atomic.Pointer[string], so that when GODEBUG +// changes only that single atomic string pointer +// needs to be updated. +// +// A name appears in the values map either if it is the +// name of a Setting for which Value has been called +// at least once, or if the name has ever appeared in +// a name=value pair in the $GODEBUG environment variable. +// Once entered into the map, the name is never removed. +var cache sync.Map // name string -> value *atomic.Pointer[string] + +var empty value + +// Value returns the current value for the GODEBUG setting s. +// +// Value maintains an internal cache that is synchronized +// with changes to the $GODEBUG environment variable, +// making Value efficient to call as frequently as needed. +// Clients should therefore typically not attempt their own +// caching of Value's result. +func (s *Setting) Value() string { + s.once.Do(func() { + s.setting = lookup(s.Name()) + if s.info == nil && !s.Undocumented() { + panic("godebug: Value of name not listed in godebugs.All: " + s.name) + } + }) + v := *s.value.Load() + if v.bisect != nil && !v.bisect.Stack(&stderr) { + return "" + } + return v.text +} + +// lookup returns the unique *setting value for the given name. +func lookup(name string) *setting { + if v, ok := cache.Load(name); ok { + return v.(*setting) + } + s := new(setting) + s.info = godebugs.Lookup(name) + s.value.Store(&empty) + if v, loaded := cache.LoadOrStore(name, s); loaded { + // Lost race: someone else created it. Use theirs. + return v.(*setting) + } + + return s +} + +var godebugDefault string +var godebugUpdate atomic.Pointer[func(string, string)] +var godebugEnv atomic.Pointer[string] // set by parsedebugvars +var godebugNewIncNonDefault atomic.Pointer[func(string) func()] + +// setUpdate mimcs runtime.godebug_setUpdate +func setUpdate(update func(string, string)) { + p := new(func(string, string)) + *p = update + godebugUpdate.Store(p) + godebugNotify(false) +} + +// godebugNotify mimics runtime.godebugNotify +func godebugNotify(envChanged bool) { + update := godebugUpdate.Load() + var env string + if p := godebugEnv.Load(); p != nil { + env = *p + } + // NOTE: code specific to github.com/ooni/oohttp + // we omit this check since the only invocation we have is with envChanged = false + // this is done to avoid forking a large portion of the runtime and internal packages + // if envChanged { + // reparsedebugvars(env) + // } + if update != nil { + (*update)(godebugDefault, env) + } +} + +// registerMetric is an empty function +func registerMetric(name string, read func() uint64) { + // we do not register metrics for this godebug fork +} + +// setNewIncNonDefault mimics runtime.godebug_setNewIncNonDefault +func setNewIncNonDefault(newIncNonDefault func(string) func()) { + p := new(func(string) func()) + *p = newIncNonDefault + godebugNewIncNonDefault.Store(p) +} + +func init() { + setUpdate(update) + setNewIncNonDefault(newIncNonDefault) +} + +func newIncNonDefault(name string) func() { + s := New(name) + s.Value() + return s.IncNonDefault +} + +var updateMu sync.Mutex + +// update records an updated GODEBUG setting. +// def is the default GODEBUG setting for the running binary, +// and env is the current value of the $GODEBUG environment variable. +func update(def, env string) { + updateMu.Lock() + defer updateMu.Unlock() + + // Update all the cached values, creating new ones as needed. + // We parse the environment variable first, so that any settings it has + // are already locked in place (did[name] = true) before we consider + // the defaults. + did := make(map[string]bool) + parse(did, env) + parse(did, def) + + // Clear any cached values that are no longer present. + cache.Range(func(name, s any) bool { + if !did[name.(string)] { + s.(*setting).value.Store(&empty) + } + return true + }) +} + +// parse parses the GODEBUG setting string s, +// which has the form k=v,k2=v2,k3=v3. +// Later settings override earlier ones. +// Parse only updates settings k=v for which did[k] = false. +// It also sets did[k] = true for settings that it updates. +// Each value v can also have the form v#pattern, +// in which case the GODEBUG is only enabled for call stacks +// matching pattern, for use with golang.org/x/tools/cmd/bisect. +func parse(did map[string]bool, s string) { + // Scan the string backward so that later settings are used + // and earlier settings are ignored. + // Note that a forward scan would cause cached values + // to temporarily use the ignored value before being + // updated to the "correct" one. + end := len(s) + eq := -1 + for i := end - 1; i >= -1; i-- { + if i == -1 || s[i] == ',' { + if eq >= 0 { + name, arg := s[i+1:eq], s[eq+1:end] + if !did[name] { + did[name] = true + v := &value{text: arg} + for j := 0; j < len(arg); j++ { + if arg[j] == '#' { + v.text = arg[:j] + v.bisect, _ = bisect.New(arg[j+1:]) + break + } + } + lookup(name).value.Store(v) + } + } + eq = -1 + end = i + } else if s[i] == '=' { + eq = i + } + } +} + +type runtimeStderr struct{} + +var stderr runtimeStderr + +func (*runtimeStderr) Write(b []byte) (int, error) { + if len(b) > 0 { + write(2, unsafe.Pointer(&b[0]), int32(len(b))) + } + return len(b), nil +} + +// Since we cannot import os or syscall, use the runtime's write function +// to print to standard error. +// +//go:linkname write runtime.write +func write(fd uintptr, p unsafe.Pointer, n int32) int32 diff --git a/internal/godebugs/table.go b/internal/godebugs/table.go new file mode 100644 index 00000000..11c5b7d6 --- /dev/null +++ b/internal/godebugs/table.go @@ -0,0 +1,77 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package godebugs provides a table of known GODEBUG settings, +// for use by a variety of other packages, including internal/godebug, +// runtime, runtime/metrics, and cmd/go/internal/load. +package godebugs + +// An Info describes a single known GODEBUG setting. +type Info struct { + Name string // name of the setting ("panicnil") + Package string // package that uses the setting ("runtime") + Changed int // minor version when default changed, if any; 21 means Go 1.21 + Old string // value that restores behavior prior to Changed + Opaque bool // setting does not export information to runtime/metrics using [internal/godebug.Setting.IncNonDefault] +} + +// All is the table of known settings, sorted by Name. +// +// Note: After adding entries to this table, run 'go generate runtime/metrics' +// to update the runtime/metrics doc comment. +// (Otherwise the runtime/metrics test will fail.) +// +// Note: After adding entries to this table, update the list in doc/godebug.md as well. +// (Otherwise the test in this package will fail.) +var All = []Info{ + {Name: "execerrdot", Package: "os/exec"}, + {Name: "gocachehash", Package: "cmd/go"}, + {Name: "gocachetest", Package: "cmd/go"}, + {Name: "gocacheverify", Package: "cmd/go"}, + {Name: "gotypesalias", Package: "go/types"}, + {Name: "http2client", Package: "net/http"}, + {Name: "http2debug", Package: "net/http", Opaque: true}, + {Name: "http2server", Package: "net/http"}, + {Name: "httplaxcontentlength", Package: "net/http", Changed: 22, Old: "1"}, + {Name: "httpmuxgo121", Package: "net/http", Changed: 22, Old: "1"}, + {Name: "installgoroot", Package: "go/build"}, + {Name: "jstmpllitinterp", Package: "html/template"}, + //{Name: "multipartfiles", Package: "mime/multipart"}, + {Name: "multipartmaxheaders", Package: "mime/multipart"}, + {Name: "multipartmaxparts", Package: "mime/multipart"}, + {Name: "multipathtcp", Package: "net"}, + {Name: "netdns", Package: "net", Opaque: true}, + {Name: "netedns0", Package: "net", Changed: 19, Old: "0"}, + {Name: "panicnil", Package: "runtime", Changed: 21, Old: "1"}, + {Name: "randautoseed", Package: "math/rand"}, + {Name: "tarinsecurepath", Package: "archive/tar"}, + {Name: "tls10server", Package: "crypto/tls", Changed: 22, Old: "1"}, + {Name: "tlsmaxrsasize", Package: "crypto/tls"}, + {Name: "tlsrsakex", Package: "crypto/tls", Changed: 22, Old: "1"}, + {Name: "tlsunsafeekm", Package: "crypto/tls", Changed: 22, Old: "1"}, + {Name: "x509sha1", Package: "crypto/x509"}, + {Name: "x509usefallbackroots", Package: "crypto/x509"}, + {Name: "x509usepolicies", Package: "crypto/x509"}, + {Name: "zipinsecurepath", Package: "archive/zip"}, +} + +// Lookup returns the Info with the given name. +func Lookup(name string) *Info { + // binary search, avoiding import of sort. + lo := 0 + hi := len(All) + for lo < hi { + m := int(uint(lo+hi) >> 1) + mid := All[m].Name + if name == mid { + return &All[m] + } + if name < mid { + hi = m + } else { + lo = m + 1 + } + } + return nil +} diff --git a/servemux121.go b/servemux121.go index c0a4b770..cf79bfaf 100644 --- a/servemux121.go +++ b/servemux121.go @@ -11,11 +11,12 @@ package http // they mostly involve renaming functions, usually by unexporting them. import ( - "internal/godebug" "net/url" "sort" "strings" "sync" + + "github.com/ooni/oohttp/internal/godebug" ) var httpmuxgo121 = godebug.New("httpmuxgo121") diff --git a/transfer.go b/transfer.go index 895a5fad..9ae339ec 100644 --- a/transfer.go +++ b/transfer.go @@ -9,7 +9,6 @@ import ( "bytes" "errors" "fmt" - "internal/godebug" "io" "net/textproto" "reflect" @@ -22,6 +21,7 @@ import ( httptrace "github.com/ooni/oohttp/httptrace" internal "github.com/ooni/oohttp/internal" ascii "github.com/ooni/oohttp/internal/ascii" + "github.com/ooni/oohttp/internal/godebug" "golang.org/x/net/http/httpguts" ) From 9e5eae337e56db976657357b4386cadfad148751 Mon Sep 17 00:00:00 2001 From: decfox Date: Sun, 24 Nov 2024 23:09:15 +0530 Subject: [PATCH 93/93] chore: upgrade dependencies --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 13987b60..bca4dd8e 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,6 @@ module github.com/ooni/oohttp go 1.21 -require golang.org/x/net v0.22.0 +require golang.org/x/net v0.31.0 -require golang.org/x/text v0.14.0 // indirect +require golang.org/x/text v0.20.0 // indirect diff --git a/go.sum b/go.sum index 6ce074b7..2a7ab588 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,4 @@ -golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= -golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= +golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=