From f35a7fa466ffb06c38dcb3216e30c13aa8e14ce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20Br=C3=BCheim?= Date: Tue, 30 Mar 2021 02:47:19 +0200 Subject: [PATCH] encode,staticfiles: Content negotiation, precompressed files (#4045) * encode: implement prefer setting * encode: minimum_length configurable via caddyfile * encode: configurable content-types which to encode * file_server: support precompressed files * encode: use ReponseMatcher for conditional encoding of content * linting error & documentation of encode.PrecompressedOrder * encode: allow just one response matcher also change the namespace of the encoders back, I accidently changed to precompressed >.> default matchers include a * to match to any charset, that may be appended * rounding of the PR * added integration tests for new caddyfile directives * improved various doc strings (punctuation and typos) * added json tag for file_server precompress order and encode matcher * file_server: add vary header, remove accept-ranges when serving precompressed files * encode: move Suffix implementation to precompressed modules --- .../caddyfile_adapt/encode_options.txt | 66 +++++ .../file_server_precompressed.txt | 41 +++ caddytest/integration/caddyfile_adapt_test.go | 7 + .../encode/brotli/brotli_precompressed.go | 31 +++ modules/caddyhttp/encode/caddyfile.go | 139 ++++++++-- modules/caddyhttp/encode/encode.go | 104 ++++++-- modules/caddyhttp/encode/encode_test.go | 251 ++++++++++++++++++ .../encode/gzip/gzip_precompressed.go | 28 ++ .../encode/zstd/zstd_precompressed.go | 28 ++ modules/caddyhttp/fileserver/caddyfile.go | 31 ++- modules/caddyhttp/fileserver/staticfiles.go | 90 ++++++- modules/caddyhttp/standard/imports.go | 1 + 12 files changed, 768 insertions(+), 49 deletions(-) create mode 100644 caddytest/integration/caddyfile_adapt/encode_options.txt create mode 100644 caddytest/integration/caddyfile_adapt/file_server_precompressed.txt create mode 100644 modules/caddyhttp/encode/brotli/brotli_precompressed.go create mode 100644 modules/caddyhttp/encode/gzip/gzip_precompressed.go create mode 100644 modules/caddyhttp/encode/zstd/zstd_precompressed.go diff --git a/caddytest/integration/caddyfile_adapt/encode_options.txt b/caddytest/integration/caddyfile_adapt/encode_options.txt new file mode 100644 index 00000000000..a43c7e0285c --- /dev/null +++ b/caddytest/integration/caddyfile_adapt/encode_options.txt @@ -0,0 +1,66 @@ +:80 + +encode gzip zstd { + minimum_length 256 + prefer zstd gzip + match { + status 2xx 4xx 500 + header Content-Type text/* + header Content-Type application/json* + header Content-Type application/javascript* + header Content-Type application/xhtml+xml* + header Content-Type application/atom+xml* + header Content-Type application/rss+xml* + header Content-Type image/svg+xml* + } +} +---------- +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":80" + ], + "routes": [ + { + "handle": [ + { + "encodings": { + "gzip": {}, + "zstd": {} + }, + "handler": "encode", + "match": { + "headers": { + "Content-Type": [ + "text/*", + "application/json*", + "application/javascript*", + "application/xhtml+xml*", + "application/atom+xml*", + "application/rss+xml*", + "image/svg+xml*" + ] + }, + "status_code": [ + 2, + 4, + 500 + ] + }, + "minimum_length": 256, + "prefer": [ + "zstd", + "gzip" + ] + } + ] + } + ] + } + } + } + } +} diff --git a/caddytest/integration/caddyfile_adapt/file_server_precompressed.txt b/caddytest/integration/caddyfile_adapt/file_server_precompressed.txt new file mode 100644 index 00000000000..ac7d7eddd99 --- /dev/null +++ b/caddytest/integration/caddyfile_adapt/file_server_precompressed.txt @@ -0,0 +1,41 @@ +:80 + +file_server { + precompressed zstd br gzip +} +---------- +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":80" + ], + "routes": [ + { + "handle": [ + { + "handler": "file_server", + "hide": [ + "./Caddyfile" + ], + "precompressed": { + "br": {}, + "gzip": {}, + "zstd": {} + }, + "precompressed_order": [ + "zstd", + "br", + "gzip" + ] + } + ] + } + ] + } + } + } + } +} diff --git a/caddytest/integration/caddyfile_adapt_test.go b/caddytest/integration/caddyfile_adapt_test.go index 8794fe93a81..e10989994e3 100644 --- a/caddytest/integration/caddyfile_adapt_test.go +++ b/caddytest/integration/caddyfile_adapt_test.go @@ -1,7 +1,10 @@ package integration import ( + jsonMod "encoding/json" + "fmt" "io/ioutil" + "path/filepath" "regexp" "strings" "testing" @@ -39,6 +42,10 @@ func TestCaddyfileAdaptToJSON(t *testing.T) { // replace windows newlines in the json with unix newlines json = winNewlines.ReplaceAllString(json, "\n") + // replace os-specific default path for file_server's hide field + replacePath, _ := jsonMod.Marshal(fmt.Sprint(".", string(filepath.Separator), "Caddyfile")) + json = strings.ReplaceAll(json, `"./Caddyfile"`, string(replacePath)) + // run the test ok := caddytest.CompareAdapt(t, filename, caddyfile, "caddyfile", json) if !ok { diff --git a/modules/caddyhttp/encode/brotli/brotli_precompressed.go b/modules/caddyhttp/encode/brotli/brotli_precompressed.go new file mode 100644 index 00000000000..fbd04418e45 --- /dev/null +++ b/modules/caddyhttp/encode/brotli/brotli_precompressed.go @@ -0,0 +1,31 @@ +package caddybrotli + +import ( + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode" +) + +func init() { + caddy.RegisterModule(BrotliPrecompressed{}) +} + +// BrotliPrecompressed provides the file extension for files precompressed with brotli encoding. +type BrotliPrecompressed struct{} + +// CaddyModule returns the Caddy module information. +func (BrotliPrecompressed) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "http.precompressed.br", + New: func() caddy.Module { return new(BrotliPrecompressed) }, + } +} + +// AcceptEncoding returns the name of the encoding as +// used in the Accept-Encoding request headers. +func (BrotliPrecompressed) AcceptEncoding() string { return "br" } + +// Suffix returns the filename suffix of precompressed files. +func (BrotliPrecompressed) Suffix() string { return ".br" } + +// Interface guards +var _ encode.Precompressed = (*BrotliPrecompressed)(nil) diff --git a/modules/caddyhttp/encode/caddyfile.go b/modules/caddyhttp/encode/caddyfile.go index 2f11ca06b7f..c45f159e66c 100644 --- a/modules/caddyhttp/encode/caddyfile.go +++ b/modules/caddyhttp/encode/caddyfile.go @@ -15,7 +15,9 @@ package encode import ( - "fmt" + "net/http" + "strconv" + "strings" "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig" @@ -40,21 +42,31 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) // UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax: // // encode [] { -// gzip [] +// gzip [] // zstd +// minimum_length +// prefer +// # response matcher block +// match { +// status +// header [] +// } +// # or response matcher single line syntax +// match [header []] | [status ] // } // // Specifying the formats on the first line will use those formats' defaults. func (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + responseMatchers := make(map[string]caddyhttp.ResponseMatcher) for d.Next() { for _, arg := range d.RemainingArgs() { mod, err := caddy.GetModule("http.encoders." + arg) if err != nil { - return fmt.Errorf("finding encoder module '%s': %v", mod, err) + return d.Errf("finding encoder module '%s': %v", mod, err) } encoding, ok := mod.New().(Encoding) if !ok { - return fmt.Errorf("module %s is not an HTTP encoding", mod) + return d.Errf("module %s is not an HTTP encoding", mod) } if enc.EncodingsRaw == nil { enc.EncodingsRaw = make(caddy.ModuleMap) @@ -63,25 +75,118 @@ func (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { } for d.NextBlock(0) { - name := d.Val() - modID := "http.encoders." + name - unm, err := caddyfile.UnmarshalModule(d, modID) - if err != nil { - return err - } - encoding, ok := unm.(Encoding) - if !ok { - return fmt.Errorf("module %s is not an HTTP encoding; is %T", modID, unm) + switch d.Val() { + case "minimum_length": + if !d.NextArg() { + return d.ArgErr() + } + minLength, err := strconv.Atoi(d.Val()) + if err != nil { + return err + } + enc.MinLength = minLength + case "prefer": + var encs []string + for d.NextArg() { + encs = append(encs, d.Val()) + } + if len(encs) == 0 { + return d.ArgErr() + } + enc.Prefer = encs + case "match": + err := enc.parseNamedResponseMatcher(d.NewFromNextSegment(), responseMatchers) + if err != nil { + return err + } + matcher := responseMatchers["match"] + enc.Matcher = &matcher + default: + name := d.Val() + modID := "http.encoders." + name + unm, err := caddyfile.UnmarshalModule(d, modID) + if err != nil { + return err + } + encoding, ok := unm.(Encoding) + if !ok { + return d.Errf("module %s is not an HTTP encoding; is %T", modID, unm) + } + if enc.EncodingsRaw == nil { + enc.EncodingsRaw = make(caddy.ModuleMap) + } + enc.EncodingsRaw[name] = caddyconfig.JSON(encoding, nil) } - if enc.EncodingsRaw == nil { - enc.EncodingsRaw = make(caddy.ModuleMap) - } - enc.EncodingsRaw[name] = caddyconfig.JSON(encoding, nil) } } return nil } +// Parse the tokens of a named response matcher. +// +// match { +// header [] +// status +// } +// +// Or, single line syntax: +// +// match [header []] | [status ] +// +func (enc *Encode) parseNamedResponseMatcher(d *caddyfile.Dispenser, matchers map[string]caddyhttp.ResponseMatcher) error { + for d.Next() { + definitionName := d.Val() + + if _, ok := matchers[definitionName]; ok { + return d.Errf("matcher is defined more than once: %s", definitionName) + } + + matcher := caddyhttp.ResponseMatcher{} + for nesting := d.Nesting(); d.NextArg() || d.NextBlock(nesting); { + switch d.Val() { + case "header": + if matcher.Headers == nil { + matcher.Headers = http.Header{} + } + + // reuse the header request matcher's unmarshaler + headerMatcher := caddyhttp.MatchHeader(matcher.Headers) + err := headerMatcher.UnmarshalCaddyfile(d.NewFromNextSegment()) + if err != nil { + return err + } + + matcher.Headers = http.Header(headerMatcher) + case "status": + if matcher.StatusCode == nil { + matcher.StatusCode = []int{} + } + + args := d.RemainingArgs() + if len(args) == 0 { + return d.ArgErr() + } + + for _, arg := range args { + if len(arg) == 3 && strings.HasSuffix(arg, "xx") { + arg = arg[:1] + } + statusNum, err := strconv.Atoi(arg) + if err != nil { + return d.Errf("bad status value '%s': %v", arg, err) + } + matcher.StatusCode = append(matcher.StatusCode, statusNum) + } + default: + return d.Errf("unrecognized response matcher %s", d.Val()) + } + } + + matchers[definitionName] = matcher + } + return nil +} + // Interface guard var _ caddyfile.Unmarshaler = (*Encode)(nil) diff --git a/modules/caddyhttp/encode/encode.go b/modules/caddyhttp/encode/encode.go index e42eeed8cb1..a7abb3f90d3 100644 --- a/modules/caddyhttp/encode/encode.go +++ b/modules/caddyhttp/encode/encode.go @@ -23,6 +23,7 @@ import ( "bytes" "fmt" "io" + "math" "net/http" "sort" "strconv" @@ -43,12 +44,16 @@ type Encode struct { // will be chosen based on the client's Accept-Encoding header. EncodingsRaw caddy.ModuleMap `json:"encodings,omitempty" caddy:"namespace=http.encoders"` - // If the client has no strong preference, choose this encoding. TODO: Not yet implemented - // Prefer []string `json:"prefer,omitempty"` + // If the client has no strong preference, choose these encodings in order. + Prefer []string `json:"prefer,omitempty"` // Only encode responses that are at least this many bytes long. MinLength int `json:"minimum_length,omitempty"` + // Only encode responses that match against this ResponseMmatcher. + // The default is a collection of text-based Content-Type headers. + Matcher *caddyhttp.ResponseMatcher `json:"match,omitempty"` + writerPools map[string]*sync.Pool // TODO: these pools do not get reused through config reloads... } @@ -75,11 +80,46 @@ func (enc *Encode) Provision(ctx caddy.Context) error { if enc.MinLength == 0 { enc.MinLength = defaultMinLength } + + if enc.Matcher == nil { + // common text-based content types + enc.Matcher = &caddyhttp.ResponseMatcher{ + Headers: http.Header{ + "Content-Type": []string{ + "text/*", + "application/json*", + "application/javascript*", + "application/xhtml+xml*", + "application/atom+xml*", + "application/rss+xml*", + "image/svg+xml*", + }, + }, + } + } + + return nil +} + +// Validate ensures that enc's configuration is valid. +func (enc *Encode) Validate() error { + check := make(map[string]bool) + for _, encName := range enc.Prefer { + if _, ok := enc.writerPools[encName]; !ok { + return fmt.Errorf("encoding %s not enabled", encName) + } + + if _, ok := check[encName]; ok { + return fmt.Errorf("encoding %s is duplicated in prefer", encName) + } + check[encName] = true + } + return nil } func (enc *Encode) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { - for _, encName := range acceptedEncodings(r) { + for _, encName := range AcceptedEncodings(r, enc.Prefer) { if _, ok := enc.writerPools[encName]; !ok { continue // encoding not offered } @@ -150,6 +190,11 @@ func (rw *responseWriter) WriteHeader(status int) { rw.statusCode = status } +// Match determines, if encoding should be done based on the ResponseMatcher. +func (enc *Encode) Match(rw *responseWriter) bool { + return enc.Matcher.Match(rw.statusCode, rw.Header()) +} + // Write writes to the response. If the response qualifies, // it is encoded using the encoder, which is initialized // if not done so already. @@ -240,7 +285,10 @@ func (rw *responseWriter) Close() error { // init should be called before we write a response, if rw.buf has contents. func (rw *responseWriter) init() { - if rw.Header().Get("Content-Encoding") == "" && rw.buf.Len() >= rw.config.MinLength { + if rw.Header().Get("Content-Encoding") == "" && + rw.buf.Len() >= rw.config.MinLength && + rw.config.Match(rw) { + rw.w = rw.config.writerPools[rw.encodingName].Get().(Encoder) rw.w.Reset(rw.ResponseWriter) rw.Header().Del("Content-Length") // https://github.com/golang/go/issues/14975 @@ -250,12 +298,14 @@ func (rw *responseWriter) init() { rw.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-encoded content } -// acceptedEncodings returns the list of encodings that the -// client supports, in descending order of preference. If +// AcceptedEncodings returns the list of encodings that the +// client supports, in descending order of preference. +// The client preference via q-factor and the server +// preference via Prefer setting are taken into account. If // the Sec-WebSocket-Key header is present then non-identity // encodings are not considered. See // http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html. -func acceptedEncodings(r *http.Request) []string { +func AcceptedEncodings(r *http.Request, preferredOrder []string) []string { acceptEncHeader := r.Header.Get("Accept-Encoding") websocketKey := r.Header.Get("Sec-WebSocket-Key") if acceptEncHeader == "" { @@ -292,18 +342,29 @@ func acceptedEncodings(r *http.Request) []string { continue } + // set server preference + prefOrder := -1 + for i, p := range preferredOrder { + if encName == p { + prefOrder = len(preferredOrder) - i + break + } + } + prefs = append(prefs, encodingPreference{ - encoding: encName, - q: qFactor, + encoding: encName, + q: qFactor, + preferOrder: prefOrder, }) } - // sort preferences by descending q-factor - sort.Slice(prefs, func(i, j int) bool { return prefs[i].q > prefs[j].q }) - - // TODO: If no preference, or same pref for all encodings, - // and not websocket, use default encoding ordering (enc.Prefer) - // for those which are accepted by the client + // sort preferences by descending q-factor first, then by preferOrder + sort.Slice(prefs, func(i, j int) bool { + if math.Abs(prefs[i].q-prefs[j].q) < 0.00001 { + return prefs[i].preferOrder > prefs[j].preferOrder + } + return prefs[i].q > prefs[j].q + }) prefEncNames := make([]string, len(prefs)) for i := range prefs { @@ -315,8 +376,9 @@ func acceptedEncodings(r *http.Request) []string { // encodingPreference pairs an encoding with its q-factor. type encodingPreference struct { - encoding string - q float64 + encoding string + q float64 + preferOrder int } // Encoder is a type which can encode a stream of data. @@ -332,6 +394,13 @@ type Encoding interface { NewEncoder() Encoder } +// Precompressed is a type which returns filename suffix of precompressed +// file and Accept-Encoding header to use when serving this file. +type Precompressed interface { + AcceptEncoding() string + Suffix() string +} + var bufPool = sync.Pool{ New: func() interface{} { return new(bytes.Buffer) @@ -344,6 +413,7 @@ const defaultMinLength = 512 // Interface guards var ( _ caddy.Provisioner = (*Encode)(nil) + _ caddy.Validator = (*Encode)(nil) _ caddyhttp.MiddlewareHandler = (*Encode)(nil) _ caddyhttp.HTTPInterfaces = (*responseWriter)(nil) ) diff --git a/modules/caddyhttp/encode/encode_test.go b/modules/caddyhttp/encode/encode_test.go index 79eae3c560e..5f1e3f2a307 100644 --- a/modules/caddyhttp/encode/encode_test.go +++ b/modules/caddyhttp/encode/encode_test.go @@ -1,6 +1,8 @@ package encode import ( + "net/http" + "sync" "testing" ) @@ -10,3 +12,252 @@ func BenchmarkOpenResponseWriter(b *testing.B) { enc.openResponseWriter("test", nil) } } + +func TestPreferOrder(t *testing.T) { + testCases := []struct { + name string + accept string + prefer []string + expected []string + }{ + { + name: "PreferOrder(): 4 accept, 3 prefer", + accept: "deflate, gzip, br, zstd", + prefer: []string{"zstd", "br", "gzip"}, + expected: []string{"zstd", "br", "gzip", "deflate"}, + }, + { + name: "PreferOrder(): 2 accept, 3 prefer", + accept: "deflate, zstd", + prefer: []string{"zstd", "br", "gzip"}, + expected: []string{"zstd", "deflate"}, + }, + { + name: "PreferOrder(): 2 accept (1 empty), 3 prefer", + accept: "gzip,,zstd", + prefer: []string{"zstd", "br", "gzip"}, + expected: []string{"zstd", "gzip", ""}, + }, + { + name: "PreferOrder(): 1 accept, 2 prefer", + accept: "gzip", + prefer: []string{"zstd", "gzip"}, + expected: []string{"gzip"}, + }, + { + name: "PreferOrder(): 4 accept (1 duplicate), 1 prefer", + accept: "deflate, gzip, br, br", + prefer: []string{"br"}, + expected: []string{"br", "br", "deflate", "gzip"}, + }, + { + name: "PreferOrder(): empty accept, 0 prefer", + accept: "", + prefer: []string{}, + expected: []string{}, + }, + { + name: "PreferOrder(): empty accept, 1 prefer", + accept: "", + prefer: []string{"gzip"}, + expected: []string{}, + }, + { + name: "PreferOrder(): with q-factor", + accept: "deflate;q=0.8, gzip;q=0.4, br;q=0.2, zstd", + prefer: []string{"gzip"}, + expected: []string{"zstd", "deflate", "gzip", "br"}, + }, + { + name: "PreferOrder(): with q-factor, no prefer", + accept: "deflate;q=0.8, gzip;q=0.4, br;q=0.2, zstd", + prefer: []string{}, + expected: []string{"zstd", "deflate", "gzip", "br"}, + }, + { + name: "PreferOrder(): q-factor=0 filtered out", + accept: "deflate;q=0.1, gzip;q=0.4, br;q=0.5, zstd;q=0", + prefer: []string{"gzip"}, + expected: []string{"br", "gzip", "deflate"}, + }, + { + name: "PreferOrder(): q-factor=0 filtered out, no prefer", + accept: "deflate;q=0.1, gzip;q=0.4, br;q=0.5, zstd;q=0", + prefer: []string{}, + expected: []string{"br", "gzip", "deflate"}, + }, + { + name: "PreferOrder(): with invalid q-factor", + accept: "br, deflate, gzip;q=2, zstd;q=0.1", + prefer: []string{"zstd", "gzip"}, + expected: []string{"gzip", "br", "deflate", "zstd"}, + }, + { + name: "PreferOrder(): with invalid q-factor, no prefer", + accept: "br, deflate, gzip;q=2, zstd;q=0.1", + prefer: []string{}, + expected: []string{"br", "deflate", "gzip", "zstd"}, + }, + } + + enc := new(Encode) + r, _ := http.NewRequest("", "", nil) + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + + if test.accept == "" { + r.Header.Del("Accept-Encoding") + } else { + r.Header.Set("Accept-Encoding", test.accept) + } + enc.Prefer = test.prefer + result := AcceptedEncodings(r, enc.Prefer) + if !sliceEqual(result, test.expected) { + t.Errorf("AcceptedEncodings() actual: %s expected: %s", + result, + test.expected) + } + }) + } +} + +func sliceEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestValidate(t *testing.T) { + type testCase struct { + name string + prefer []string + wantErr bool + } + + var err error + var testCases []testCase + enc := new(Encode) + + enc.writerPools = map[string]*sync.Pool{ + "zstd": nil, + "gzip": nil, + "br": nil, + } + testCases = []testCase{ + { + name: "ValidatePrefer (zstd, gzip & br enabled): valid order with all encoder", + prefer: []string{"zstd", "br", "gzip"}, + wantErr: false, + }, + { + name: "ValidatePrefer (zstd, gzip & br enabled): valid order with 2 out of 3 encoders", + prefer: []string{"br", "gzip"}, + wantErr: false, + }, + { + name: "ValidatePrefer (zstd, gzip & br enabled): valid order with 1 out of 3 encoders", + prefer: []string{"gzip"}, + wantErr: false, + }, + { + name: "ValidatePrefer (zstd, gzip & br enabled): 1 duplicated (once) encoder", + prefer: []string{"gzip", "zstd", "gzip"}, + wantErr: true, + }, + { + name: "ValidatePrefer (zstd, gzip & br enabled): 1 not enabled encoder in prefer list", + prefer: []string{"br", "zstd", "gzip", "deflate"}, + wantErr: true, + }, + { + name: "ValidatePrefer (zstd, gzip & br enabled): no prefer list", + prefer: []string{}, + wantErr: false, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + enc.Prefer = test.prefer + err = enc.Validate() + if (err != nil) != test.wantErr { + t.Errorf("Validate() error = %v, wantErr = %v", err, test.wantErr) + } + }) + } + + enc.writerPools = map[string]*sync.Pool{ + "zstd": nil, + "gzip": nil, + } + testCases = []testCase{ + { + name: "ValidatePrefer (zstd & gzip enabled): 1 not enabled encoder in prefer list", + prefer: []string{"zstd", "br", "gzip"}, + wantErr: true, + }, + { + name: "ValidatePrefer (zstd & gzip enabled): 2 not enabled encoder in prefer list", + prefer: []string{"br", "zstd", "gzip", "deflate"}, + wantErr: true, + }, + { + name: "ValidatePrefer (zstd & gzip enabled): only not enabled encoder in prefer list", + prefer: []string{"deflate", "br", "gzip"}, + wantErr: true, + }, + { + name: "ValidatePrefer (zstd & gzip enabled): 1 duplicated (once) encoder in prefer list", + prefer: []string{"gzip", "zstd", "gzip"}, + wantErr: true, + }, + { + name: "ValidatePrefer (zstd & gzip enabled): 1 duplicated (twice) encoder in prefer list", + prefer: []string{"gzip", "zstd", "gzip", "gzip"}, + wantErr: true, + }, + { + name: "ValidatePrefer (zstd & gzip enabled): 1 duplicated encoder in prefer list", + prefer: []string{"zstd", "zstd", "gzip", "gzip"}, + wantErr: true, + }, + { + name: "ValidatePrefer (zstd & gzip enabled): 1 duplicated not enabled encoder in prefer list", + prefer: []string{"br", "br", "gzip"}, + wantErr: true, + }, + { + name: "ValidatePrefer (zstd & gzip enabled): 2 duplicated not enabled encoder in prefer list", + prefer: []string{"br", "deflate", "br", "deflate"}, + wantErr: true, + }, + { + name: "ValidatePrefer (zstd & gzip enabled): valid order zstd first", + prefer: []string{"zstd", "gzip"}, + wantErr: false, + }, + { + name: "ValidatePrefer (zstd & gzip enabled): valid order gzip first", + prefer: []string{"gzip", "zstd"}, + wantErr: false, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + enc.Prefer = test.prefer + err = enc.Validate() + if (err != nil) != test.wantErr { + t.Errorf("Validate() error = %v, wantErr = %v", err, test.wantErr) + } + }) + + } +} diff --git a/modules/caddyhttp/encode/gzip/gzip_precompressed.go b/modules/caddyhttp/encode/gzip/gzip_precompressed.go new file mode 100644 index 00000000000..7103cc8d913 --- /dev/null +++ b/modules/caddyhttp/encode/gzip/gzip_precompressed.go @@ -0,0 +1,28 @@ +package caddygzip + +import ( + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode" +) + +func init() { + caddy.RegisterModule(GzipPrecompressed{}) +} + +// GzipPrecompressed provides the file extension for files precompressed with gzip encoding. +type GzipPrecompressed struct { + Gzip +} + +// CaddyModule returns the Caddy module information. +func (GzipPrecompressed) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "http.precompressed.gzip", + New: func() caddy.Module { return new(GzipPrecompressed) }, + } +} + +// Suffix returns the filename suffix of precompressed files. +func (GzipPrecompressed) Suffix() string { return ".gz" } + +var _ encode.Precompressed = (*GzipPrecompressed)(nil) diff --git a/modules/caddyhttp/encode/zstd/zstd_precompressed.go b/modules/caddyhttp/encode/zstd/zstd_precompressed.go new file mode 100644 index 00000000000..522f41735ab --- /dev/null +++ b/modules/caddyhttp/encode/zstd/zstd_precompressed.go @@ -0,0 +1,28 @@ +package caddyzstd + +import ( + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode" +) + +func init() { + caddy.RegisterModule(ZstdPrecompressed{}) +} + +// ZstdPrecompressed provides the file extension for files precompressed with zstandard encoding. +type ZstdPrecompressed struct { + Zstd +} + +// CaddyModule returns the Caddy module information. +func (ZstdPrecompressed) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "http.precompressed.zstd", + New: func() caddy.Module { return new(ZstdPrecompressed) }, + } +} + +// Suffix returns the filename suffix of precompressed files. +func (ZstdPrecompressed) Suffix() string { return ".zst" } + +var _ encode.Precompressed = (*ZstdPrecompressed)(nil) diff --git a/modules/caddyhttp/fileserver/caddyfile.go b/modules/caddyhttp/fileserver/caddyfile.go index 3acbfa94b41..2ba53f2b1c3 100644 --- a/modules/caddyhttp/fileserver/caddyfile.go +++ b/modules/caddyhttp/fileserver/caddyfile.go @@ -19,8 +19,10 @@ import ( "strings" "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig" "github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile" "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode" "github.com/caddyserver/caddy/v2/modules/caddyhttp/rewrite" ) @@ -33,10 +35,11 @@ func init() { // server and configures it with this syntax: // // file_server [] [browse] { -// root -// hide -// index -// browse [] +// root +// hide +// index +// browse [] +// precompressed // } // func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) { @@ -77,6 +80,26 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) } fsrv.Browse = new(Browse) h.Args(&fsrv.Browse.TemplateFile) + case "precompressed": + var order []string + for h.NextArg() { + modID := "http.precompressed." + h.Val() + mod, err := caddy.GetModule(modID) + if err != nil { + return nil, h.Errf("getting module named '%s': %v", modID, err) + } + inst := mod.New() + precompress, ok := inst.(encode.Precompressed) + if !ok { + return nil, h.Errf("module %s is not a precompressor; is %T", modID, inst) + } + if fsrv.PrecompressedRaw == nil { + fsrv.PrecompressedRaw = make(caddy.ModuleMap) + } + fsrv.PrecompressedRaw[h.Val()] = caddyconfig.JSON(precompress, nil) + order = append(order, h.Val()) + } + fsrv.PrecompressedOrder = order default: return nil, h.Errf("unknown subdirective '%s'", h.Val()) } diff --git a/modules/caddyhttp/fileserver/staticfiles.go b/modules/caddyhttp/fileserver/staticfiles.go index f58dfe0257e..c6707885786 100644 --- a/modules/caddyhttp/fileserver/staticfiles.go +++ b/modules/caddyhttp/fileserver/staticfiles.go @@ -31,6 +31,7 @@ import ( "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode" "go.uber.org/zap" ) @@ -79,6 +80,16 @@ type FileServer struct { // a 404 error. By default, this is false (disabled). PassThru bool `json:"pass_thru,omitempty"` + // Selection of encoders to use to check for precompressed files. + PrecompressedRaw caddy.ModuleMap `json:"precompressed,omitempty" caddy:"namespace=http.precompressed"` + + // If the client has no strong preference (q-factor), choose these encodings in order. + // If no order specified here, the first encoding from the Accept-Encoding header + // that both client and server support is used + PrecompressedOrder []string `json:"precompressed_order,omitempty"` + + precompressors map[string]encode.Precompressed + logger *zap.Logger } @@ -129,6 +140,32 @@ func (fsrv *FileServer) Provision(ctx caddy.Context) error { } } + mods, err := ctx.LoadModule(fsrv, "PrecompressedRaw") + if err != nil { + return fmt.Errorf("loading encoder modules: %v", err) + } + for modName, modIface := range mods.(map[string]interface{}) { + p, ok := modIface.(encode.Precompressed) + if !ok { + return fmt.Errorf("module %s is not precompressor", modName) + } + ae := p.AcceptEncoding() + if ae == "" { + return fmt.Errorf("precompressor does not specify an Accept-Encoding value") + } + suffix := p.Suffix() + if suffix == "" { + return fmt.Errorf("precompressor does not specify a Suffix value") + } + if _, ok := fsrv.precompressors[ae]; ok { + return fmt.Errorf("precompressor already added: %s", ae) + } + if fsrv.precompressors == nil { + fsrv.precompressors = make(map[string]encode.Precompressed) + } + fsrv.precompressors[ae] = p + } + return nil } @@ -205,8 +242,6 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c return fsrv.notFound(w, r, next) } - // TODO: content negotiation (brotli sidecar files, etc...) - // one last check to ensure the file isn't hidden (we might // have changed the filename from when we last checked) if fileHidden(filename, filesToHide) { @@ -230,18 +265,51 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c } } - fsrv.logger.Debug("opening file", zap.String("filename", filename)) + var file *os.File - // open the file - file, err := fsrv.openFile(filename, w) - if err != nil { - if herr, ok := err.(caddyhttp.HandlerError); ok && - herr.StatusCode == http.StatusNotFound { - return fsrv.notFound(w, r, next) + // check for precompressed files + for _, ae := range encode.AcceptedEncodings(r, fsrv.PrecompressedOrder) { + precompress, ok := fsrv.precompressors[ae] + if !ok { + continue + } + compressedFilename := filename + precompress.Suffix() + compressedInfo, err := os.Stat(compressedFilename) + if err != nil || compressedInfo.IsDir() { + fsrv.logger.Debug("precompressed file not accessible", zap.String("filename", compressedFilename), zap.Error(err)) + continue + } + fsrv.logger.Debug("opening compressed sidecar file", zap.String("filename", compressedFilename), zap.Error(err)) + file, err = fsrv.openFile(compressedFilename, w) + if err != nil { + fsrv.logger.Warn("opening precompressed file failed", zap.String("filename", compressedFilename), zap.Error(err)) + if caddyErr, ok := err.(caddyhttp.HandlerError); ok && caddyErr.StatusCode == http.StatusServiceUnavailable { + return err + } + continue + } + defer file.Close() + w.Header().Set("Content-Encoding", ae) + w.Header().Del("Accept-Ranges") + w.Header().Add("Vary", "Accept-Encoding") + break + } + + // no precompressed file found, use the actual file + if file == nil { + fsrv.logger.Debug("opening file", zap.String("filename", filename)) + + // open the file + file, err = fsrv.openFile(filename, w) + if err != nil { + if herr, ok := err.(caddyhttp.HandlerError); ok && + herr.StatusCode == http.StatusNotFound { + return fsrv.notFound(w, r, next) + } + return err // error is already structured } - return err // error is already structured + defer file.Close() } - defer file.Close() // set the ETag - note that a conditional If-None-Match request is handled // by http.ServeContent below, which checks against this ETag value diff --git a/modules/caddyhttp/standard/imports.go b/modules/caddyhttp/standard/imports.go index 0aeef842eba..0e2203c1ebd 100644 --- a/modules/caddyhttp/standard/imports.go +++ b/modules/caddyhttp/standard/imports.go @@ -5,6 +5,7 @@ import ( _ "github.com/caddyserver/caddy/v2/modules/caddyhttp" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyauth" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode" + _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/brotli" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/gzip" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/zstd" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/fileserver"