encode,staticfiles: Content negotiation, precompressed files (#4045)

* encode: implement prefer setting

* encode: minimum_length configurable via caddyfile

* encode: configurable content-types which to encode

* file_server: support precompressed files

* encode: use ReponseMatcher for conditional encoding of content

* linting error & documentation of encode.PrecompressedOrder

* encode: allow just one response matcher

also change the namespace of the encoders back, I accidently changed to precompressed >.>
default matchers include a *  to match to any charset, that may be appended

* rounding of the PR

* added integration tests for new caddyfile directives
* improved various doc strings (punctuation and typos)
* added json tag for file_server precompress order and encode matcher

* file_server: add vary header, remove accept-ranges when serving precompressed files

* encode: move Suffix implementation to precompressed modules

caddytest/integration/caddyfile_adapt/encode_options.txt

@@ -0,0 +1,66 @@
+:80
+
+encode gzip zstd {
+	minimum_length 256
+	prefer zstd gzip
+	match {
+		status 2xx 4xx 500
+		header Content-Type text/*
+		header Content-Type application/json*
+		header Content-Type application/javascript*
+		header Content-Type application/xhtml+xml*
+		header Content-Type application/atom+xml*
+		header Content-Type application/rss+xml*
+		header Content-Type image/svg+xml*
+	}
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"encodings": {
+										"gzip": {},
+										"zstd": {}
+									},
+									"handler": "encode",
+									"match": {
+										"headers": {
+											"Content-Type": [
+												"text/*",
+												"application/json*",
+												"application/javascript*",
+												"application/xhtml+xml*",
+												"application/atom+xml*",
+												"application/rss+xml*",
+												"image/svg+xml*"
+											]
+										},
+										"status_code": [
+											2,
+											4,
+											500
+										]
+									},
+									"minimum_length": 256,
+									"prefer": [
+										"zstd",
+										"gzip"
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt/file_server_precompressed.txt

@@ -0,0 +1,41 @@
+:80
+
+file_server {
+	precompressed zstd br gzip
+}
+----------
+{
+	"apps": {
+		"http": {
+			"servers": {
+				"srv0": {
+					"listen": [
+						":80"
+					],
+					"routes": [
+						{
+							"handle": [
+								{
+									"handler": "file_server",
+									"hide": [
+										"./Caddyfile"
+									],
+									"precompressed": {
+										"br": {},
+										"gzip": {},
+										"zstd": {}
+									},
+									"precompressed_order": [
+										"zstd",
+										"br",
+										"gzip"
+									]
+								}
+							]
+						}
+					]
+				}
+			}
+		}
+	}
+}

caddytest/integration/caddyfile_adapt_test.go

@@ -1,7 +1,10 @@
 package integration
 
 import (
+	jsonMod "encoding/json"
+	"fmt"
 	"io/ioutil"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"testing"
@@ -39,6 +42,10 @@ func TestCaddyfileAdaptToJSON(t *testing.T) {
 		// replace windows newlines in the json with unix newlines
 		json = winNewlines.ReplaceAllString(json, "\n")
 
+		// replace os-specific default path for file_server's hide field
+		replacePath, _ := jsonMod.Marshal(fmt.Sprint(".", string(filepath.Separator), "Caddyfile"))
+		json = strings.ReplaceAll(json, `"./Caddyfile"`, string(replacePath))
+
 		// run the test
 		ok := caddytest.CompareAdapt(t, filename, caddyfile, "caddyfile", json)
 		if !ok {

modules/caddyhttp/encode/brotli/brotli_precompressed.go

@@ -0,0 +1,31 @@
+package caddybrotli
+
+import (
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
+)
+
+func init() {
+	caddy.RegisterModule(BrotliPrecompressed{})
+}
+
+// BrotliPrecompressed provides the file extension for files precompressed with brotli encoding.
+type BrotliPrecompressed struct{}
+
+// CaddyModule returns the Caddy module information.
+func (BrotliPrecompressed) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "http.precompressed.br",
+		New: func() caddy.Module { return new(BrotliPrecompressed) },
+	}
+}
+
+// AcceptEncoding returns the name of the encoding as
+// used in the Accept-Encoding request headers.
+func (BrotliPrecompressed) AcceptEncoding() string { return "br" }
+
+// Suffix returns the filename suffix of precompressed files.
+func (BrotliPrecompressed) Suffix() string { return ".br" }
+
+// Interface guards
+var _ encode.Precompressed = (*BrotliPrecompressed)(nil)

modules/caddyhttp/encode/caddyfile.go

@@ -15,7 +15,9 @@
 package encode
 
 import (
-	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/caddyconfig"
@@ -40,21 +42,31 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 // UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax:
 //
 //     encode [<matcher>] <formats...> {
-//         gzip [<level>]
+//         gzip           [<level>]
 //         zstd
+//         minimum_length <length>
+//         prefer         <formats...>
+//         # response matcher block
+//         match {
+//             status <code...>
+//             header <field> [<value>]
+//         }
+//         # or response matcher single line syntax
+//         match [header <field> [<value>]] | [status <code...>]
 //     }
 //
 // Specifying the formats on the first line will use those formats' defaults.
 func (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
+	responseMatchers := make(map[string]caddyhttp.ResponseMatcher)
 	for d.Next() {
 		for _, arg := range d.RemainingArgs() {
 			mod, err := caddy.GetModule("http.encoders." + arg)
 			if err != nil {
-				return fmt.Errorf("finding encoder module '%s': %v", mod, err)
+				return d.Errf("finding encoder module '%s': %v", mod, err)
 			}
 			encoding, ok := mod.New().(Encoding)
 			if !ok {
-				return fmt.Errorf("module %s is not an HTTP encoding", mod)
+				return d.Errf("module %s is not an HTTP encoding", mod)
 			}
 			if enc.EncodingsRaw == nil {
 				enc.EncodingsRaw = make(caddy.ModuleMap)
@@ -63,25 +75,118 @@ func (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 		}
 
 		for d.NextBlock(0) {
-			name := d.Val()
-			modID := "http.encoders." + name
-			unm, err := caddyfile.UnmarshalModule(d, modID)
-			if err != nil {
-				return err
-			}
-			encoding, ok := unm.(Encoding)
-			if !ok {
-				return fmt.Errorf("module %s is not an HTTP encoding; is %T", modID, unm)
+			switch d.Val() {
+			case "minimum_length":
+				if !d.NextArg() {
+					return d.ArgErr()
+				}
+				minLength, err := strconv.Atoi(d.Val())
+				if err != nil {
+					return err
+				}
+				enc.MinLength = minLength
+			case "prefer":
+				var encs []string
+				for d.NextArg() {
+					encs = append(encs, d.Val())
+				}
+				if len(encs) == 0 {
+					return d.ArgErr()
+				}
+				enc.Prefer = encs
+			case "match":
+				err := enc.parseNamedResponseMatcher(d.NewFromNextSegment(), responseMatchers)
+				if err != nil {
+					return err
+				}
+				matcher := responseMatchers["match"]
+				enc.Matcher = &matcher
+			default:
+				name := d.Val()
+				modID := "http.encoders." + name
+				unm, err := caddyfile.UnmarshalModule(d, modID)
+				if err != nil {
+					return err
+				}
+				encoding, ok := unm.(Encoding)
+				if !ok {
+					return d.Errf("module %s is not an HTTP encoding; is %T", modID, unm)
+				}
+				if enc.EncodingsRaw == nil {
+					enc.EncodingsRaw = make(caddy.ModuleMap)
+				}
+				enc.EncodingsRaw[name] = caddyconfig.JSON(encoding, nil)
 			}
-			if enc.EncodingsRaw == nil {
-				enc.EncodingsRaw = make(caddy.ModuleMap)
-			}
-			enc.EncodingsRaw[name] = caddyconfig.JSON(encoding, nil)
 		}
 	}
 
 	return nil
 }
 
+// Parse the tokens of a named response matcher.
+//
+//     match {
+//         header <field> [<value>]
+//         status <code...>
+//     }
+//
+// Or, single line syntax:
+//
+//     match [header <field> [<value>]] | [status <code...>]
+//
+func (enc *Encode) parseNamedResponseMatcher(d *caddyfile.Dispenser, matchers map[string]caddyhttp.ResponseMatcher) error {
+	for d.Next() {
+		definitionName := d.Val()
+
+		if _, ok := matchers[definitionName]; ok {
+			return d.Errf("matcher is defined more than once: %s", definitionName)
+		}
+
+		matcher := caddyhttp.ResponseMatcher{}
+		for nesting := d.Nesting(); d.NextArg() || d.NextBlock(nesting); {
+			switch d.Val() {
+			case "header":
+				if matcher.Headers == nil {
+					matcher.Headers = http.Header{}
+				}
+
+				// reuse the header request matcher's unmarshaler
+				headerMatcher := caddyhttp.MatchHeader(matcher.Headers)
+				err := headerMatcher.UnmarshalCaddyfile(d.NewFromNextSegment())
+				if err != nil {
+					return err
+				}
+
+				matcher.Headers = http.Header(headerMatcher)
+			case "status":
+				if matcher.StatusCode == nil {
+					matcher.StatusCode = []int{}
+				}
+
+				args := d.RemainingArgs()
+				if len(args) == 0 {
+					return d.ArgErr()
+				}
+
+				for _, arg := range args {
+					if len(arg) == 3 && strings.HasSuffix(arg, "xx") {
+						arg = arg[:1]
+					}
+					statusNum, err := strconv.Atoi(arg)
+					if err != nil {
+						return d.Errf("bad status value '%s': %v", arg, err)
+					}
+					matcher.StatusCode = append(matcher.StatusCode, statusNum)
+				}
+			default:
+				return d.Errf("unrecognized response matcher %s", d.Val())
+			}
+		}
+
+		matchers[definitionName] = matcher
+	}
+	return nil
+}
+
 // Interface guard
 var _ caddyfile.Unmarshaler = (*Encode)(nil)