From 59a1d6d833e2b6add73d2e7febe624d6f4e5a905 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Thu, 8 Dec 2022 13:45:26 +0100 Subject: [PATCH] Support multiple headers Signed-off-by: Julien Pivotto --- config/headers.go | 103 +++++++++++------- config/headers_test.go | 6 +- config/http_config_test.go | 2 +- .../http.conf.headers-duplicate-1.bad.yaml | 4 +- .../http.conf.headers-duplicate-2.bad.yaml | 4 +- .../http.conf.headers-duplicate-3.bad.yaml | 4 +- .../http.conf.headers-duplicate-4.bad.yaml | 6 +- .../http.conf.headers-reserved-1.bad.yaml | 2 +- .../http.conf.headers-reserved-2.bad.yaml | 2 +- .../http.conf.headers-reserved-3.bad.yaml | 2 +- config/testdata/http.conf.headers.good.yaml | 6 +- 11 files changed, 80 insertions(+), 61 deletions(-) diff --git a/config/headers.go b/config/headers.go index 68fd0b23..188f1cb3 100644 --- a/config/headers.go +++ b/config/headers.go @@ -19,7 +19,6 @@ package config import ( "fmt" "net/http" - "net/textproto" "os" "strings" ) @@ -27,33 +26,33 @@ import ( // reservedHeaders that change the connection, are set by Prometheus, or car be set // otherwise can't be changed. var reservedHeaders = map[string]struct{}{ - "authorization": {}, - "host": {}, - "content-encoding": {}, - "content-length": {}, - "content-type": {}, - "user-agent": {}, - "connection": {}, - "keep-alive": {}, - "proxy-authenticate": {}, - "proxy-authorization": {}, - "www-authenticate": {}, - "accept-encoding": {}, - "x-prometheus-remote-write-version": {}, - "x-prometheus-remote-read-version": {}, - "x-prometheus-scrape-timeout-seconds": {}, + "Authorization": {}, + "Host": {}, + "Content-Encoding": {}, + "Content-Length": {}, + "Content-Type": {}, + "User-Agent": {}, + "Connection": {}, + "Keep-Alive": {}, + "Proxy-Authenticate": {}, + "Proxy-Authorization": {}, + "Www-Authenticate": {}, + "Accept-Encoding": {}, + "X-Prometheus-Remote-Write-Version": {}, + "X-Prometheus-Remote-Read-Version": {}, + "X-Prometheus-Scrape-Timeout-Seconds": {}, // Added by SigV4. - "x-amz-date": {}, - "x-amz-security-token": {}, - "x-amz-content-sha256": {}, + "X-Amz-Date": {}, + "X-Amz-Security-Token": {}, + "X-Amz-Content-Sha256": {}, } // Headers represents the configuration for HTTP headers. type Headers struct { - Headers map[string]string `yaml:"headers,omitempty" json:"headers,omitempty"` - SecretHeaders map[string]Secret `yaml:"secret_headers,omitempty" json:"secret_headers,omitempty"` - Files map[string]string `yaml:"files,omitempty" json:"files,omitempty"` + Headers map[string][]string `yaml:"headers,omitempty" json:"headers,omitempty"` + SecretHeaders map[string][]Secret `yaml:"secret_headers,omitempty" json:"secret_headers,omitempty"` + Files map[string][]string `yaml:"files,omitempty" json:"files,omitempty"` dir string } @@ -67,28 +66,30 @@ func (h *Headers) SetDirectory(dir string) { func (h *Headers) Validate() error { uniqueHeaders := make(map[string]struct{}, len(h.Headers)) for k := range h.Headers { - uniqueHeaders[strings.ToLower(k)] = struct{}{} + uniqueHeaders[http.CanonicalHeaderKey(k)] = struct{}{} } for k := range h.SecretHeaders { - if _, ok := uniqueHeaders[strings.ToLower(k)]; ok { - return fmt.Errorf("header %q is defined in multiple sections", textproto.CanonicalMIMEHeaderKey(k)) + if _, ok := uniqueHeaders[http.CanonicalHeaderKey(k)]; ok { + return fmt.Errorf("header %q is defined in multiple sections", http.CanonicalHeaderKey(k)) } - uniqueHeaders[strings.ToLower(k)] = struct{}{} + uniqueHeaders[http.CanonicalHeaderKey(k)] = struct{}{} } for k, v := range h.Files { - if _, ok := uniqueHeaders[strings.ToLower(k)]; ok { - return fmt.Errorf("header %q is defined in multiple sections", textproto.CanonicalMIMEHeaderKey(k)) + if _, ok := uniqueHeaders[http.CanonicalHeaderKey(k)]; ok { + return fmt.Errorf("header %q is defined in multiple sections", http.CanonicalHeaderKey(k)) } - uniqueHeaders[strings.ToLower(k)] = struct{}{} - f := JoinDir(h.dir, v) - _, err := os.ReadFile(f) - if err != nil { - return fmt.Errorf("unable to read header %q from file %s: %w", textproto.CanonicalMIMEHeaderKey(k), f, err) + uniqueHeaders[http.CanonicalHeaderKey(k)] = struct{}{} + for _, file := range v { + f := JoinDir(h.dir, file) + _, err := os.ReadFile(f) + if err != nil { + return fmt.Errorf("unable to read header %q from file %s: %w", http.CanonicalHeaderKey(k), f, err) + } } } for k := range uniqueHeaders { - if _, ok := reservedHeaders[strings.ToLower(k)]; ok { - return fmt.Errorf("setting header %q is not allowed", textproto.CanonicalMIMEHeaderKey(k)) + if _, ok := reservedHeaders[http.CanonicalHeaderKey(k)]; ok { + return fmt.Errorf("setting header %q is not allowed", http.CanonicalHeaderKey(k)) } } return nil @@ -115,18 +116,36 @@ type headersRoundTripper struct { func (rt *headersRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { req = cloneRequest(req) for k, v := range rt.config.Headers { - req.Header.Set(textproto.CanonicalMIMEHeaderKey(k), v) + for i, h := range v { + if i == 0 { + req.Header.Set(k, h) + continue + } + req.Header.Add(k, h) + } } for k, v := range rt.config.SecretHeaders { - req.Header.Set(textproto.CanonicalMIMEHeaderKey(k), string(v)) + for i, h := range v { + if i == 0 { + req.Header.Set(k, string(h)) + continue + } + req.Header.Add(k, string(h)) + } } for k, v := range rt.config.Files { - f := JoinDir(rt.config.dir, v) - b, err := os.ReadFile(f) - if err != nil { - return nil, fmt.Errorf("unable to read headers file %s: %w", f, err) + for i, h := range v { + f := JoinDir(rt.config.dir, h) + b, err := os.ReadFile(f) + if err != nil { + return nil, fmt.Errorf("unable to read headers file %s: %w", f, err) + } + if i == 0 { + req.Header.Set(k, strings.TrimSpace(string(b))) + continue + } + req.Header.Add(k, strings.TrimSpace(string(b))) } - req.Header.Set(textproto.CanonicalMIMEHeaderKey(k), strings.TrimSpace(string(b))) } return rt.next.RoundTrip(req) } diff --git a/config/headers_test.go b/config/headers_test.go index 8b0c7640..ef0403fa 100644 --- a/config/headers_test.go +++ b/config/headers_test.go @@ -17,15 +17,15 @@ package config import ( - "strings" + "net/http" "testing" ) func TestReservedHeaders(t *testing.T) { for k := range reservedHeaders { - l := strings.ToLower(k) + l := http.CanonicalHeaderKey(k) if k != l { - t.Errorf("reservedHeaders keys should be lowercase: got %q, expected %q", k, strings.ToLower(k)) + t.Errorf("reservedHeaders keys should be lowercase: got %q, expected %q", k, http.CanonicalHeaderKey(k)) } } } diff --git a/config/http_config_test.go b/config/http_config_test.go index c00b2e03..5c2f15d2 100644 --- a/config/http_config_test.go +++ b/config/http_config_test.go @@ -1144,7 +1144,7 @@ func TestInvalidHTTPConfigs(t *testing.T) { for _, ee := range invalidHTTPClientConfigs { _, _, err := LoadHTTPConfigFile(ee.httpClientConfigFile) if err == nil { - t.Error("Expected error with config but got none") + t.Errorf("Expected error with config %q but got none", ee.httpClientConfigFile) continue } if !strings.Contains(err.Error(), ee.errMsg) { diff --git a/config/testdata/http.conf.headers-duplicate-1.bad.yaml b/config/testdata/http.conf.headers-duplicate-1.bad.yaml index 879c6cfc..c469391e 100644 --- a/config/testdata/http.conf.headers-duplicate-1.bad.yaml +++ b/config/testdata/http.conf.headers-duplicate-1.bad.yaml @@ -1,5 +1,5 @@ http_headers: headers: - Foo: bar + Foo: [bar] secret_headers: - foO: bar + foO: [bar] diff --git a/config/testdata/http.conf.headers-duplicate-2.bad.yaml b/config/testdata/http.conf.headers-duplicate-2.bad.yaml index c0fd8a59..5df57353 100644 --- a/config/testdata/http.conf.headers-duplicate-2.bad.yaml +++ b/config/testdata/http.conf.headers-duplicate-2.bad.yaml @@ -1,5 +1,5 @@ http_headers: files: - Foo: bar + Foo: [bar] secret_headers: - foO: bar + foO: [bar] diff --git a/config/testdata/http.conf.headers-duplicate-3.bad.yaml b/config/testdata/http.conf.headers-duplicate-3.bad.yaml index 391de30d..4a0e5874 100644 --- a/config/testdata/http.conf.headers-duplicate-3.bad.yaml +++ b/config/testdata/http.conf.headers-duplicate-3.bad.yaml @@ -1,5 +1,5 @@ http_headers: headers: - Foo: bar + Foo: [bar] files: - foO: bar + foO: [bar] diff --git a/config/testdata/http.conf.headers-duplicate-4.bad.yaml b/config/testdata/http.conf.headers-duplicate-4.bad.yaml index 571c7731..66771b5c 100644 --- a/config/testdata/http.conf.headers-duplicate-4.bad.yaml +++ b/config/testdata/http.conf.headers-duplicate-4.bad.yaml @@ -1,7 +1,7 @@ http_headers: headers: - Foo: bar + Foo: [bar] secret_headers: - foO: bar + foO: [bar] files: - foO: bar + foO: [bar] diff --git a/config/testdata/http.conf.headers-reserved-1.bad.yaml b/config/testdata/http.conf.headers-reserved-1.bad.yaml index 1bb30632..2dcf05d0 100644 --- a/config/testdata/http.conf.headers-reserved-1.bad.yaml +++ b/config/testdata/http.conf.headers-reserved-1.bad.yaml @@ -1,3 +1,3 @@ http_headers: headers: - user-Agent: bar + user-Agent: [bar] diff --git a/config/testdata/http.conf.headers-reserved-2.bad.yaml b/config/testdata/http.conf.headers-reserved-2.bad.yaml index 40ae4289..4347f90a 100644 --- a/config/testdata/http.conf.headers-reserved-2.bad.yaml +++ b/config/testdata/http.conf.headers-reserved-2.bad.yaml @@ -1,3 +1,3 @@ http_headers: secret_headers: - user-Agent: bar + user-Agent: [bar] diff --git a/config/testdata/http.conf.headers-reserved-3.bad.yaml b/config/testdata/http.conf.headers-reserved-3.bad.yaml index e7fb5878..57b404bb 100644 --- a/config/testdata/http.conf.headers-reserved-3.bad.yaml +++ b/config/testdata/http.conf.headers-reserved-3.bad.yaml @@ -1,3 +1,3 @@ http_headers: files: - user-Agent: testdata/headers-file + user-Agent: [testdata/headers-file] diff --git a/config/testdata/http.conf.headers.good.yaml b/config/testdata/http.conf.headers.good.yaml index 3a35d9ad..257c1e65 100644 --- a/config/testdata/http.conf.headers.good.yaml +++ b/config/testdata/http.conf.headers.good.yaml @@ -1,7 +1,7 @@ http_headers: headers: - one: value1 + one: [value1] secret_headers: - two: value2 + two: [value2] files: - three: testdata/headers-file + three: [testdata/headers-file]