e2e: end-to-end testing framework and status check test

The e2e tests are implemented using docker-compose, go test, makefile and bash.

There are 4 containers:
- Proxy
- Upstream Proxy
- Test backend (httpbin)
- Test runner

Container configuration is changed by adding environment variables from the override files (see override directory).
This enables to mix and match the files, this could be called a "mixin" model, to enable different features.

All but last container are managed by docker-compose.
On start we wait until all services ready using the readiness check in the api.
The last container is injected to the docker-compose network to run the compiled go test program.

Makefile implements helpers for working with docker and building the test binary.
All the concrete test invocations are done from the run.sh shell script.

The status check test is ran against all combinations of http/https proxy and backend, example output.

```
>>> DIRECT proxy=http httpbin=http
[+] Running 3/3
 ⠿ Container forwarder-e2e-upstream-proxy-1  Healthy                                                                                                                                                                                                        1.8s
 ⠿ Container forwarder-e2e-httpbin-1         Healthy                                                                                                                                                                                                        1.8s
 ⠿ Container forwarder-e2e-proxy-1           Healthy                                                                                                                                                                                                        2.3s
PASS
>>> DIRECT proxy=https httpbin=https
[+] Running 3/3
 ⠿ Container forwarder-e2e-upstream-proxy-1  Healthy                                                                                                                                                                                                        1.7s
 ⠿ Container forwarder-e2e-httpbin-1         Healthy                                                                                                                                                                                                        2.3s
 ⠿ Container forwarder-e2e-proxy-1           Healthy                                                                                                                                                                                                        2.3s
PASS
>>> DIRECT proxy=https httpbin=https
[+] Running 3/3
 ⠿ Container forwarder-e2e-upstream-proxy-1  Healthy                                                                                                                                                                                                        2.1s
 ⠿ Container forwarder-e2e-httpbin-1         Healthy                                                                                                                                                                                                        2.1s
 ⠿ Container forwarder-e2e-proxy-1           Healthy                                                                                                                                                                                                        2.1s
PASS
>>> DIRECT proxy=http httpbin=http
[+] Running 3/3
 ⠿ Container forwarder-e2e-upstream-proxy-1  Healthy                                                                                                                                                                                                        2.1s
 ⠿ Container forwarder-e2e-proxy-1           Healthy                                                                                                                                                                                                        2.1s
 ⠿ Container forwarder-e2e-httpbin-1         Healthy                                                                                                                                                                                                        2.1s
PASS
```

The test found a bug in 1XX response handling in proxy, reported in #113.

.golangci.yml

@@ -5,6 +5,8 @@
 ---
 run:
   timeout: 5m
+  build-tags:
+    - e2e
   skip-files:
     - httpbin/httpbin.go
     - middleware/delegator.go
@@ -73,6 +75,10 @@ issues:
         - paralleltest
         - testpackage
 
+    - path: e2e/args.go
+      linters:
+        - gochecknoglobals
+
     - path: internal/version/version.go
       linters:
         - gochecknoglobals

e2e/.env

@@ -0,0 +1 @@
+COMPOSE_PROJECT_NAME=forwarder-e2e

e2e/Makefile

@@ -0,0 +1,34 @@
+# Ignore CPU pinning in CI.
+ifdef CI
+CPUSET := ""
+else
+CPUSET := -f override/cpuset.yaml
+endif
+
+# Docker tag to be tested.
+ifndef FORWARDER_VERSION
+export FORWARDER_VERSION := devel
+endif
+
+.PHONY: up
+up:
+	@docker-compose -f docker-compose.yaml $(CPUSET) $(foreach f,$(CONF),-f $(f) ) \
+	up -d --wait --force-recreate --remove-orphans
+
+.PHONY: down
+down:
+	@docker-compose down -v --remove-orphans
+
+.PHONY: dump-logs
+dump-logs:
+	@docker-compose logs $(SRV)
+
+.PHONY: test
+test: RUN=.
+test: e2e.test
+	@docker run --name "test-runner" --network "forwarder-e2e_default" --cpuset-cpus 0 \
+	-v "$(PWD)/e2e.test:/usr/bin/e2e.test" -i --read-only --rm ubuntu \
+	e2e.test -test.run $(RUN) $(ARGS)
+
+e2e.test: *.go
+	@CGO_ENABLED=0 GOOS=linux go test -tags e2e -c -o e2e.test .

e2e/args.go

@@ -0,0 +1,9 @@
+package e2e
+
+import "flag"
+
+var (
+	proxy              = flag.String("proxy", "", "URL of the proxy to test against")
+	httpbin            = flag.String("httpbin", "", "URL of the httpbin server to test against")
+	insecureSkipVerify = flag.Bool("insecure-skip-verify", false, "Skip TLS certificate verification")
+)

e2e/client.go

@@ -0,0 +1,40 @@
+package e2e
+
+import (
+	"flag"
+	"net/http"
+	"net/url"
+	"testing"
+
+	"github.com/saucelabs/forwarder"
+)
+
+func newHTTPClient(t *testing.T) *http.Client {
+	t.Helper()
+
+	if !flag.Parsed() {
+		flag.Parse()
+	}
+
+	if *proxy == "" {
+		t.Fatal("proxy URL not set")
+	}
+
+	cfg := forwarder.DefaultHTTPTransportConfig()
+	cfg.InsecureSkipVerify = *insecureSkipVerify
+
+	rt := forwarder.NewHTTPTransport(cfg, nil)
+	if *proxy == "" {
+		t.Log("proxy not set, running without proxy")
+	} else {
+		proxyURL, err := url.Parse(*proxy)
+		if err != nil {
+			t.Fatal(err)
+		}
+		rt.Proxy = http.ProxyURL(proxyURL)
+	}
+
+	return &http.Client{
+		Transport: rt,
+	}
+}

e2e/docker-compose.yaml

@@ -0,0 +1,15 @@
+version: "3.8"
+
+services:
+  proxy:
+    image: saucelabs/forwarder:${FORWARDER_VERSION}
+    environment:
+      FORWARDER_VERBOSE: "true"
+    healthcheck:
+      interval: 1s
+      timeout: 250ms
+      retries: 10
+
+  upstream-proxy:
+    extends: proxy
+

e2e/httpbin-http.yaml

@@ -0,0 +1,10 @@
+version: "3.8"
+
+services:
+  httpbin:
+    extends:
+      file: docker-compose.yaml
+      service: proxy
+    command: httpbin --address :80
+    healthcheck:
+      test: ["CMD", "curl", "-s", "-S", "-f", "http://localhost/status/200"]

e2e/httpbin-https.yaml

@@ -0,0 +1,10 @@
+version: "3.8"
+
+services:
+  httpbin:
+    extends:
+      file: docker-compose.yaml
+      service: proxy
+    command: httpbin --protocol HTTPS --address :443
+    healthcheck:
+      test: ["CMD", "curl", "-s", "-S", "-f", "-k", "https://localhost/status/200"]

e2e/override/cpuset.yaml

@@ -0,0 +1,9 @@
+version: "3.8"
+
+services:
+  proxy:
+    cpuset: "1"
+  upstream-proxy:
+    cpuset: "2"
+  httpbin:
+    cpuset: "3"

e2e/override/proxy-http.yaml

@@ -0,0 +1,6 @@
+version: "3.8"
+
+services:
+  proxy:
+    environment:
+      FORWARDER_PROTOCOL: "HTTP"

e2e/override/proxy-https.yaml

@@ -0,0 +1,6 @@
+version: "3.8"
+
+services:
+  proxy:
+    environment:
+      FORWARDER_PROTOCOL: "HTTPS"

e2e/run.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+on_error() {
+  make dump-logs
+}
+
+run_direct_test() {
+  trap 'on_error' ERR
+
+  PROXY_SCHEME=$1
+  HTTPBIN_SCHEME=$1
+
+  echo ">>> DIRECT proxy=${PROXY_SCHEME} httpbin=${HTTPBIN_SCHEME}"
+  make up CONF="override/proxy-${PROXY_SCHEME}.yaml httpbin-${HTTPBIN_SCHEME}.yaml"
+  make test ARGS="-proxy ${PROXY_SCHEME}://proxy:3128 -httpbin ${HTTPBIN_SCHEME}://httpbin -insecure-skip-verify"
+}
+
+run_direct_test http http
+run_direct_test https http
+run_direct_test https https
+run_direct_test http https

e2e/status_test.go

@@ -0,0 +1,51 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"testing"
+)
+
+func TestStatusCodes(t *testing.T) {
+	if *httpbin == "" {
+		t.Fatal("httpbin URL not set")
+	}
+
+	// List of all valid status codes plus some non-standard ones.
+	// See https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml
+	validStatusCodes := []int{
+		// FIXME: proxy wrongly supports 1xx, see #113
+		// 100, 101, 102, 103, 122,
+		200, 201, 202, 203, 204, 205, 206, 207, 208, 226,
+		300, 301, 302, 303, 304, 305, 306, 307, 308,
+		400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 421, 422, 423, 424, 425, 426, 428, 429, 431, 451,
+		500, 501, 502, 503, 504, 505, 506, 507, 508, 510, 511, 599,
+	}
+
+	var (
+		ctx    = context.Background()
+		client = newHTTPClient(t)
+	)
+
+	for i := range validStatusCodes {
+		code := validStatusCodes[i]
+		t.Run(fmt.Sprint(code), func(t *testing.T) {
+			req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/status/%d", *httpbin, code), http.NoBody)
+			if err != nil {
+				t.Fatal(err)
+			}
+			resp, err := client.Do(req)
+			if err != nil {
+				t.Fatal(err)
+			}
+			defer resp.Body.Close()
+
+			if resp.StatusCode != code {
+				t.Fatalf("expected status code %d, got %d", code, resp.StatusCode)
+			}
+		})
+	}
+}

log.go

@@ -1,5 +1,7 @@
 package forwarder
 
+import "strings"
+
 // Logger is the logger used by the forwarder package.
 type Logger interface {
 	Errorf(format string, args ...interface{})
@@ -27,5 +29,10 @@ type goproxyLogger struct {
 }
 
 func (l goproxyLogger) Printf(format string, v ...interface{}) {
-	l.Debugf(format, v...)
+	if strings.HasPrefix(format, "[%03d] WARN: ") {
+		l.Logger.Infof(format[13:], v...)
+		return
+	}
+
+	l.Logger.Debugf(strings.Replace(format, "INFO: ", "", 1), v...)
 }