Add benchmark testing framework

.github/workflows/benchmark.yaml

@@ -0,0 +1,53 @@
+name: Benchmarking Tests at Scale
+on:
+  pull_request:
+    branches:
+      - "main"
+      - "release/v*"
+  workflow_dispatch:
+    inputs:
+      rps:
+        description: "The target requests-per-second rate. Default: 1000"
+        default: '1000'
+        type: string
+        required: false
+      connections:
+        description: "The maximum allowed number of concurrent connections per event loop. HTTP/1 only. Default: 100."
+        default: '100'
+        type: string
+        required: false
+      duration:
+        description: "The number of seconds that the test should run. Default: 90."
+        default: '90'
+        type: string
+        required: false
+      cpu_limits:
+        description: "The CPU resource limits for the envoy gateway, in unit 'm'. Default: 1000."
+        default: '1000'
+        type: string
+        required: false
+      memory_limits:
+        description: "The memory resource limits for the envoy gateway, in unit 'Mi'. Default: 1024."
+        default: '1024'
+        type: string
+        required: false
+
+jobs:
+  benchmark-test:
+    name: Benchmark Test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: ./tools/github-actions/setup-deps
+
+      - name: Run Benchmark tests
+        env:
+          KIND_NODE_TAG: v1.28.0
+          IMAGE_PULL_POLICY: IfNotPresent
+          BENCHMARK_RPS: ${{ github.event.inputs.rps || 1000 }}
+          BENCHMARK_CONNECTIONS: ${{ github.event.inputs.connections || 100 }}
+          BENCHMARK_DURATION: ${{ github.event.inputs.duration || 90 }}
+          BENCHMARK_CPU_LIMITS: ${{ github.event.inputs.cpu_limits || 1000 }}
+          BENCHMARK_MEMORY_LIMITS: ${{ github.event.inputs.memory_limits || 2048 }}
+        run: make benchmark

go.mod

@@ -164,7 +164,7 @@ require (
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/client_model v0.6.1
 	github.com/prometheus/procfs v0.15.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect

test/benchmark/benchmark_test.go

@@ -0,0 +1,56 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+//go:build benchmark
+// +build benchmark
+
+package benchmark
+
+import (
+	"flag"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+
+	"github.com/envoyproxy/gateway/test/benchmark/suite"
+	"github.com/envoyproxy/gateway/test/benchmark/tests"
+)
+
+func TestBenchmark(t *testing.T) {
+	cfg, err := config.GetConfig()
+	require.NoError(t, err)
+
+	cli, err := client.New(cfg, client.Options{})
+	require.NoError(t, err)
+
+	// Install all the scheme for kubernetes client.
+	suite.CheckInstallScheme(t, cli)
+
+	// Parse benchmark options.
+	flag.Parse()
+	options := suite.NewBenchmarkOptions(
+		*suite.RPS,
+		*suite.Connections,
+		*suite.Duration,
+		*suite.Concurrency,
+	)
+
+	bSuite, err := suite.NewBenchmarkTestSuite(
+		cli,
+		options,
+		"config/gateway.yaml",
+		"config/httproute.yaml",
+		"config/nighthawk-client.yaml",
+		*suite.ReportSavePath,
+	)
+	if err != nil {
+		t.Fatalf("Failed to create BenchmarkTestSuite: %v", err)
+	}
+
+	t.Logf("Running %d benchmark tests", len(tests.BenchmarkTests))
+	bSuite.Run(t, tests.BenchmarkTests)
+}

test/benchmark/config/gateway.yaml

@@ -0,0 +1,14 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: Gateway
+metadata:
+  name: "{GATEWAY_NAME}"
+  namespace: benchmark-test
+spec:
+  gatewayClassName: envoy-gateway
+  listeners:
+    - name: http
+      port: 8081
+      protocol: HTTP
+      allowedRoutes:
+        namespaces:
+          from: Same

test/benchmark/config/gatewayclass.yaml

@@ -0,0 +1,82 @@
+kind: GatewayClass
+apiVersion: gateway.networking.k8s.io/v1
+metadata:
+  name: envoy-gateway
+spec:
+  controllerName: gateway.envoyproxy.io/gatewayclass-controller
+  parametersRef:
+    group: gateway.envoyproxy.io
+    kind: EnvoyProxy
+    name: proxy-config
+    namespace: envoy-gateway-system
+---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: EnvoyProxy
+metadata:
+  name: proxy-config
+  namespace: envoy-gateway-system
+spec:
+  provider:
+    type: Kubernetes
+    kubernetes:
+      envoyDeployment:
+        container:
+          resources:
+            limits:
+              memory: "1024Mi"
+              cpu: "1000m"
+            requests:
+              memory: "256Mi"
+              cpu: "500m"
+  telemetry:
+    metrics:
+      prometheus: {}
+      sinks:
+        - type: OpenTelemetry
+          openTelemetry:
+            backendRefs:
+              - name: otel-collector
+                namespace: monitoring
+                port: 4317
+    accessLog:
+      settings:
+        - format:
+            type: Text
+            text: |
+              [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%"
+          sinks:
+            - type: File
+              file:
+                path: /dev/stdout
+            - type: OpenTelemetry
+              openTelemetry:
+                backendRefs:
+                  - name: otel-collector
+                    namespace: monitoring
+                    port: 4317
+                resources:
+                  k8s.cluster.name: "envoy-gateway"
+    tracing:
+      provider:
+        backendRefs:
+          - name: otel-collector
+            namespace: monitoring
+            port: 4317
+      customTags:
+        "k8s.cluster.name":
+          type: Literal
+          literal:
+            value: "envoy-gateway"
+        "k8s.pod.name":
+          type: Environment
+          environment:
+            name: ENVOY_POD_NAME
+            defaultValue: "-"
+        "k8s.namespace.name":
+          type: Environment
+          environment:
+            name: ENVOY_GATEWAY_NAMESPACE
+            defaultValue: "envoy-gateway-system"
+  shutdown:
+    drainTimeout: 5s
+    minDrainDuration: 1s

test/benchmark/config/httproute.yaml

@@ -0,0 +1,22 @@
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: "{HTTPROUTE_NAME}"
+  namespace: benchmark-test
+spec:
+  parentRefs:
+    - name: "{REF_GATEWAY_NAME}"
+  hostnames:
+    - "www.benchmark.com"
+  rules:
+    - backendRefs:
+        - group: ""
+          kind: Service
+          name: nighthawk-test-server
+          namespace: benchmark-test
+          port: 8080
+          weight: 1
+      matches:
+        - path:
+            type: PathPrefix
+            value: /

test/benchmark/config/nighthawk-client.yaml

@@ -0,0 +1,18 @@
+### Nighthawk test client job template
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "{NIGHTHAWK_CLIENT_NAME}"
+  namespace: benchmark-test
+  labels:
+    benchmark-test/client: "true"
+spec:
+  template:
+    spec:
+      containers:
+        - name: nighthawk-client
+          image: envoyproxy/nighthawk-dev:latest
+          imagePullPolicy: IfNotPresent
+          args: ["nighthawk_client"]  # Fill-up args at runtime
+      restartPolicy: Never
+  backoffLimit: 3

test/benchmark/config/nighthawk-test-server-config.yaml

@@ -0,0 +1,44 @@
+static_resources:
+  listeners:
+    # define an origin server on :10000 that always returns "lorem ipsum..."
+    - address:
+        socket_address:
+          address: 0.0.0.0
+          port_value: 8080
+      filter_chains:
+        - filters:
+            - name: envoy.filters.network.http_connection_manager
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+                generate_request_id: false
+                codec_type: AUTO
+                stat_prefix: ingress_http
+                route_config:
+                  name: local_route
+                  virtual_hosts:
+                    - name: service
+                      domains:
+                        - "*"
+                http_filters:
+                  - name: dynamic-delay
+                    typed_config:
+                      "@type": type.googleapis.com/nighthawk.server.DynamicDelayConfiguration
+                      static_delay: 0s
+                  - name: test-server  # before envoy.router because order matters!
+                    typed_config:
+                      "@type": type.googleapis.com/nighthawk.server.ResponseOptions
+                      response_body_size: 10
+                      v3_response_headers:
+                        - {header: {key: "foo", value: "bar"}}
+                        - {header: {key: "foo", value: "bar2"}, append: true}
+                        - {header: {key: "x-nh", value: "1"}}
+                  - name: envoy.filters.http.router
+                    typed_config:
+                      "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+                      dynamic_stats: false
+admin:
+  access_log_path: /tmp/envoy.log
+  address:
+    socket_address:
+      address: 0.0.0.0
+      port_value: 8081

test/benchmark/config/nighthawk-test-server.yaml

@@ -0,0 +1,53 @@
+### Nighthawk test server deployment & service
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nighthawk-test-server
+  namespace: benchmark-test
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: nighthawk-test-server
+  template:
+    metadata:
+      labels:
+        app: nighthawk-test-server
+    spec:
+      serviceAccountName: default
+      containers:
+        - name: nighthawk-server
+          image: envoyproxy/nighthawk-dev:latest
+          imagePullPolicy: IfNotPresent
+          args: ["nighthawk_test_server", "-c", "/etc/test-server-config/nighthawk-test-server-config.yaml"]
+          ports:
+            - containerPort: 8080
+          volumeMounts:
+            - name: test-server-config
+              mountPath: "/etc/test-server-config"
+          env:
+            - name: PORT
+              value: "8080"
+          resources:
+            requests:
+              cpu: "2"
+            limits:
+              cpu: "2"
+      volumes:
+        - name: test-server-config
+          configMap:
+            name: test-server-config  # Created directly from file
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: nighthawk-test-server
+  namespace: benchmark-test
+spec:
+  type: ClusterIP
+  selector:
+    app: nighthawk-test-server
+  ports:
+    - name: http
+      port: 8080
+      targetPort: 8080

test/benchmark/suite/client.go

@@ -0,0 +1,32 @@
+// Copyright Envoy Gateway Authors
+// SPDX-License-Identifier: Apache-2.0
+// The full text of the Apache license is available in the LICENSE file at
+// the root of the repo.
+
+//go:build benchmark
+// +build benchmark
+
+package suite
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	batchv1 "k8s.io/api/batch/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
+	gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
+	gwapiv1a3 "sigs.k8s.io/gateway-api/apis/v1alpha3"
+	gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1"
+
+	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
+)
+
+func CheckInstallScheme(t *testing.T, c client.Client) {
+	require.NoError(t, gwapiv1a3.Install(c.Scheme()))
+	require.NoError(t, gwapiv1a2.Install(c.Scheme()))
+	require.NoError(t, gwapiv1b1.Install(c.Scheme()))
+	require.NoError(t, gwapiv1.Install(c.Scheme()))
+	require.NoError(t, egv1a1.AddToScheme(c.Scheme()))
+	require.NoError(t, batchv1.AddToScheme(c.Scheme()))
+}