From 9bc6e99d403e2bb3fa63b10f45f5ca87dc88d246 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Wed, 12 Jun 2024 21:41:34 +0800 Subject: [PATCH 01/30] initial implementation of benchmark test suite Signed-off-by: shawnh2 --- test/benchmark/benchmark_test.go | 43 +++++++ test/benchmark/config/gateway.yaml | 14 +++ test/benchmark/config/gatewayclass.yaml | 84 ++++++++++++++ test/benchmark/config/httproute.yaml | 22 ++++ test/benchmark/config/test-server-config.yaml | 44 +++++++ test/benchmark/config/test-server.yaml | 52 +++++++++ test/benchmark/tests/scale_httproutes.go | 33 ++++++ test/benchmark/tests/tests.go | 13 +++ test/benchmark/utils/client.go | 27 +++++ test/benchmark/utils/suite.go | 107 ++++++++++++++++++ test/benchmark/utils/test.go | 27 +++++ tools/make/kube.mk | 18 +++ 12 files changed, 484 insertions(+) create mode 100644 test/benchmark/benchmark_test.go create mode 100644 test/benchmark/config/gateway.yaml create mode 100644 test/benchmark/config/gatewayclass.yaml create mode 100644 test/benchmark/config/httproute.yaml create mode 100644 test/benchmark/config/test-server-config.yaml create mode 100644 test/benchmark/config/test-server.yaml create mode 100644 test/benchmark/tests/scale_httproutes.go create mode 100644 test/benchmark/tests/tests.go create mode 100644 test/benchmark/utils/client.go create mode 100644 test/benchmark/utils/suite.go create mode 100644 test/benchmark/utils/test.go diff --git a/test/benchmark/benchmark_test.go b/test/benchmark/benchmark_test.go new file mode 100644 index 00000000000..d78b5b5a11f --- /dev/null +++ b/test/benchmark/benchmark_test.go @@ -0,0 +1,43 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package benchmark + +import ( + "testing" + + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + + "github.com/envoyproxy/gateway/test/benchmark/tests" + "github.com/envoyproxy/gateway/test/benchmark/utils" +) + +func TestBenchmark(t *testing.T) { + cfg, err := config.GetConfig() + require.NoError(t, err) + + cli, err := client.New(cfg, client.Options{}) + require.NoError(t, err) + + // Install all the scheme for kubernetes client. + utils.CheckInstallScheme(t, cli) + + bSuite, err := utils.NewBenchmarkTestSuite( + cli, + "config/gateway.yaml", + "config/httproute.yaml", + ) + if err != nil { + t.Fatalf("Failed to create BenchmarkTestSuite: %v", err) + } + + t.Logf("Running %d benchmark tests", len(tests.BenchmarkTests)) + bSuite.Run(t, tests.BenchmarkTests) +} diff --git a/test/benchmark/config/gateway.yaml b/test/benchmark/config/gateway.yaml new file mode 100644 index 00000000000..d7863e86ac5 --- /dev/null +++ b/test/benchmark/config/gateway.yaml @@ -0,0 +1,14 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: "{GATEWAY_NAME}" + namespace: benchmark-test +spec: + gatewayClassName: envoy-gateway + listeners: + - name: http + port: 8081 + protocol: HTTP + allowedRoutes: + namespaces: + from: Same diff --git a/test/benchmark/config/gatewayclass.yaml b/test/benchmark/config/gatewayclass.yaml new file mode 100644 index 00000000000..280af5a1769 --- /dev/null +++ b/test/benchmark/config/gatewayclass.yaml @@ -0,0 +1,84 @@ +kind: GatewayClass +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: envoy-gateway +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: proxy-config + namespace: envoy-gateway-system +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: proxy-config + namespace: envoy-gateway-system +spec: + provider: + type: Kubernetes + kubernetes: + envoyDeployment: + container: + volumeMounts: + - mountPath: /var/run/ext-proc + name: socket-dir + pod: + volumes: + - name: socket-dir + hostPath: + path: /var/run/ext-proc + type: '' + telemetry: + metrics: + prometheus: {} + sinks: + - type: OpenTelemetry + openTelemetry: + backendRefs: + - name: otel-collector + namespace: monitoring + port: 4317 + accessLog: + settings: + - format: + type: Text + text: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% %RESPONSE_FLAGS% %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "%REQ(X-FORWARDED-FOR)%" "%REQ(USER-AGENT)%" "%REQ(X-REQUEST-ID)%" "%REQ(:AUTHORITY)%" "%UPSTREAM_HOST%" + sinks: + - type: File + file: + path: /dev/stdout + - type: OpenTelemetry + openTelemetry: + backendRefs: + - name: otel-collector + namespace: monitoring + port: 4317 + resources: + k8s.cluster.name: "envoy-gateway" + tracing: + provider: + backendRefs: + - name: otel-collector + namespace: monitoring + port: 4317 + customTags: + "k8s.cluster.name": + type: Literal + literal: + value: "envoy-gateway" + "k8s.pod.name": + type: Environment + environment: + name: ENVOY_POD_NAME + defaultValue: "-" + "k8s.namespace.name": + type: Environment + environment: + name: ENVOY_GATEWAY_NAMESPACE + defaultValue: "envoy-gateway-system" + shutdown: + drainTimeout: 5s + minDrainDuration: 1s diff --git a/test/benchmark/config/httproute.yaml b/test/benchmark/config/httproute.yaml new file mode 100644 index 00000000000..b422d4f2d85 --- /dev/null +++ b/test/benchmark/config/httproute.yaml @@ -0,0 +1,22 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: "{HTTPROUTE_NAME}" + namespace: benchmark-test +spec: + parentRefs: + - name: "{REF_GATEWAY_NAME}" + hostnames: + - "www.benchmark.com" + rules: + - backendRefs: + - group: "" + kind: Service + name: nighthawk-test-server + namespace: benchmark-test + port: 8080 + weight: 1 + matches: + - path: + type: PathPrefix + value: / diff --git a/test/benchmark/config/test-server-config.yaml b/test/benchmark/config/test-server-config.yaml new file mode 100644 index 00000000000..fc63a00b18c --- /dev/null +++ b/test/benchmark/config/test-server-config.yaml @@ -0,0 +1,44 @@ +static_resources: + listeners: + # define an origin server on :10000 that always returns "lorem ipsum..." + - address: + socket_address: + address: 0.0.0.0 + port_value: 8080 + filter_chains: + - filters: + - name: envoy.filters.network.http_connection_manager + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + generate_request_id: false + codec_type: AUTO + stat_prefix: ingress_http + route_config: + name: local_route + virtual_hosts: + - name: service + domains: + - "*" + http_filters: + - name: dynamic-delay + typed_config: + "@type": type.googleapis.com/nighthawk.server.DynamicDelayConfiguration + static_delay: 0.5s + - name: test-server # before envoy.router because order matters! + typed_config: + "@type": type.googleapis.com/nighthawk.server.ResponseOptions + response_body_size: 10 + v3_response_headers: + - {header: {key: "foo", value: "bar"}} + - {header: {key: "foo", value: "bar2"}, append: true} + - {header: {key: "x-nh", value: "1"}} + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + dynamic_stats: false +admin: + access_log_path: /tmp/envoy.log + address: + socket_address: + address: 0.0.0.0 + port_value: 8081 diff --git a/test/benchmark/config/test-server.yaml b/test/benchmark/config/test-server.yaml new file mode 100644 index 00000000000..23cef6876e5 --- /dev/null +++ b/test/benchmark/config/test-server.yaml @@ -0,0 +1,52 @@ +### Nighthawk test server deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nighthawk-test-server + namespace: benchmark-test +spec: + replicas: 1 + selector: + matchLabels: + app: nighthawk-test-server + template: + metadata: + labels: + app: nighthawk-test-server + spec: + serviceAccountName: default + containers: + - name: nighthawk-server + image: envoyproxy/nighthawk-dev:latest + args: ["nighthawk_test_server", "-c", "/etc/test-server-config/test-server-config.yaml"] + ports: + - containerPort: 8080 + volumeMounts: + - name: test-server-config + mountPath: "/etc/test-server-config" + env: + - name: PORT + value: "8080" + resources: + requests: + cpu: "2" + limits: + cpu: "2" + volumes: + - name: test-server-config + configMap: + name: test-server-config # Created directly from file +--- +apiVersion: v1 +kind: Service +metadata: + name: nighthawk-test-server + namespace: benchmark-test +spec: + type: ClusterIP + selector: + app: nighthawk-test-server + ports: + - name: http + port: 8080 + targetPort: 8080 diff --git a/test/benchmark/tests/scale_httproutes.go b/test/benchmark/tests/scale_httproutes.go new file mode 100644 index 00000000000..194f20e58d5 --- /dev/null +++ b/test/benchmark/tests/scale_httproutes.go @@ -0,0 +1,33 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package tests + +import ( + "github.com/envoyproxy/gateway/test/benchmark/utils" +) + +func init() { + BenchmarkTests = append(BenchmarkTests, ScaleHTTPRoutes) +} + +var ScaleHTTPRoutes = utils.BenchmarkTest{ + ShortName: "ScaleHTTPRoute", + Description: "Fixed one Gateway with different scale of HTTPRoutes", + GatewayTargets: map[string]*utils.GatewayTarget{ + "benchmark-gateway": { + Bounds: []uint16{1}, + }, + }, + HTTPRoutTargets: map[string]*utils.HTTPRouteTarget{ + "benchmark-route": { + TargetGateway: "benchmark-gateway", + Bounds: []uint16{10, 100, 300, 500}, + }, + }, +} diff --git a/test/benchmark/tests/tests.go b/test/benchmark/tests/tests.go new file mode 100644 index 00000000000..df4502f7b26 --- /dev/null +++ b/test/benchmark/tests/tests.go @@ -0,0 +1,13 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package tests + +import "github.com/envoyproxy/gateway/test/benchmark/utils" + +var BenchmarkTests []utils.BenchmarkTest diff --git a/test/benchmark/utils/client.go b/test/benchmark/utils/client.go new file mode 100644 index 00000000000..4f24eddf2c9 --- /dev/null +++ b/test/benchmark/utils/client.go @@ -0,0 +1,27 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package utils + +import ( + "testing" + + "github.com/stretchr/testify/require" + "sigs.k8s.io/controller-runtime/pkg/client" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwapiv1a3 "sigs.k8s.io/gateway-api/apis/v1alpha3" + gwapiv1b1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" +) + +func CheckInstallScheme(t *testing.T, c client.Client) { + require.NoError(t, gwapiv1a3.Install(c.Scheme())) + require.NoError(t, gwapiv1a2.Install(c.Scheme())) + require.NoError(t, gwapiv1b1.Install(c.Scheme())) + require.NoError(t, gwapiv1.Install(c.Scheme())) + require.NoError(t, egv1a1.AddToScheme(c.Scheme())) +} diff --git a/test/benchmark/utils/suite.go b/test/benchmark/utils/suite.go new file mode 100644 index 00000000000..4fd9721f691 --- /dev/null +++ b/test/benchmark/utils/suite.go @@ -0,0 +1,107 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package utils + +import ( + "fmt" + "os" + "testing" + + "sigs.k8s.io/controller-runtime/pkg/client" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/yaml" +) + +type BenchmarkTestSuite struct { + Client client.Client + + // Resources template for supported benchmark targets. + GatewayTemplate *gwapiv1.Gateway + HTTPRouteTemplate *gwapiv1.HTTPRoute +} + +func NewBenchmarkTestSuite(client client.Client, gatewayManifest, httpRouteManifest string) (*BenchmarkTestSuite, error) { + var ( + gateway *gwapiv1.Gateway + httproute *gwapiv1.HTTPRoute + ) + + gm, err := os.ReadFile(gatewayManifest) + if err != nil { + return nil, err + } + + hm, err := os.ReadFile(httpRouteManifest) + if err != nil { + return nil, err + } + + if err = yaml.Unmarshal(gm, gateway); err != nil { + return nil, err + } + + if err = yaml.Unmarshal(hm, httproute); err != nil { + return nil, err + } + + return &BenchmarkTestSuite{ + Client: client, + GatewayTemplate: gateway, + HTTPRouteTemplate: httproute, + }, nil +} + +func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { + t.Logf("Running benchmark test") + + for _, test := range tests { + t.Logf("Running benchmark test: %s", test.ShortName) + + b.runTarget(t, test) + } +} + +func (b *BenchmarkTestSuite) runTarget(t *testing.T, test BenchmarkTest) { + for gatewayName, gatewayTarget := range test.GatewayTargets { + if gatewayTarget == nil { + continue + } + + matched := false + for httprouteName, httprouteTarget := range test.HTTPRoutTargets { + if httprouteTarget != nil && + gatewayName == httprouteTarget.TargetGateway { + t.Logf("Running benchmark test: %s for '%s' gateway and '%s' httproutes", + test.ShortName, gatewayName, httprouteName) + + matched = true + b.runBounds(t, test, gatewayTarget, httprouteTarget) + break + } + } + + if !matched { + t.Errorf("Unable to find any matched httproutes for gateway '%s'", gatewayName) + } + } +} + +func (b *BenchmarkTestSuite) runBounds(t *testing.T, test BenchmarkTest, gatewayTarget *GatewayTarget, httprouteTarget *HTTPRouteTarget) { + for _, gatewayBound := range gatewayTarget.Bounds { + for _, httprouteBound := range httprouteTarget.Bounds { + name := fmt.Sprintf("gateway scale = %d and httproute scale = %d", gatewayBound, httprouteBound) + + t.Logf("Running benchmark test: %s with %s", test.ShortName, name) + + t.Run(name, func(t *testing.T) { + // TODO: implement benchmark run + }) + } + } +} diff --git a/test/benchmark/utils/test.go b/test/benchmark/utils/test.go new file mode 100644 index 00000000000..64a20f85b92 --- /dev/null +++ b/test/benchmark/utils/test.go @@ -0,0 +1,27 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package utils + +type BenchmarkTest struct { + ShortName string + Description string + + // Define tha benchmark targets, key is target name. + GatewayTargets map[string]*GatewayTarget + HTTPRoutTargets map[string]*HTTPRouteTarget +} + +type GatewayTarget struct { + Bounds []uint16 +} + +type HTTPRouteTarget struct { + Bounds []uint16 + TargetGateway string +} diff --git a/tools/make/kube.mk b/tools/make/kube.mk index eec42ea2395..75035eb7e70 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -103,6 +103,9 @@ conformance: create-cluster kube-install-image kube-deploy run-conformance delet .PHONY: experimental-conformance ## Create a kind cluster, deploy EG into it, run Gateway API experimental conformance, and clean up. experimental-conformance: create-cluster kube-install-image kube-deploy run-experimental-conformance delete-cluster ## Create a kind cluster, deploy EG into it, run Gateway API conformance, and clean up. +.PHONY: benchmark +benchmark: create-cluster kube-install-image kube-deploy run-benchmark delete-cluster ## Create a kind cluster, deploy EG into it, run Envoy Gateway benchmark test, and clean up. + .PHONY: e2e e2e: create-cluster kube-install-image kube-deploy install-ratelimit run-e2e delete-cluster @@ -138,6 +141,21 @@ else endif endif +.PHONY: run-benchmark +run-benchmark: install-benchmark-server ## Run benchmark tests + @$(LOG_TARGET) + kubectl wait --timeout=$(WAIT_TIMEOUT) -n benchmark-test deployment/nighthawk-test-server --for=condition=Available + kubectl wait --timeout=$(WAIT_TIMEOUT) -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available + kubectl apply -f test/benchmark/config/gatewayclass.yaml + go test -v -tags benchmark ./test/benchmark + +.PHONY: install-benchmark-server +install-benchmark-server: ## Install nighthawk server for benchmark test + @$(LOG_TARGET) + kubectl create namespace benchmark-test + kubectl -n benchmark-test create configmap test-server-config --from-file=test/benchmark/config/test-server-config.yaml --output yaml + kubectl apply -f test/benchmark/config/test-server.yaml + .PHONY: install-e2e-telemetry install-e2e-telemetry: prepare-helm-repo install-fluent-bit install-loki install-tempo install-otel-collector install-prometheus @$(LOG_TARGET) From 32ecca3a358651d59bd0019376923ae21b9bcf24 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Thu, 13 Jun 2024 23:01:00 +0800 Subject: [PATCH 02/30] implement benchmark test run Signed-off-by: shawnh2 --- test/benchmark/benchmark_test.go | 19 +- test/benchmark/config/nighthawk-client.yaml | 16 ++ ...yaml => nighthawk-test-server-config.yaml} | 0 ...server.yaml => nighthawk-test-server.yaml} | 3 +- test/benchmark/{utils => suite}/client.go | 7 +- test/benchmark/suite/flags.go | 19 ++ test/benchmark/suite/suite.go | 207 ++++++++++++++++++ test/benchmark/suite/test.go | 38 ++++ test/benchmark/tests/scale_httproutes.go | 64 ++++-- test/benchmark/tests/tests.go | 4 +- test/benchmark/utils/suite.go | 107 --------- test/benchmark/utils/test.go | 27 --- tools/make/kube.mk | 4 +- 13 files changed, 359 insertions(+), 156 deletions(-) create mode 100644 test/benchmark/config/nighthawk-client.yaml rename test/benchmark/config/{test-server-config.yaml => nighthawk-test-server-config.yaml} (100%) rename test/benchmark/config/{test-server.yaml => nighthawk-test-server.yaml} (93%) rename test/benchmark/{utils => suite}/client.go (85%) create mode 100644 test/benchmark/suite/flags.go create mode 100644 test/benchmark/suite/suite.go create mode 100644 test/benchmark/suite/test.go delete mode 100644 test/benchmark/utils/suite.go delete mode 100644 test/benchmark/utils/test.go diff --git a/test/benchmark/benchmark_test.go b/test/benchmark/benchmark_test.go index d78b5b5a11f..48c470cde5b 100644 --- a/test/benchmark/benchmark_test.go +++ b/test/benchmark/benchmark_test.go @@ -9,14 +9,15 @@ package benchmark import ( + "flag" "testing" "github.com/stretchr/testify/require" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/config" + "github.com/envoyproxy/gateway/test/benchmark/suite" "github.com/envoyproxy/gateway/test/benchmark/tests" - "github.com/envoyproxy/gateway/test/benchmark/utils" ) func TestBenchmark(t *testing.T) { @@ -27,12 +28,24 @@ func TestBenchmark(t *testing.T) { require.NoError(t, err) // Install all the scheme for kubernetes client. - utils.CheckInstallScheme(t, cli) + suite.CheckInstallScheme(t, cli) + + // Parse benchmark options. + flag.Parse() + options := suite.NewBenchmarkOptions( + *suite.RPS, + *suite.Connections, + *suite.Duration, + *suite.Concurrency, + *suite.PrefetchConnections, + ) - bSuite, err := utils.NewBenchmarkTestSuite( + bSuite, err := suite.NewBenchmarkTestSuite( cli, + options, "config/gateway.yaml", "config/httproute.yaml", + "config/nighthawk-client.yaml", ) if err != nil { t.Fatalf("Failed to create BenchmarkTestSuite: %v", err) diff --git a/test/benchmark/config/nighthawk-client.yaml b/test/benchmark/config/nighthawk-client.yaml new file mode 100644 index 00000000000..a9621d3fb72 --- /dev/null +++ b/test/benchmark/config/nighthawk-client.yaml @@ -0,0 +1,16 @@ +### Nighthawk test client job template +apiVersion: batch/v1 +kind: Job +metadata: + name: "{NIGHTHAWK_CLIENT_NAME}" + namespace: benchmark-test +spec: + template: + spec: + containers: + - name: nighthawk-client + image: envoyproxy/nighthawk-dev:latest + imagePullPolicy: IfNotPresent + args: ["nighthawk_client"] # Fill-up args at runtime + restartPolicy: Never + backoffLimit: 3 diff --git a/test/benchmark/config/test-server-config.yaml b/test/benchmark/config/nighthawk-test-server-config.yaml similarity index 100% rename from test/benchmark/config/test-server-config.yaml rename to test/benchmark/config/nighthawk-test-server-config.yaml diff --git a/test/benchmark/config/test-server.yaml b/test/benchmark/config/nighthawk-test-server.yaml similarity index 93% rename from test/benchmark/config/test-server.yaml rename to test/benchmark/config/nighthawk-test-server.yaml index 23cef6876e5..1165789e1f7 100644 --- a/test/benchmark/config/test-server.yaml +++ b/test/benchmark/config/nighthawk-test-server.yaml @@ -1,4 +1,4 @@ -### Nighthawk test server deployment +### Nighthawk test server deployment & service apiVersion: apps/v1 kind: Deployment metadata: @@ -18,6 +18,7 @@ spec: containers: - name: nighthawk-server image: envoyproxy/nighthawk-dev:latest + imagePullPolicy: IfNotPresent args: ["nighthawk_test_server", "-c", "/etc/test-server-config/test-server-config.yaml"] ports: - containerPort: 8080 diff --git a/test/benchmark/utils/client.go b/test/benchmark/suite/client.go similarity index 85% rename from test/benchmark/utils/client.go rename to test/benchmark/suite/client.go index 4f24eddf2c9..3db4c16c52d 100644 --- a/test/benchmark/utils/client.go +++ b/test/benchmark/suite/client.go @@ -3,12 +3,16 @@ // The full text of the Apache license is available in the LICENSE file at // the root of the repo. -package utils +//go:build benchmark +// +build benchmark + +package suite import ( "testing" "github.com/stretchr/testify/require" + batchv1 "k8s.io/api/batch/v1" "sigs.k8s.io/controller-runtime/pkg/client" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2" @@ -24,4 +28,5 @@ func CheckInstallScheme(t *testing.T, c client.Client) { require.NoError(t, gwapiv1b1.Install(c.Scheme())) require.NoError(t, gwapiv1.Install(c.Scheme())) require.NoError(t, egv1a1.AddToScheme(c.Scheme())) + require.NoError(t, batchv1.AddToScheme(c.Scheme())) } diff --git a/test/benchmark/suite/flags.go b/test/benchmark/suite/flags.go new file mode 100644 index 00000000000..7238917226a --- /dev/null +++ b/test/benchmark/suite/flags.go @@ -0,0 +1,19 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package suite + +import "flag" + +var ( + RPS = flag.String("rps", "1000", "RPS of benchmark test client") + Connections = flag.String("connections", "10", "Connections of benchmark test client") + Duration = flag.String("duration", "60", "Duration of benchmark test client") + Concurrency = flag.String("concurrency", "auto", "Concurrency of benchmark test client") + PrefetchConnections = flag.Bool("prefetch-connections", true, "PrefetchConnections of benchmark test client") +) diff --git a/test/benchmark/suite/suite.go b/test/benchmark/suite/suite.go new file mode 100644 index 00000000000..9f0cc8cea16 --- /dev/null +++ b/test/benchmark/suite/suite.go @@ -0,0 +1,207 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package suite + +import ( + "context" + "fmt" + "os" + "testing" + + batchv1 "k8s.io/api/batch/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/conformance/utils/config" + "sigs.k8s.io/yaml" +) + +const ( + ScaledLabelKey = "benchmark-test/scaled" + DefaultControllerName = "gateway.envoyproxy.io/gatewayclass-controller" +) + +type BenchmarkTestSuite struct { + Client client.Client + TimeoutConfig config.TimeoutConfig + ControllerName string + Options BenchmarkOptions + + // Resources template for supported benchmark targets. + GatewayTemplate *gwapiv1.Gateway + HTTPRouteTemplate *gwapiv1.HTTPRoute + + // Template for benchmark test client. + BenchmarkClient *batchv1.Job + + // Indicates which resources are scaled. + scaledLabel map[string]string +} + +func NewBenchmarkTestSuite(client client.Client, options BenchmarkOptions, + gatewayManifest, httpRouteManifest, benchmarkClientManifest string) (*BenchmarkTestSuite, error) { + var ( + gateway = new(gwapiv1.Gateway) + httproute = new(gwapiv1.HTTPRoute) + benchmarkClient = new(batchv1.Job) + timeoutConfig = config.TimeoutConfig{} + ) + + data, err := os.ReadFile(gatewayManifest) + if err != nil { + return nil, err + } + if err = yaml.Unmarshal(data, gateway); err != nil { + return nil, err + } + + data, err = os.ReadFile(httpRouteManifest) + if err != nil { + return nil, err + } + if err = yaml.Unmarshal(data, httproute); err != nil { + return nil, err + } + + data, err = os.ReadFile(benchmarkClientManifest) + if err != nil { + return nil, err + } + if err = yaml.Unmarshal(data, benchmarkClient); err != nil { + return nil, err + } + + config.SetupTimeoutConfig(&timeoutConfig) + + // Prepare static options for benchmark client. + srcArgs := prepareBenchmarkClientStaticArgs(options) + dstArgs := benchmarkClient.Spec.Template.Spec.Containers[0].Args + dstArgs = append(dstArgs, srcArgs...) + + return &BenchmarkTestSuite{ + Client: client, + Options: options, + TimeoutConfig: timeoutConfig, + ControllerName: DefaultControllerName, + GatewayTemplate: gateway, + HTTPRouteTemplate: httproute, + BenchmarkClient: benchmarkClient, + scaledLabel: map[string]string{ + ScaledLabelKey: "true", + }, + }, nil +} + +func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { + t.Logf("Running %d benchmark test", len(tests)) + + for _, test := range tests { + t.Logf("Running benchmark test: %s", test.ShortName) + + test.Test(t, b) + } +} + +// Benchmark prepares and runs benchmark test as a Kubernetes Job. +// +// TODO: currently running benchmark test via nighthawk-client, +// consider switching to gRPC nighthawk-service for benchmark test. +// ref: https://github.com/envoyproxy/nighthawk/blob/main/api/client/service.proto +func (b *BenchmarkTestSuite) Benchmark() { + // TODO: + // 1. prepare job + // 2. create and run job + // 3. wait job complete + // 4. scrap job log as report +} + +func prepareBenchmarkClientStaticArgs(options BenchmarkOptions) []string { + staticArgs := []string{ + "--rps", options.RPS, + "--connections", options.Connections, + "--duration", options.Duration, + "--concurrency", options.Concurrency, + } + if options.PrefetchConnections { + staticArgs = append(staticArgs, "--prefetch-connections") + } + return staticArgs +} + +func prepareBenchmarkClientArgs(gatewayHost string, requestHeaders ...string) []string { + args := make([]string, 0, len(requestHeaders)*2+1) + + for _, reqHeader := range requestHeaders { + args = append(args, "--request-header", reqHeader) + } + args = append(args, gatewayHost) + + return args +} + +// ScaleHTTPRoutes scales HTTPRoutes that are all referenced to one Gateway according to +// the scale range: (begin, end]. The afterCreation is a callback function that only runs +// everytime after one HTTPRoutes has been created successfully. +// +// All scaled resources will be labeled with ScaledLabelKey. +func (b *BenchmarkTestSuite) ScaleHTTPRoutes(ctx context.Context, scaleRange [2]uint16, targetName, refGateway string, afterCreation func(route *gwapiv1.HTTPRoute)) error { + var ( + i uint16 + begin, end = scaleRange[0], scaleRange[1] + ) + + for i = begin + 1; i <= end; i++ { + routeName := fmt.Sprintf(targetName, i) + newRoute := b.HTTPRouteTemplate.DeepCopy() + newRoute.SetName(routeName) + newRoute.SetLabels(b.scaledLabel) + newRoute.Spec.ParentRefs[0].Name = gwapiv1.ObjectName(refGateway) + + if err := b.CreateResource(ctx, newRoute); err != nil { + return err + } + + if afterCreation != nil { + afterCreation(newRoute) + } + } + + return nil +} + +func (b *BenchmarkTestSuite) CreateResource(ctx context.Context, object client.Object) error { + if err := b.Client.Create(ctx, object); err != nil { + if !kerrors.IsAlreadyExists(err) { + return err + } else { + return nil + } + } + return nil +} + +func (b *BenchmarkTestSuite) CleanupResource(ctx context.Context, object client.Object) error { + if err := b.Client.Delete(ctx, object); err != nil { + if !kerrors.IsNotFound(err) { + return err + } else { + return nil + } + } + return nil +} + +// CleanupScaledResources only cleanups all the resources with Scaled label under benchmark-test namespace. +func (b *BenchmarkTestSuite) CleanupScaledResources(ctx context.Context, object client.Object) error { + if err := b.Client.DeleteAllOf(ctx, object, + client.MatchingLabels{ScaledLabelKey: "true"}, client.InNamespace("benchmark-test")); err != nil { + return err + } + return nil +} diff --git a/test/benchmark/suite/test.go b/test/benchmark/suite/test.go new file mode 100644 index 00000000000..9e916b1b4d9 --- /dev/null +++ b/test/benchmark/suite/test.go @@ -0,0 +1,38 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package suite + +import "testing" + +type BenchmarkTest struct { + ShortName string + Description string + Test func(*testing.T, *BenchmarkTestSuite) +} + +// BenchmarkOptions for nighthawk-client. +type BenchmarkOptions struct { + RPS string + Connections string + Duration string + Concurrency string + PrefetchConnections bool +} + +func NewBenchmarkOptions( + rps, connections, duration, concurrency string, prefetchConnections bool, +) BenchmarkOptions { + return BenchmarkOptions{ + RPS: rps, + Connections: connections, + Duration: duration, + Concurrency: concurrency, + PrefetchConnections: prefetchConnections, + } +} diff --git a/test/benchmark/tests/scale_httproutes.go b/test/benchmark/tests/scale_httproutes.go index 194f20e58d5..83b73962fad 100644 --- a/test/benchmark/tests/scale_httproutes.go +++ b/test/benchmark/tests/scale_httproutes.go @@ -9,25 +9,63 @@ package tests import ( - "github.com/envoyproxy/gateway/test/benchmark/utils" + "context" + "testing" + + "github.com/envoyproxy/gateway/test/benchmark/suite" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/types" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" ) func init() { BenchmarkTests = append(BenchmarkTests, ScaleHTTPRoutes) } -var ScaleHTTPRoutes = utils.BenchmarkTest{ +var ScaleHTTPRoutes = suite.BenchmarkTest{ ShortName: "ScaleHTTPRoute", - Description: "Fixed one Gateway with different scale of HTTPRoutes", - GatewayTargets: map[string]*utils.GatewayTarget{ - "benchmark-gateway": { - Bounds: []uint16{1}, - }, - }, - HTTPRoutTargets: map[string]*utils.HTTPRouteTarget{ - "benchmark-route": { - TargetGateway: "benchmark-gateway", - Bounds: []uint16{10, 100, 300, 500}, - }, + Description: "Fixed one Gateway with different scale of HTTPRoutes: [10, 50, 100, 300, 500].", + Test: func(t *testing.T, suite *suite.BenchmarkTestSuite) { + var ( + ctx = context.Background() + ns = "benchmark-test" + err error + ) + + // Setup and create a gateway. + gatewayNN := types.NamespacedName{Name: "benchmark", Namespace: ns} + gateway := suite.GatewayTemplate.DeepCopy() + gateway.SetName(gatewayNN.Name) + err = suite.CreateResource(ctx, gateway) + require.NoError(t, err) + + // Setup and scale httproutes. + httpRouteName := "benchmark-route-%d" + httpRouteScales := []uint16{5, 10, 20} + httpRouteNNs := make([]types.NamespacedName, 0, httpRouteScales[len(httpRouteScales)-1]) + + var start uint16 = 0 + for _, scale := range httpRouteScales { + t.Logf("Scaling HTTPRoutes to %d", scale) + + err = suite.ScaleHTTPRoutes(ctx, [2]uint16{start, scale}, httpRouteName, gatewayNN.Name, func(route *gwapiv1.HTTPRoute) { + routeNN := types.NamespacedName{Name: route.Name, Namespace: route.Namespace} + httpRouteNNs = append(httpRouteNNs, routeNN) + }) + require.NoError(t, err) + start = scale + + kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gatewayNN), httpRouteNNs...) + + // TODO: run benchmark and report + } + + // Cleanup + err = suite.CleanupResource(ctx, gateway) + require.NoError(t, err) + + err = suite.CleanupScaledResources(ctx, &gwapiv1.HTTPRoute{}) + require.NoError(t, err) }, } diff --git a/test/benchmark/tests/tests.go b/test/benchmark/tests/tests.go index df4502f7b26..0aa49a13bb1 100644 --- a/test/benchmark/tests/tests.go +++ b/test/benchmark/tests/tests.go @@ -8,6 +8,6 @@ package tests -import "github.com/envoyproxy/gateway/test/benchmark/utils" +import "github.com/envoyproxy/gateway/test/benchmark/suite" -var BenchmarkTests []utils.BenchmarkTest +var BenchmarkTests []suite.BenchmarkTest diff --git a/test/benchmark/utils/suite.go b/test/benchmark/utils/suite.go deleted file mode 100644 index 4fd9721f691..00000000000 --- a/test/benchmark/utils/suite.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -//go:build benchmark -// +build benchmark - -package utils - -import ( - "fmt" - "os" - "testing" - - "sigs.k8s.io/controller-runtime/pkg/client" - gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" - "sigs.k8s.io/yaml" -) - -type BenchmarkTestSuite struct { - Client client.Client - - // Resources template for supported benchmark targets. - GatewayTemplate *gwapiv1.Gateway - HTTPRouteTemplate *gwapiv1.HTTPRoute -} - -func NewBenchmarkTestSuite(client client.Client, gatewayManifest, httpRouteManifest string) (*BenchmarkTestSuite, error) { - var ( - gateway *gwapiv1.Gateway - httproute *gwapiv1.HTTPRoute - ) - - gm, err := os.ReadFile(gatewayManifest) - if err != nil { - return nil, err - } - - hm, err := os.ReadFile(httpRouteManifest) - if err != nil { - return nil, err - } - - if err = yaml.Unmarshal(gm, gateway); err != nil { - return nil, err - } - - if err = yaml.Unmarshal(hm, httproute); err != nil { - return nil, err - } - - return &BenchmarkTestSuite{ - Client: client, - GatewayTemplate: gateway, - HTTPRouteTemplate: httproute, - }, nil -} - -func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { - t.Logf("Running benchmark test") - - for _, test := range tests { - t.Logf("Running benchmark test: %s", test.ShortName) - - b.runTarget(t, test) - } -} - -func (b *BenchmarkTestSuite) runTarget(t *testing.T, test BenchmarkTest) { - for gatewayName, gatewayTarget := range test.GatewayTargets { - if gatewayTarget == nil { - continue - } - - matched := false - for httprouteName, httprouteTarget := range test.HTTPRoutTargets { - if httprouteTarget != nil && - gatewayName == httprouteTarget.TargetGateway { - t.Logf("Running benchmark test: %s for '%s' gateway and '%s' httproutes", - test.ShortName, gatewayName, httprouteName) - - matched = true - b.runBounds(t, test, gatewayTarget, httprouteTarget) - break - } - } - - if !matched { - t.Errorf("Unable to find any matched httproutes for gateway '%s'", gatewayName) - } - } -} - -func (b *BenchmarkTestSuite) runBounds(t *testing.T, test BenchmarkTest, gatewayTarget *GatewayTarget, httprouteTarget *HTTPRouteTarget) { - for _, gatewayBound := range gatewayTarget.Bounds { - for _, httprouteBound := range httprouteTarget.Bounds { - name := fmt.Sprintf("gateway scale = %d and httproute scale = %d", gatewayBound, httprouteBound) - - t.Logf("Running benchmark test: %s with %s", test.ShortName, name) - - t.Run(name, func(t *testing.T) { - // TODO: implement benchmark run - }) - } - } -} diff --git a/test/benchmark/utils/test.go b/test/benchmark/utils/test.go deleted file mode 100644 index 64a20f85b92..00000000000 --- a/test/benchmark/utils/test.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Envoy Gateway Authors -// SPDX-License-Identifier: Apache-2.0 -// The full text of the Apache license is available in the LICENSE file at -// the root of the repo. - -//go:build benchmark -// +build benchmark - -package utils - -type BenchmarkTest struct { - ShortName string - Description string - - // Define tha benchmark targets, key is target name. - GatewayTargets map[string]*GatewayTarget - HTTPRoutTargets map[string]*HTTPRouteTarget -} - -type GatewayTarget struct { - Bounds []uint16 -} - -type HTTPRouteTarget struct { - Bounds []uint16 - TargetGateway string -} diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 75035eb7e70..be884e863ca 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -153,8 +153,8 @@ run-benchmark: install-benchmark-server ## Run benchmark tests install-benchmark-server: ## Install nighthawk server for benchmark test @$(LOG_TARGET) kubectl create namespace benchmark-test - kubectl -n benchmark-test create configmap test-server-config --from-file=test/benchmark/config/test-server-config.yaml --output yaml - kubectl apply -f test/benchmark/config/test-server.yaml + kubectl -n benchmark-test create configmap test-server-config --from-file=test/benchmark/config/nighthawk-test-server-config.yaml + kubectl apply -f test/benchmark/config/nighthawk-test-server.yaml .PHONY: install-e2e-telemetry install-e2e-telemetry: prepare-helm-repo install-fluent-bit install-loki install-tempo install-otel-collector install-prometheus From 6ea7c9110287412e165cd4c4280e72edafaa9fe8 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Fri, 14 Jun 2024 16:23:40 +0800 Subject: [PATCH 03/30] add benchmark in ci and spawn a job for benchmark test run Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 41 ++++++ test/benchmark/benchmark_test.go | 1 - test/benchmark/config/nighthawk-client.yaml | 2 + test/benchmark/suite/flags.go | 9 +- test/benchmark/suite/suite.go | 130 ++++++++++++++++---- test/benchmark/suite/test.go | 22 ++-- test/benchmark/tests/scale_httproutes.go | 38 +++--- tools/make/kube.mk | 2 +- 8 files changed, 183 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/benchmark.yaml diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml new file mode 100644 index 00000000000..9ef9f064022 --- /dev/null +++ b/.github/workflows/benchmark.yaml @@ -0,0 +1,41 @@ +name: Benchmarking Tests at Scale +on: + pull_request: + branches: + - "main" + - "release/v*" + workflow_dispatch: + inputs: + rps: + description: "The target requests-per-second rate. Default: 1000" + default: '1000' + type: string + required: false + connections: + description: "The maximum allowed number of concurrent connections per event loop. HTTP/1 only. Default: 100." + default: '100' + type: string + required: false + duration: + description: "The number of seconds that the test should run. Default: 90." + default: '90' + type: string + required: false + +jobs: + benchmark-test: + name: Benchmark Test + runs-on: ubuntu-latest + needs: [build] + steps: + - uses: actions/checkout@v2 + - uses: ./tools/github-actions/setup-deps + + - name: Run Benchmark tests + env: + KIND_NODE_TAG: v1.28.0 + IMAGE_PULL_POLICY: IfNotPresent + RPS: ${{ github.event.inputs.rps || 1000 }} + CONNECTIONS: ${{ github.event.inputs.connections || 100 }} + DURATION: ${{ github.event.inputs.duration || 90 }} + run: make benchmark diff --git a/test/benchmark/benchmark_test.go b/test/benchmark/benchmark_test.go index 48c470cde5b..052bcf40a2c 100644 --- a/test/benchmark/benchmark_test.go +++ b/test/benchmark/benchmark_test.go @@ -37,7 +37,6 @@ func TestBenchmark(t *testing.T) { *suite.Connections, *suite.Duration, *suite.Concurrency, - *suite.PrefetchConnections, ) bSuite, err := suite.NewBenchmarkTestSuite( diff --git a/test/benchmark/config/nighthawk-client.yaml b/test/benchmark/config/nighthawk-client.yaml index a9621d3fb72..3f2b3b7c2b8 100644 --- a/test/benchmark/config/nighthawk-client.yaml +++ b/test/benchmark/config/nighthawk-client.yaml @@ -4,6 +4,8 @@ kind: Job metadata: name: "{NIGHTHAWK_CLIENT_NAME}" namespace: benchmark-test + labels: + benchmark-test/client: true spec: template: spec: diff --git a/test/benchmark/suite/flags.go b/test/benchmark/suite/flags.go index 7238917226a..5b209a3392f 100644 --- a/test/benchmark/suite/flags.go +++ b/test/benchmark/suite/flags.go @@ -11,9 +11,8 @@ package suite import "flag" var ( - RPS = flag.String("rps", "1000", "RPS of benchmark test client") - Connections = flag.String("connections", "10", "Connections of benchmark test client") - Duration = flag.String("duration", "60", "Duration of benchmark test client") - Concurrency = flag.String("concurrency", "auto", "Concurrency of benchmark test client") - PrefetchConnections = flag.Bool("prefetch-connections", true, "PrefetchConnections of benchmark test client") + RPS = flag.String("rps", "1000", "The target requests-per-second rate.") + Connections = flag.String("connections", "10", "The maximum allowed number of concurrent connections per event loop. HTTP/1 only.") + Duration = flag.String("duration", "60", "The number of seconds that the test should run.") + Concurrency = flag.String("concurrency", "auto", "The number of concurrent event loops that should be used.") ) diff --git a/test/benchmark/suite/suite.go b/test/benchmark/suite/suite.go index 9f0cc8cea16..5b3a4ce84eb 100644 --- a/test/benchmark/suite/suite.go +++ b/test/benchmark/suite/suite.go @@ -12,10 +12,14 @@ import ( "context" "fmt" "os" + "strconv" "testing" + "time" batchv1 "k8s.io/api/batch/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/gateway-api/conformance/utils/config" @@ -23,8 +27,9 @@ import ( ) const ( - ScaledLabelKey = "benchmark-test/scaled" - DefaultControllerName = "gateway.envoyproxy.io/gatewayclass-controller" + BenchmarkTestScaledKey = "benchmark-test/scaled" + BenchmarkTestClientKey = "benchmark-test/client" + DefaultControllerName = "gateway.envoyproxy.io/gatewayclass-controller" ) type BenchmarkTestSuite struct { @@ -80,9 +85,9 @@ func NewBenchmarkTestSuite(client client.Client, options BenchmarkOptions, config.SetupTimeoutConfig(&timeoutConfig) // Prepare static options for benchmark client. - srcArgs := prepareBenchmarkClientStaticArgs(options) - dstArgs := benchmarkClient.Spec.Template.Spec.Containers[0].Args - dstArgs = append(dstArgs, srcArgs...) + staticArgs := prepareBenchmarkClientStaticArgs(options) + container := &benchmarkClient.Spec.Template.Spec.Containers[0] + container.Args = append(container.Args, staticArgs...) return &BenchmarkTestSuite{ Client: client, @@ -93,7 +98,7 @@ func NewBenchmarkTestSuite(client client.Client, options BenchmarkOptions, HTTPRouteTemplate: httproute, BenchmarkClient: benchmarkClient, scaledLabel: map[string]string{ - ScaledLabelKey: "true", + BenchmarkTestScaledKey: "true", }, }, nil } @@ -113,12 +118,64 @@ func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { // TODO: currently running benchmark test via nighthawk-client, // consider switching to gRPC nighthawk-service for benchmark test. // ref: https://github.com/envoyproxy/nighthawk/blob/main/api/client/service.proto -func (b *BenchmarkTestSuite) Benchmark() { - // TODO: - // 1. prepare job - // 2. create and run job - // 3. wait job complete - // 4. scrap job log as report +func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, gatewayHostPort string, requestHeaders ...string) error { + t.Helper() + + t.Logf("Running benchmark test: %s", name) + + jobNN, err := b.createBenchmarkClientJob(ctx, name, gatewayHostPort, requestHeaders...) + if err != nil { + return err + } + + duration, err := strconv.ParseInt(b.Options.Duration, 10, 64) + if err != nil { + return err + } + + // Wait from benchmark test job to complete. + if err = wait.PollUntilContextTimeout(ctx, 5*time.Second, time.Duration(duration*5)*time.Second, true, func(ctx context.Context) (bool, error) { + job := new(batchv1.Job) + if err = b.Client.Get(ctx, *jobNN, job); err != nil { + return false, err + } + + for _, condition := range job.Status.Conditions { + if condition.Type == batchv1.JobComplete && condition.Status == "true" { + return true, nil + } + + // Early return if job already failed. + if condition.Type == batchv1.JobFailed && condition.Status == "true" && + condition.Reason == batchv1.JobReasonBackoffLimitExceeded { + return false, fmt.Errorf("job alreay failed") + } + } + + t.Logf("Job %s still not complete", name) + return false, nil + }); err != nil { + t.Errorf("Failed to run benchmark test: %v", err) + return err + } + + t.Logf("Running benchmark test: %s successfully", name) + return nil +} + +func (b *BenchmarkTestSuite) createBenchmarkClientJob(ctx context.Context, name, gatewayHostPort string, requestHeaders ...string) (*types.NamespacedName, error) { + job := b.BenchmarkClient.DeepCopy() + job.SetName(name) + + runtimeArgs := prepareBenchmarkClientRuntimeArgs(gatewayHostPort, requestHeaders...) + container := &job.Spec.Template.Spec.Containers[0] + container.Args = append(container.Args, runtimeArgs...) + + if err := b.CreateResource(ctx, job); err != nil { + return nil, err + } + + return &types.NamespacedName{Name: job.Name, Namespace: job.Namespace}, nil } func prepareBenchmarkClientStaticArgs(options BenchmarkOptions) []string { @@ -128,33 +185,32 @@ func prepareBenchmarkClientStaticArgs(options BenchmarkOptions) []string { "--duration", options.Duration, "--concurrency", options.Concurrency, } - if options.PrefetchConnections { - staticArgs = append(staticArgs, "--prefetch-connections") - } return staticArgs } -func prepareBenchmarkClientArgs(gatewayHost string, requestHeaders ...string) []string { +func prepareBenchmarkClientRuntimeArgs(gatewayHostPort string, requestHeaders ...string) []string { args := make([]string, 0, len(requestHeaders)*2+1) for _, reqHeader := range requestHeaders { args = append(args, "--request-header", reqHeader) } - args = append(args, gatewayHost) + args = append(args, fmt.Sprintf("http://%s/", gatewayHostPort)) return args } // ScaleHTTPRoutes scales HTTPRoutes that are all referenced to one Gateway according to // the scale range: (begin, end]. The afterCreation is a callback function that only runs -// everytime after one HTTPRoutes has been created successfully. +// every time after one HTTPRoutes has been created successfully. // -// All scaled resources will be labeled with ScaledLabelKey. -func (b *BenchmarkTestSuite) ScaleHTTPRoutes(ctx context.Context, scaleRange [2]uint16, targetName, refGateway string, afterCreation func(route *gwapiv1.HTTPRoute)) error { - var ( - i uint16 - begin, end = scaleRange[0], scaleRange[1] - ) +// All scaled resources will be labeled with BenchmarkTestScaledKey. +func (b *BenchmarkTestSuite) ScaleHTTPRoutes(t *testing.T, ctx context.Context, scaleRange [2]uint16, targetName, refGateway string, afterCreation func(route *gwapiv1.HTTPRoute)) error { + t.Helper() + + var i, begin, end uint16 + begin, end = scaleRange[0], scaleRange[1] + + t.Logf("Scaling HTTPRoutes to %d", end) for i = begin + 1; i <= end; i++ { routeName := fmt.Sprintf(targetName, i) @@ -172,6 +228,8 @@ func (b *BenchmarkTestSuite) ScaleHTTPRoutes(ctx context.Context, scaleRange [2] } } + t.Logf("Finish scaling HTTPRoutes to %d", end) + return nil } @@ -186,6 +244,17 @@ func (b *BenchmarkTestSuite) CreateResource(ctx context.Context, object client.O return nil } +// RegisterCleanup registers cleanup functions for all benchmark test resources. +func (b *BenchmarkTestSuite) RegisterCleanup(t *testing.T, ctx context.Context, object, scaledObject client.Object) { + t.Cleanup(func() { + t.Logf("Start to cleanup benchmark test resources") + + _ = b.CleanupResource(ctx, object) + _ = b.CleanupScaledResources(ctx, scaledObject) + _ = b.CleanupBenchmarkClientJobs(ctx) + }) +} + func (b *BenchmarkTestSuite) CleanupResource(ctx context.Context, object client.Object) error { if err := b.Client.Delete(ctx, object); err != nil { if !kerrors.IsNotFound(err) { @@ -197,10 +266,19 @@ func (b *BenchmarkTestSuite) CleanupResource(ctx context.Context, object client. return nil } -// CleanupScaledResources only cleanups all the resources with Scaled label under benchmark-test namespace. +// CleanupScaledResources only cleanups all the resources under benchmark-test namespace. func (b *BenchmarkTestSuite) CleanupScaledResources(ctx context.Context, object client.Object) error { if err := b.Client.DeleteAllOf(ctx, object, - client.MatchingLabels{ScaledLabelKey: "true"}, client.InNamespace("benchmark-test")); err != nil { + client.MatchingLabels{BenchmarkTestScaledKey: "true"}, client.InNamespace("benchmark-test")); err != nil { + return err + } + return nil +} + +// CleanupBenchmarkClientJobs only cleanups all the jobs under benchmark-test namespace. +func (b *BenchmarkTestSuite) CleanupBenchmarkClientJobs(ctx context.Context) error { + if err := b.Client.DeleteAllOf(ctx, &batchv1.Job{}, + client.MatchingLabels{BenchmarkTestClientKey: "true"}, client.InNamespace("benchmark-test")); err != nil { return err } return nil diff --git a/test/benchmark/suite/test.go b/test/benchmark/suite/test.go index 9e916b1b4d9..24d0ef8c1bb 100644 --- a/test/benchmark/suite/test.go +++ b/test/benchmark/suite/test.go @@ -18,21 +18,17 @@ type BenchmarkTest struct { // BenchmarkOptions for nighthawk-client. type BenchmarkOptions struct { - RPS string - Connections string - Duration string - Concurrency string - PrefetchConnections bool + RPS string + Connections string + Duration string + Concurrency string } -func NewBenchmarkOptions( - rps, connections, duration, concurrency string, prefetchConnections bool, -) BenchmarkOptions { +func NewBenchmarkOptions(rps, connections, duration, concurrency string) BenchmarkOptions { return BenchmarkOptions{ - RPS: rps, - Connections: connections, - Duration: duration, - Concurrency: concurrency, - PrefetchConnections: prefetchConnections, + RPS: rps, + Connections: connections, + Duration: duration, + Concurrency: concurrency, } } diff --git a/test/benchmark/tests/scale_httproutes.go b/test/benchmark/tests/scale_httproutes.go index 83b73962fad..bc77b6b2a95 100644 --- a/test/benchmark/tests/scale_httproutes.go +++ b/test/benchmark/tests/scale_httproutes.go @@ -10,13 +10,15 @@ package tests import ( "context" + "fmt" "testing" - "github.com/envoyproxy/gateway/test/benchmark/suite" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/types" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" + + "github.com/envoyproxy/gateway/test/benchmark/suite" ) func init() { @@ -25,12 +27,16 @@ func init() { var ScaleHTTPRoutes = suite.BenchmarkTest{ ShortName: "ScaleHTTPRoute", - Description: "Fixed one Gateway with different scale of HTTPRoutes: [10, 50, 100, 300, 500].", + Description: "Fixed one Gateway and different number of HTTPRoutes on a scale of (10, 50, 100, 300, 500).", Test: func(t *testing.T, suite *suite.BenchmarkTestSuite) { var ( - ctx = context.Background() - ns = "benchmark-test" - err error + ctx = context.Background() + ns = "benchmark-test" + err error + requestHeaders = []string{ + "Host: www.benchmark.com", + "Host: x-nighthawk-test-server-config: {response_body_size:20, static_delay:\"0s\"}", + } ) // Setup and create a gateway. @@ -42,30 +48,30 @@ var ScaleHTTPRoutes = suite.BenchmarkTest{ // Setup and scale httproutes. httpRouteName := "benchmark-route-%d" - httpRouteScales := []uint16{5, 10, 20} + httpRouteScales := []uint16{5, 10, 20} // TODO: for test, update later httpRouteNNs := make([]types.NamespacedName, 0, httpRouteScales[len(httpRouteScales)-1]) + suite.RegisterCleanup(t, ctx, gateway, &gwapiv1.HTTPRoute{}) + var start uint16 = 0 for _, scale := range httpRouteScales { t.Logf("Scaling HTTPRoutes to %d", scale) - err = suite.ScaleHTTPRoutes(ctx, [2]uint16{start, scale}, httpRouteName, gatewayNN.Name, func(route *gwapiv1.HTTPRoute) { + err = suite.ScaleHTTPRoutes(t, ctx, [2]uint16{start, scale}, httpRouteName, gatewayNN.Name, func(route *gwapiv1.HTTPRoute) { routeNN := types.NamespacedName{Name: route.Name, Namespace: route.Namespace} httpRouteNNs = append(httpRouteNNs, routeNN) }) require.NoError(t, err) start = scale - kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gatewayNN), httpRouteNNs...) - - // TODO: run benchmark and report - } + gatewayAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gatewayNN), httpRouteNNs...) - // Cleanup - err = suite.CleanupResource(ctx, gateway) - require.NoError(t, err) + // Run benchmark test at different scale. + name := fmt.Sprintf("scale-httproutes-%d", scale) + err = suite.Benchmark(t, ctx, name, gatewayAddr, requestHeaders...) + require.NoError(t, err) - err = suite.CleanupScaledResources(ctx, &gwapiv1.HTTPRoute{}) - require.NoError(t, err) + // TODO: Scrape the benchmark results. + } }, } diff --git a/tools/make/kube.mk b/tools/make/kube.mk index be884e863ca..e850c81f946 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -147,7 +147,7 @@ run-benchmark: install-benchmark-server ## Run benchmark tests kubectl wait --timeout=$(WAIT_TIMEOUT) -n benchmark-test deployment/nighthawk-test-server --for=condition=Available kubectl wait --timeout=$(WAIT_TIMEOUT) -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available kubectl apply -f test/benchmark/config/gatewayclass.yaml - go test -v -tags benchmark ./test/benchmark + go test -v -tags benchmark ./test/benchmark --rps=$RPS --connections $CONNECTIONS --duration $DURATION .PHONY: install-benchmark-server install-benchmark-server: ## Install nighthawk server for benchmark test From c2cfb96aecc14d5b7b68b39e7fd5a6c43b47ab09 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Fri, 14 Jun 2024 16:29:34 +0800 Subject: [PATCH 04/30] fix lint Signed-off-by: shawnh2 --- test/benchmark/suite/suite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmark/suite/suite.go b/test/benchmark/suite/suite.go index 5b3a4ce84eb..88d2bca369c 100644 --- a/test/benchmark/suite/suite.go +++ b/test/benchmark/suite/suite.go @@ -148,7 +148,7 @@ func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, // Early return if job already failed. if condition.Type == batchv1.JobFailed && condition.Status == "true" && condition.Reason == batchv1.JobReasonBackoffLimitExceeded { - return false, fmt.Errorf("job alreay failed") + return false, fmt.Errorf("job already failed") } } From 83abe3fe4a69a9bda3b3d0d5e3ffb2e34c369221 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Fri, 14 Jun 2024 17:03:36 +0800 Subject: [PATCH 05/30] fix ci config Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 9ef9f064022..b29dac2e487 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -26,7 +26,6 @@ jobs: benchmark-test: name: Benchmark Test runs-on: ubuntu-latest - needs: [build] steps: - uses: actions/checkout@v2 - uses: ./tools/github-actions/setup-deps From bb8f6eed458090f626f27989fa15a8109f900251 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Sat, 15 Jun 2024 13:32:25 +0800 Subject: [PATCH 06/30] save benchmark test result into report Signed-off-by: shawnh2 --- test/benchmark/config/nighthawk-client.yaml | 2 +- .../config/nighthawk-test-server-config.yaml | 2 +- .../config/nighthawk-test-server.yaml | 2 +- test/benchmark/suite/report.go | 85 +++++++++++++++++++ test/benchmark/suite/suite.go | 47 ++++++---- test/benchmark/suite/test.go | 2 +- test/benchmark/tests/scale_httproutes.go | 22 ++--- tools/make/kube.mk | 4 +- 8 files changed, 132 insertions(+), 34 deletions(-) create mode 100644 test/benchmark/suite/report.go diff --git a/test/benchmark/config/nighthawk-client.yaml b/test/benchmark/config/nighthawk-client.yaml index 3f2b3b7c2b8..90375d8c05c 100644 --- a/test/benchmark/config/nighthawk-client.yaml +++ b/test/benchmark/config/nighthawk-client.yaml @@ -5,7 +5,7 @@ metadata: name: "{NIGHTHAWK_CLIENT_NAME}" namespace: benchmark-test labels: - benchmark-test/client: true + benchmark-test/client: "true" spec: template: spec: diff --git a/test/benchmark/config/nighthawk-test-server-config.yaml b/test/benchmark/config/nighthawk-test-server-config.yaml index fc63a00b18c..f8e69f6f1cb 100644 --- a/test/benchmark/config/nighthawk-test-server-config.yaml +++ b/test/benchmark/config/nighthawk-test-server-config.yaml @@ -23,7 +23,7 @@ static_resources: - name: dynamic-delay typed_config: "@type": type.googleapis.com/nighthawk.server.DynamicDelayConfiguration - static_delay: 0.5s + static_delay: 0s - name: test-server # before envoy.router because order matters! typed_config: "@type": type.googleapis.com/nighthawk.server.ResponseOptions diff --git a/test/benchmark/config/nighthawk-test-server.yaml b/test/benchmark/config/nighthawk-test-server.yaml index 1165789e1f7..dfd91aae464 100644 --- a/test/benchmark/config/nighthawk-test-server.yaml +++ b/test/benchmark/config/nighthawk-test-server.yaml @@ -19,7 +19,7 @@ spec: - name: nighthawk-server image: envoyproxy/nighthawk-dev:latest imagePullPolicy: IfNotPresent - args: ["nighthawk_test_server", "-c", "/etc/test-server-config/test-server-config.yaml"] + args: ["nighthawk_test_server", "-c", "/etc/test-server-config/nighthawk-test-server-config.yaml"] ports: - containerPort: 8080 volumeMounts: diff --git a/test/benchmark/suite/report.go b/test/benchmark/suite/report.go new file mode 100644 index 00000000000..bb1fa6d8930 --- /dev/null +++ b/test/benchmark/suite/report.go @@ -0,0 +1,85 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package suite + +import ( + "bytes" + "context" + "io" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client/config" +) + +type BenchmarkReport struct { + RawResult []byte +} + +func NewBenchmarkReport() *BenchmarkReport { + return &BenchmarkReport{} +} + +func (r *BenchmarkReport) Print(t *testing.T, name string) { + t.Logf("The report of benchmark test: %s", name) + + t.Logf("=== Benchmark Result: \n %s \n", string(r.RawResult)) +} + +func (r *BenchmarkReport) GetResultFromJob(t *testing.T, ctx context.Context, job *types.NamespacedName) error { + cfg, err := config.GetConfig() + if err != nil { + return err + } + + clientset, err := kubernetes.NewForConfig(cfg) + if err != nil { + return err + } + + pods, err := clientset.CoreV1().Pods(job.Namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + job.Name}) + if len(pods.Items) > 1 { + t.Logf("Got %d pod(s) associated job %s, should be 1 pod, could be pod err and job backoff then restart, please check your pod(s) status", + len(pods.Items), job.Name) + } + + pod := &pods.Items[0] + if err = r.getResultFromPodLogs(ctx, clientset, + &types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}); err != nil { + return err + } + + return nil +} + +// getResultFromPodLogs scrapes the logs directly from the pod (default container) +// and save it as the raw result in benchmark report. +func (r *BenchmarkReport) getResultFromPodLogs(ctx context.Context, clientset *kubernetes.Clientset, pod *types.NamespacedName) error { + podLogOpts := corev1.PodLogOptions{} + + req := clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) + podLogs, err := req.Stream(ctx) + if err != nil { + return err + } + + defer podLogs.Close() + + buf := new(bytes.Buffer) + _, err = io.Copy(buf, podLogs) + if err != nil { + return err + } + + r.RawResult = buf.Bytes() + return nil +} diff --git a/test/benchmark/suite/suite.go b/test/benchmark/suite/suite.go index 88d2bca369c..ae0b9ee686b 100644 --- a/test/benchmark/suite/suite.go +++ b/test/benchmark/suite/suite.go @@ -17,6 +17,7 @@ import ( "time" batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -109,44 +110,43 @@ func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { for _, test := range tests { t.Logf("Running benchmark test: %s", test.ShortName) - test.Test(t, b) + // TODO: generate a benchmark report for human + _ = test.Test(t, b) } } -// Benchmark prepares and runs benchmark test as a Kubernetes Job. +// Benchmark runs benchmark test as a Kubernetes Job, and return the benchmark result. // -// TODO: currently running benchmark test via nighthawk-client, +// TODO: currently running benchmark test via nighthawk_client, // consider switching to gRPC nighthawk-service for benchmark test. // ref: https://github.com/envoyproxy/nighthawk/blob/main/api/client/service.proto -func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, gatewayHostPort string, requestHeaders ...string) error { - t.Helper() - +func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, gatewayHostPort string, requestHeaders ...string) (*BenchmarkReport, error) { t.Logf("Running benchmark test: %s", name) jobNN, err := b.createBenchmarkClientJob(ctx, name, gatewayHostPort, requestHeaders...) if err != nil { - return err + return nil, err } duration, err := strconv.ParseInt(b.Options.Duration, 10, 64) if err != nil { - return err + return nil, err } // Wait from benchmark test job to complete. - if err = wait.PollUntilContextTimeout(ctx, 5*time.Second, time.Duration(duration*5)*time.Second, true, func(ctx context.Context) (bool, error) { + if err = wait.PollUntilContextTimeout(ctx, 10*time.Second, time.Duration(duration*3)*time.Second, true, func(ctx context.Context) (bool, error) { job := new(batchv1.Job) if err = b.Client.Get(ctx, *jobNN, job); err != nil { return false, err } for _, condition := range job.Status.Conditions { - if condition.Type == batchv1.JobComplete && condition.Status == "true" { + if condition.Type == batchv1.JobComplete && condition.Status == "True" { return true, nil } // Early return if job already failed. - if condition.Type == batchv1.JobFailed && condition.Status == "true" && + if condition.Type == batchv1.JobFailed && condition.Status == "True" && condition.Reason == batchv1.JobReasonBackoffLimitExceeded { return false, fmt.Errorf("job already failed") } @@ -156,11 +156,17 @@ func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, return false, nil }); err != nil { t.Errorf("Failed to run benchmark test: %v", err) - return err + return nil, err } t.Logf("Running benchmark test: %s successfully", name) - return nil + + report := NewBenchmarkReport() + if err = report.GetResultFromJob(t, ctx, jobNN); err != nil { + return nil, err + } + + return report, nil } func (b *BenchmarkTestSuite) createBenchmarkClientJob(ctx context.Context, name, gatewayHostPort string, requestHeaders ...string) (*types.NamespacedName, error) { @@ -194,7 +200,7 @@ func prepareBenchmarkClientRuntimeArgs(gatewayHostPort string, requestHeaders .. for _, reqHeader := range requestHeaders { args = append(args, "--request-header", reqHeader) } - args = append(args, fmt.Sprintf("http://%s/", gatewayHostPort)) + args = append(args, "http://"+gatewayHostPort) return args } @@ -251,7 +257,8 @@ func (b *BenchmarkTestSuite) RegisterCleanup(t *testing.T, ctx context.Context, _ = b.CleanupResource(ctx, object) _ = b.CleanupScaledResources(ctx, scaledObject) - _ = b.CleanupBenchmarkClientJobs(ctx) + + t.Logf("Clean up complete!") }) } @@ -275,11 +282,17 @@ func (b *BenchmarkTestSuite) CleanupScaledResources(ctx context.Context, object return nil } -// CleanupBenchmarkClientJobs only cleanups all the jobs under benchmark-test namespace. -func (b *BenchmarkTestSuite) CleanupBenchmarkClientJobs(ctx context.Context) error { +// CleanupBenchmarkClientJobs only cleanups all the jobs and its associated pods under benchmark-test namespace. +func (b *BenchmarkTestSuite) CleanupBenchmarkClientJobs(ctx context.Context, name string) error { if err := b.Client.DeleteAllOf(ctx, &batchv1.Job{}, client.MatchingLabels{BenchmarkTestClientKey: "true"}, client.InNamespace("benchmark-test")); err != nil { return err } + + if err := b.Client.DeleteAllOf(ctx, &corev1.Pod{}, + client.MatchingLabels{"job-name": name}, client.InNamespace("benchmark-test")); err != nil { + return err + } + return nil } diff --git a/test/benchmark/suite/test.go b/test/benchmark/suite/test.go index 24d0ef8c1bb..112e1b922f1 100644 --- a/test/benchmark/suite/test.go +++ b/test/benchmark/suite/test.go @@ -13,7 +13,7 @@ import "testing" type BenchmarkTest struct { ShortName string Description string - Test func(*testing.T, *BenchmarkTestSuite) + Test func(*testing.T, *BenchmarkTestSuite) []BenchmarkReport } // BenchmarkOptions for nighthawk-client. diff --git a/test/benchmark/tests/scale_httproutes.go b/test/benchmark/tests/scale_httproutes.go index bc77b6b2a95..ccb3e816c54 100644 --- a/test/benchmark/tests/scale_httproutes.go +++ b/test/benchmark/tests/scale_httproutes.go @@ -28,22 +28,21 @@ func init() { var ScaleHTTPRoutes = suite.BenchmarkTest{ ShortName: "ScaleHTTPRoute", Description: "Fixed one Gateway and different number of HTTPRoutes on a scale of (10, 50, 100, 300, 500).", - Test: func(t *testing.T, suite *suite.BenchmarkTestSuite) { + Test: func(t *testing.T, bSuite *suite.BenchmarkTestSuite) (reports []suite.BenchmarkReport) { var ( ctx = context.Background() ns = "benchmark-test" err error requestHeaders = []string{ "Host: www.benchmark.com", - "Host: x-nighthawk-test-server-config: {response_body_size:20, static_delay:\"0s\"}", } ) // Setup and create a gateway. gatewayNN := types.NamespacedName{Name: "benchmark", Namespace: ns} - gateway := suite.GatewayTemplate.DeepCopy() + gateway := bSuite.GatewayTemplate.DeepCopy() gateway.SetName(gatewayNN.Name) - err = suite.CreateResource(ctx, gateway) + err = bSuite.CreateResource(ctx, gateway) require.NoError(t, err) // Setup and scale httproutes. @@ -51,27 +50,28 @@ var ScaleHTTPRoutes = suite.BenchmarkTest{ httpRouteScales := []uint16{5, 10, 20} // TODO: for test, update later httpRouteNNs := make([]types.NamespacedName, 0, httpRouteScales[len(httpRouteScales)-1]) - suite.RegisterCleanup(t, ctx, gateway, &gwapiv1.HTTPRoute{}) + bSuite.RegisterCleanup(t, ctx, gateway, &gwapiv1.HTTPRoute{}) var start uint16 = 0 for _, scale := range httpRouteScales { - t.Logf("Scaling HTTPRoutes to %d", scale) - - err = suite.ScaleHTTPRoutes(t, ctx, [2]uint16{start, scale}, httpRouteName, gatewayNN.Name, func(route *gwapiv1.HTTPRoute) { + err = bSuite.ScaleHTTPRoutes(t, ctx, [2]uint16{start, scale}, httpRouteName, gatewayNN.Name, func(route *gwapiv1.HTTPRoute) { routeNN := types.NamespacedName{Name: route.Name, Namespace: route.Namespace} httpRouteNNs = append(httpRouteNNs, routeNN) }) require.NoError(t, err) start = scale - gatewayAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gatewayNN), httpRouteNNs...) + gatewayAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, bSuite.Client, bSuite.TimeoutConfig, bSuite.ControllerName, kubernetes.NewGatewayRef(gatewayNN), httpRouteNNs...) // Run benchmark test at different scale. name := fmt.Sprintf("scale-httproutes-%d", scale) - err = suite.Benchmark(t, ctx, name, gatewayAddr, requestHeaders...) + report, err := bSuite.Benchmark(t, ctx, name, gatewayAddr, requestHeaders...) require.NoError(t, err) - // TODO: Scrape the benchmark results. + report.Print(t, name) + reports = append(reports, *report) } + + return }, } diff --git a/tools/make/kube.mk b/tools/make/kube.mk index e850c81f946..57e2496c26b 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -147,13 +147,13 @@ run-benchmark: install-benchmark-server ## Run benchmark tests kubectl wait --timeout=$(WAIT_TIMEOUT) -n benchmark-test deployment/nighthawk-test-server --for=condition=Available kubectl wait --timeout=$(WAIT_TIMEOUT) -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available kubectl apply -f test/benchmark/config/gatewayclass.yaml - go test -v -tags benchmark ./test/benchmark --rps=$RPS --connections $CONNECTIONS --duration $DURATION + go test -v -tags benchmark ./test/benchmark --rps=$(RPS) --connections=$(CONNECTIONS) --duration=$(DURATION) .PHONY: install-benchmark-server install-benchmark-server: ## Install nighthawk server for benchmark test @$(LOG_TARGET) kubectl create namespace benchmark-test - kubectl -n benchmark-test create configmap test-server-config --from-file=test/benchmark/config/nighthawk-test-server-config.yaml + kubectl -n benchmark-test create configmap test-server-config --from-file=test/benchmark/config/nighthawk-test-server-config.yaml -o yaml kubectl apply -f test/benchmark/config/nighthawk-test-server.yaml .PHONY: install-e2e-telemetry From c9c6be7c247dcbdc44759d4f77f626f57f369828 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Sun, 16 Jun 2024 17:08:17 +0800 Subject: [PATCH 07/30] add control-plane metrics to benchmark test report Signed-off-by: shawnh2 --- test/benchmark/suite/report.go | 114 +++++++++++++++++++++++++++------ test/benchmark/suite/suite.go | 14 +++- 2 files changed, 105 insertions(+), 23 deletions(-) diff --git a/test/benchmark/suite/report.go b/test/benchmark/suite/report.go index bb1fa6d8930..7486f0883e2 100644 --- a/test/benchmark/suite/report.go +++ b/test/benchmark/suite/report.go @@ -11,62 +11,100 @@ package suite import ( "bytes" "context" + "fmt" "io" + "net/http" "testing" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes" - "sigs.k8s.io/controller-runtime/pkg/client/config" + + "github.com/envoyproxy/gateway/internal/cmd/options" + kube "github.com/envoyproxy/gateway/internal/kubernetes" +) + +const ( + controlPlaneMetricsPort = 19001 ) type BenchmarkReport struct { - RawResult []byte + RawResult []byte + RawMetrics []byte + + kubeClient kube.CLIClient } -func NewBenchmarkReport() *BenchmarkReport { - return &BenchmarkReport{} +func NewBenchmarkReport() (*BenchmarkReport, error) { + kubeClient, err := kube.NewCLIClient(options.DefaultConfigFlags.ToRawKubeConfigLoader()) + if err != nil { + return nil, err + } + + return &BenchmarkReport{ + kubeClient: kubeClient, + }, nil } func (r *BenchmarkReport) Print(t *testing.T, name string) { t.Logf("The report of benchmark test: %s", name) - t.Logf("=== Benchmark Result: \n %s \n", string(r.RawResult)) + t.Logf("=== Benchmark Result: \n\n %s \n\n", r.RawResult) + t.Logf("=== Control-Plane Metrics: \n\n %s \n\n", r.RawMetrics) } -func (r *BenchmarkReport) GetResultFromJob(t *testing.T, ctx context.Context, job *types.NamespacedName) error { - cfg, err := config.GetConfig() - if err != nil { - return err - } +func (r *BenchmarkReport) GetBenchmarkResult(t *testing.T, ctx context.Context, job *types.NamespacedName) error { + pods, err := r.kubeClient.Kube().CoreV1().Pods(job.Namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + job.Name}) - clientset, err := kubernetes.NewForConfig(cfg) - if err != nil { - return err + if len(pods.Items) < 1 { + return fmt.Errorf("failed to get any pods for job %s", job.String()) } - pods, err := clientset.CoreV1().Pods(job.Namespace).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + job.Name}) if len(pods.Items) > 1 { t.Logf("Got %d pod(s) associated job %s, should be 1 pod, could be pod err and job backoff then restart, please check your pod(s) status", len(pods.Items), job.Name) } pod := &pods.Items[0] - if err = r.getResultFromPodLogs(ctx, clientset, - &types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}); err != nil { + if err = r.getBenchmarkResultFromPodLogs( + ctx, &types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, + ); err != nil { return err } return nil } -// getResultFromPodLogs scrapes the logs directly from the pod (default container) +func (r *BenchmarkReport) GetControlPlaneMetrics(t *testing.T, ctx context.Context) error { + egPods, err := r.kubeClient.Kube().CoreV1().Pods("envoy-gateway-system").List(ctx, metav1.ListOptions{LabelSelector: "control-plane=envoy-gateway"}) + if err != nil { + return err + } + + if len(egPods.Items) < 1 { + return fmt.Errorf("failed to get any pods for envoy-gateway") + } + + if len(egPods.Items) > 1 { + t.Logf("Got %d pod(s), using the first one as default envoy-gateway pod", len(egPods.Items)) + } + + egPod := &egPods.Items[0] + if err = r.getMetricsFromPodPortForwarder( + t, &types.NamespacedName{Name: egPod.Name, Namespace: egPod.Namespace}, + ); err != nil { + return err + } + + return nil +} + +// getBenchmarkResultFromPodLogs scrapes the logs directly from the pod (default container) // and save it as the raw result in benchmark report. -func (r *BenchmarkReport) getResultFromPodLogs(ctx context.Context, clientset *kubernetes.Clientset, pod *types.NamespacedName) error { +func (r *BenchmarkReport) getBenchmarkResultFromPodLogs(ctx context.Context, pod *types.NamespacedName) error { podLogOpts := corev1.PodLogOptions{} - req := clientset.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) + req := r.kubeClient.Kube().CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) podLogs, err := req.Stream(ctx) if err != nil { return err @@ -83,3 +121,39 @@ func (r *BenchmarkReport) getResultFromPodLogs(ctx context.Context, clientset *k r.RawResult = buf.Bytes() return nil } + +func (r *BenchmarkReport) getMetricsFromPodPortForwarder(t *testing.T, pod *types.NamespacedName) error { + fw, err := kube.NewLocalPortForwarder(r.kubeClient, *pod, controlPlaneMetricsPort, controlPlaneMetricsPort) + if err != nil { + return fmt.Errorf("failed to build port forwarder for pod %s: %v", pod.String(), err) + } + + if err = fw.Start(); err != nil { + fw.Stop() + return fmt.Errorf("failed to start port forwarder for pod %s: %v", pod.String(), err) + } + + // Retrieving metrics from Pod. + go func() { + defer fw.Stop() + + url := fmt.Sprintf("http://%s/metrics", fw.Address()) + resp, err := http.Get(url) + if err != nil { + t.Errorf("failed to request %s: %v", url, err) + return + } + + metrics, err := io.ReadAll(resp.Body) + if err != nil { + t.Errorf("failed to read metrics: %v", err) + return + } + + r.RawMetrics = metrics + }() + + fw.WaitForStop() + + return nil +} diff --git a/test/benchmark/suite/suite.go b/test/benchmark/suite/suite.go index ae0b9ee686b..28d820c1167 100644 --- a/test/benchmark/suite/suite.go +++ b/test/benchmark/suite/suite.go @@ -110,7 +110,7 @@ func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { for _, test := range tests { t.Logf("Running benchmark test: %s", test.ShortName) - // TODO: generate a benchmark report for human + // TODO: generate a readable benchmark report for human _ = test.Test(t, b) } } @@ -161,8 +161,16 @@ func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, t.Logf("Running benchmark test: %s successfully", name) - report := NewBenchmarkReport() - if err = report.GetResultFromJob(t, ctx, jobNN); err != nil { + report, err := NewBenchmarkReport() + if err != nil { + return nil, err + } + + // Get all the reports from this benchmark test run. + if err = report.GetBenchmarkResult(t, ctx, jobNN); err != nil { + return nil, err + } + if err = report.GetControlPlaneMetrics(t, ctx); err != nil { return nil, err } From d4710a37a89471cac416e364afc8eaee97f2bcf5 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Sun, 16 Jun 2024 17:10:30 +0800 Subject: [PATCH 08/30] change httproutes scale number to perform the benchmark test Signed-off-by: shawnh2 --- test/benchmark/tests/scale_httproutes.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmark/tests/scale_httproutes.go b/test/benchmark/tests/scale_httproutes.go index ccb3e816c54..842e5afe8f3 100644 --- a/test/benchmark/tests/scale_httproutes.go +++ b/test/benchmark/tests/scale_httproutes.go @@ -47,7 +47,7 @@ var ScaleHTTPRoutes = suite.BenchmarkTest{ // Setup and scale httproutes. httpRouteName := "benchmark-route-%d" - httpRouteScales := []uint16{5, 10, 20} // TODO: for test, update later + httpRouteScales := []uint16{10, 50, 100, 300, 500} httpRouteNNs := make([]types.NamespacedName, 0, httpRouteScales[len(httpRouteScales)-1]) bSuite.RegisterCleanup(t, ctx, gateway, &gwapiv1.HTTPRoute{}) From 63b70bd9419b5c842a07b1b4038a72311954f8ca Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Mon, 17 Jun 2024 16:52:01 +0800 Subject: [PATCH 09/30] increase poll timeout Signed-off-by: shawnh2 --- test/benchmark/suite/suite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmark/suite/suite.go b/test/benchmark/suite/suite.go index 28d820c1167..6d9b38abf0e 100644 --- a/test/benchmark/suite/suite.go +++ b/test/benchmark/suite/suite.go @@ -134,7 +134,7 @@ func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, } // Wait from benchmark test job to complete. - if err = wait.PollUntilContextTimeout(ctx, 10*time.Second, time.Duration(duration*3)*time.Second, true, func(ctx context.Context) (bool, error) { + if err = wait.PollUntilContextTimeout(ctx, 10*time.Second, time.Duration(duration*10)*time.Second, true, func(ctx context.Context) (bool, error) { job := new(batchv1.Job) if err = b.Client.Get(ctx, *jobNN, job); err != nil { return false, err From baf9be9e4dae3cfeb0f7cc6bc2e4feb7fbde1fee Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Tue, 18 Jun 2024 17:35:51 +0800 Subject: [PATCH 10/30] add longer timeout for go-test, collect reports in suite and change scale to scale-up Signed-off-by: shawnh2 --- test/benchmark/suite/suite.go | 67 ++++++++++++------------ test/benchmark/suite/test.go | 2 +- test/benchmark/tests/scale_httproutes.go | 50 +++++++++--------- tools/make/kube.mk | 3 +- 4 files changed, 63 insertions(+), 59 deletions(-) diff --git a/test/benchmark/suite/suite.go b/test/benchmark/suite/suite.go index 6d9b38abf0e..cbaa631650d 100644 --- a/test/benchmark/suite/suite.go +++ b/test/benchmark/suite/suite.go @@ -38,13 +38,12 @@ type BenchmarkTestSuite struct { TimeoutConfig config.TimeoutConfig ControllerName string Options BenchmarkOptions + Reports []BenchmarkReport // Resources template for supported benchmark targets. - GatewayTemplate *gwapiv1.Gateway - HTTPRouteTemplate *gwapiv1.HTTPRoute - - // Template for benchmark test client. - BenchmarkClient *batchv1.Job + GatewayTemplate *gwapiv1.Gateway + HTTPRouteTemplate *gwapiv1.HTTPRoute + BenchmarkClientJob *batchv1.Job // Indicates which resources are scaled. scaledLabel map[string]string @@ -91,13 +90,13 @@ func NewBenchmarkTestSuite(client client.Client, options BenchmarkOptions, container.Args = append(container.Args, staticArgs...) return &BenchmarkTestSuite{ - Client: client, - Options: options, - TimeoutConfig: timeoutConfig, - ControllerName: DefaultControllerName, - GatewayTemplate: gateway, - HTTPRouteTemplate: httproute, - BenchmarkClient: benchmarkClient, + Client: client, + Options: options, + TimeoutConfig: timeoutConfig, + ControllerName: DefaultControllerName, + GatewayTemplate: gateway, + HTTPRouteTemplate: httproute, + BenchmarkClientJob: benchmarkClient, scaledLabel: map[string]string{ BenchmarkTestScaledKey: "true", }, @@ -110,8 +109,9 @@ func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { for _, test := range tests { t.Logf("Running benchmark test: %s", test.ShortName) - // TODO: generate a readable benchmark report for human - _ = test.Test(t, b) + test.Test(t, b) + + // TODO: generate a human readable benchmark report for each test. } } @@ -120,17 +120,17 @@ func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { // TODO: currently running benchmark test via nighthawk_client, // consider switching to gRPC nighthawk-service for benchmark test. // ref: https://github.com/envoyproxy/nighthawk/blob/main/api/client/service.proto -func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, gatewayHostPort string, requestHeaders ...string) (*BenchmarkReport, error) { +func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, gatewayHostPort string, requestHeaders ...string) error { t.Logf("Running benchmark test: %s", name) jobNN, err := b.createBenchmarkClientJob(ctx, name, gatewayHostPort, requestHeaders...) if err != nil { - return nil, err + return err } duration, err := strconv.ParseInt(b.Options.Duration, 10, 64) if err != nil { - return nil, err + return err } // Wait from benchmark test job to complete. @@ -156,29 +156,32 @@ func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, return false, nil }); err != nil { t.Errorf("Failed to run benchmark test: %v", err) - return nil, err + return err } t.Logf("Running benchmark test: %s successfully", name) report, err := NewBenchmarkReport() if err != nil { - return nil, err + return err } // Get all the reports from this benchmark test run. if err = report.GetBenchmarkResult(t, ctx, jobNN); err != nil { - return nil, err + return err } if err = report.GetControlPlaneMetrics(t, ctx); err != nil { - return nil, err + return err } - return report, nil + report.Print(t, name) + b.Reports = append(b.Reports, *report) + + return nil } func (b *BenchmarkTestSuite) createBenchmarkClientJob(ctx context.Context, name, gatewayHostPort string, requestHeaders ...string) (*types.NamespacedName, error) { - job := b.BenchmarkClient.DeepCopy() + job := b.BenchmarkClientJob.DeepCopy() job.SetName(name) runtimeArgs := prepareBenchmarkClientRuntimeArgs(gatewayHostPort, requestHeaders...) @@ -213,21 +216,21 @@ func prepareBenchmarkClientRuntimeArgs(gatewayHostPort string, requestHeaders .. return args } -// ScaleHTTPRoutes scales HTTPRoutes that are all referenced to one Gateway according to -// the scale range: (begin, end]. The afterCreation is a callback function that only runs -// every time after one HTTPRoutes has been created successfully. +// ScaleUpHTTPRoutes scales up HTTPRoutes that are all referenced to one Gateway according to +// the scale range: (a, b], which a <= b. The `afterCreation` is a callback function that only +// runs every time after one HTTPRoutes has been created successfully. // // All scaled resources will be labeled with BenchmarkTestScaledKey. -func (b *BenchmarkTestSuite) ScaleHTTPRoutes(t *testing.T, ctx context.Context, scaleRange [2]uint16, targetName, refGateway string, afterCreation func(route *gwapiv1.HTTPRoute)) error { - t.Helper() - +func (b *BenchmarkTestSuite) ScaleUpHTTPRoutes(ctx context.Context, scaleRange [2]uint16, targetNameFormat, refGateway string, afterCreation func(route *gwapiv1.HTTPRoute)) error { var i, begin, end uint16 begin, end = scaleRange[0], scaleRange[1] - t.Logf("Scaling HTTPRoutes to %d", end) + if begin > end { + return fmt.Errorf("got wrong scale range, %d is less than %d", end, begin) + } for i = begin + 1; i <= end; i++ { - routeName := fmt.Sprintf(targetName, i) + routeName := fmt.Sprintf(targetNameFormat, i) newRoute := b.HTTPRouteTemplate.DeepCopy() newRoute.SetName(routeName) newRoute.SetLabels(b.scaledLabel) @@ -242,8 +245,6 @@ func (b *BenchmarkTestSuite) ScaleHTTPRoutes(t *testing.T, ctx context.Context, } } - t.Logf("Finish scaling HTTPRoutes to %d", end) - return nil } diff --git a/test/benchmark/suite/test.go b/test/benchmark/suite/test.go index 112e1b922f1..24d0ef8c1bb 100644 --- a/test/benchmark/suite/test.go +++ b/test/benchmark/suite/test.go @@ -13,7 +13,7 @@ import "testing" type BenchmarkTest struct { ShortName string Description string - Test func(*testing.T, *BenchmarkTestSuite) []BenchmarkReport + Test func(*testing.T, *BenchmarkTestSuite) } // BenchmarkOptions for nighthawk-client. diff --git a/test/benchmark/tests/scale_httproutes.go b/test/benchmark/tests/scale_httproutes.go index 842e5afe8f3..0d358335cd0 100644 --- a/test/benchmark/tests/scale_httproutes.go +++ b/test/benchmark/tests/scale_httproutes.go @@ -27,8 +27,8 @@ func init() { var ScaleHTTPRoutes = suite.BenchmarkTest{ ShortName: "ScaleHTTPRoute", - Description: "Fixed one Gateway and different number of HTTPRoutes on a scale of (10, 50, 100, 300, 500).", - Test: func(t *testing.T, bSuite *suite.BenchmarkTestSuite) (reports []suite.BenchmarkReport) { + Description: "Fixed one Gateway and different scales of HTTPRoutes.", + Test: func(t *testing.T, bSuite *suite.BenchmarkTestSuite) { var ( ctx = context.Background() ns = "benchmark-test" @@ -36,41 +36,43 @@ var ScaleHTTPRoutes = suite.BenchmarkTest{ requestHeaders = []string{ "Host: www.benchmark.com", } + routeNameFormat = "benchmark-route-%d" ) - // Setup and create a gateway. gatewayNN := types.NamespacedName{Name: "benchmark", Namespace: ns} gateway := bSuite.GatewayTemplate.DeepCopy() gateway.SetName(gatewayNN.Name) err = bSuite.CreateResource(ctx, gateway) require.NoError(t, err) - // Setup and scale httproutes. - httpRouteName := "benchmark-route-%d" - httpRouteScales := []uint16{10, 50, 100, 300, 500} - httpRouteNNs := make([]types.NamespacedName, 0, httpRouteScales[len(httpRouteScales)-1]) - bSuite.RegisterCleanup(t, ctx, gateway, &gwapiv1.HTTPRoute{}) - var start uint16 = 0 - for _, scale := range httpRouteScales { - err = bSuite.ScaleHTTPRoutes(t, ctx, [2]uint16{start, scale}, httpRouteName, gatewayNN.Name, func(route *gwapiv1.HTTPRoute) { - routeNN := types.NamespacedName{Name: route.Name, Namespace: route.Namespace} - httpRouteNNs = append(httpRouteNNs, routeNN) - }) - require.NoError(t, err) - start = scale + t.Run("scaling up httproutes", func(t *testing.T) { + routeScales := []uint16{10, 20} + routeNNs := make([]types.NamespacedName, 0, routeScales[len(routeScales)-1]) + + var start uint16 = 0 + for _, scale := range routeScales { + t.Run(fmt.Sprintf("scaling up httproutes to %d", scale), func(t *testing.T) { + err = bSuite.ScaleUpHTTPRoutes(ctx, [2]uint16{start, scale}, routeNameFormat, gatewayNN.Name, func(route *gwapiv1.HTTPRoute) { + routeNN := types.NamespacedName{Name: route.Name, Namespace: route.Namespace} + routeNNs = append(routeNNs, routeNN) + }) + require.NoError(t, err) + start = scale - gatewayAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, bSuite.Client, bSuite.TimeoutConfig, bSuite.ControllerName, kubernetes.NewGatewayRef(gatewayNN), httpRouteNNs...) + gatewayAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, bSuite.Client, bSuite.TimeoutConfig, + bSuite.ControllerName, kubernetes.NewGatewayRef(gatewayNN), routeNNs...) - // Run benchmark test at different scale. - name := fmt.Sprintf("scale-httproutes-%d", scale) - report, err := bSuite.Benchmark(t, ctx, name, gatewayAddr, requestHeaders...) - require.NoError(t, err) + // Run benchmark test at different scale. + name := fmt.Sprintf("scale-up-httproutes-%d", scale) + err = bSuite.Benchmark(t, ctx, name, gatewayAddr, requestHeaders...) + require.NoError(t, err) + }) + } + }) - report.Print(t, name) - reports = append(reports, *report) - } + // TODO: implement scaling down httproutes return }, diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 4bd2d7c8959..cd8c1845743 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -9,6 +9,7 @@ GATEWAY_API_VERSION ?= $(shell go list -m -f '{{.Version}}' sigs.k8s.io/gateway- GATEWAY_RELEASE_URL ?= https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_VERSION}/experimental-install.yaml WAIT_TIMEOUT ?= 15m +BENCHMARK_TIMEOUT ?= 60 FLUENT_BIT_CHART_VERSION ?= 0.30.4 OTEL_COLLECTOR_CHART_VERSION ?= 0.73.1 @@ -150,7 +151,7 @@ run-benchmark: install-benchmark-server ## Run benchmark tests kubectl wait --timeout=$(WAIT_TIMEOUT) -n benchmark-test deployment/nighthawk-test-server --for=condition=Available kubectl wait --timeout=$(WAIT_TIMEOUT) -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available kubectl apply -f test/benchmark/config/gatewayclass.yaml - go test -v -tags benchmark ./test/benchmark --rps=$(RPS) --connections=$(CONNECTIONS) --duration=$(DURATION) + go test -v -tags benchmark -timeout $(BENCHMARK_TIMEOUT) ./test/benchmark --rps=$(RPS) --connections=$(CONNECTIONS) --duration=$(DURATION) .PHONY: install-benchmark-server install-benchmark-server: ## Install nighthawk server for benchmark test From 88d22041c5532215112301b207ab18c722177e56 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Tue, 18 Jun 2024 19:21:47 +0800 Subject: [PATCH 11/30] update resource limits for both envoyproxy and envoygateway pod Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 19 ++++++++++++++++--- test/benchmark/config/gatewayclass.yaml | 16 +++++++--------- tools/make/kube.mk | 18 ++++++++++++++++-- 3 files changed, 39 insertions(+), 14 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index b29dac2e487..d71d70bd064 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -21,6 +21,16 @@ on: default: '90' type: string required: false + cpu_limits: + description: "The CPU resource limits for the envoy gateway. Default: 1000m." + default: '1000m' + type: string + required: false + memory_limits: + description: "The memory resource limits for the envoy gateway. Default: 1024Mi." + default: '1024Mi' + type: string + required: false jobs: benchmark-test: @@ -28,13 +38,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - uses: ./tools/github-actions/setup-deps - name: Run Benchmark tests env: KIND_NODE_TAG: v1.28.0 IMAGE_PULL_POLICY: IfNotPresent - RPS: ${{ github.event.inputs.rps || 1000 }} - CONNECTIONS: ${{ github.event.inputs.connections || 100 }} - DURATION: ${{ github.event.inputs.duration || 90 }} + BENCHMARK_RPS: ${{ github.event.inputs.rps || 1000 }} + BENCHMARK_CONNECTIONS: ${{ github.event.inputs.connections || 100 }} + BENCHMARK_DURATION: ${{ github.event.inputs.duration || 90 }} + BENCHMARK_CPU_LIMITS: ${{ github.event.inputs.cpu_limits || 1000m }} + BENCHMARK_MEMORY_LIMITS: ${{ github.event.inputs.memory_limits || 1024Mi }} run: make benchmark diff --git a/test/benchmark/config/gatewayclass.yaml b/test/benchmark/config/gatewayclass.yaml index 280af5a1769..1a62f3ba569 100644 --- a/test/benchmark/config/gatewayclass.yaml +++ b/test/benchmark/config/gatewayclass.yaml @@ -21,15 +21,13 @@ spec: kubernetes: envoyDeployment: container: - volumeMounts: - - mountPath: /var/run/ext-proc - name: socket-dir - pod: - volumes: - - name: socket-dir - hostPath: - path: /var/run/ext-proc - type: '' + resources: + limits: + memory: "1024Mi" + cpu: "1000m" + request: + memory: "256Mi" + cpu: "500m" telemetry: metrics: prometheus: {} diff --git a/tools/make/kube.mk b/tools/make/kube.mk index cd8c1845743..522f2fb2981 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -9,7 +9,13 @@ GATEWAY_API_VERSION ?= $(shell go list -m -f '{{.Version}}' sigs.k8s.io/gateway- GATEWAY_RELEASE_URL ?= https://github.com/kubernetes-sigs/gateway-api/releases/download/${GATEWAY_API_VERSION}/experimental-install.yaml WAIT_TIMEOUT ?= 15m + BENCHMARK_TIMEOUT ?= 60 +BENCHMARK_CPU_LIMITS ?= 1000m +BENCHMARK_MEMORY_LIMITS ?= 1024Mi +BENCHMARK_RPS ?= 1000 +BENCHMARK_CONNECTIONS ?= 100 +BENCHMARK_DURATION ?= 60 FLUENT_BIT_CHART_VERSION ?= 0.30.4 OTEL_COLLECTOR_CHART_VERSION ?= 0.73.1 @@ -68,6 +74,14 @@ kube-deploy: manifests helm-generate ## Install Envoy Gateway into the Kubernete @$(LOG_TARGET) helm install eg charts/gateway-helm --set deployment.envoyGateway.imagePullPolicy=$(IMAGE_PULL_POLICY) -n envoy-gateway-system --create-namespace --debug --timeout='$(WAIT_TIMEOUT)' --wait --wait-for-jobs +.PHONY: kube-deploy-for-benchmark-test +kube-deploy-for-benchmark-test: manifests helm-generate ## Install Envoy Gateway for benchmark test purpose only. + @$(LOG_TARGET) + helm install eg charts/gateway-helm --set deployment.envoyGateway.imagePullPolicy=$(IMAGE_PULL_POLICY) \ + --set deployment.envoyGateway.resources.limits.cpu=$(BENCHMARK_CPU_LIMITS) \ + --set deployment.envoyGateway.resources.limits.memory=$(BENCHMARK_MEMORY_LIMITS) \ + -n envoy-gateway-system --create-namespace --debug --timeout='$(WAIT_TIMEOUT)' --wait --wait-for-jobs + .PHONY: kube-undeploy kube-undeploy: manifests ## Uninstall the Envoy Gateway into the Kubernetes cluster specified in ~/.kube/config. @$(LOG_TARGET) @@ -105,7 +119,7 @@ conformance: create-cluster kube-install-image kube-deploy run-conformance delet experimental-conformance: create-cluster kube-install-image kube-deploy run-experimental-conformance delete-cluster ## Create a kind cluster, deploy EG into it, run Gateway API conformance, and clean up. .PHONY: benchmark -benchmark: create-cluster kube-install-image kube-deploy run-benchmark delete-cluster ## Create a kind cluster, deploy EG into it, run Envoy Gateway benchmark test, and clean up. +benchmark: create-cluster kube-install-image kube-deploy-for-benchmark-test run-benchmark delete-cluster ## Create a kind cluster, deploy EG into it, run Envoy Gateway benchmark test, and clean up. .PHONY: e2e e2e: create-cluster kube-install-image kube-deploy install-ratelimit run-e2e delete-cluster @@ -151,7 +165,7 @@ run-benchmark: install-benchmark-server ## Run benchmark tests kubectl wait --timeout=$(WAIT_TIMEOUT) -n benchmark-test deployment/nighthawk-test-server --for=condition=Available kubectl wait --timeout=$(WAIT_TIMEOUT) -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available kubectl apply -f test/benchmark/config/gatewayclass.yaml - go test -v -tags benchmark -timeout $(BENCHMARK_TIMEOUT) ./test/benchmark --rps=$(RPS) --connections=$(CONNECTIONS) --duration=$(DURATION) + go test -v -tags benchmark -timeout $(BENCHMARK_TIMEOUT) ./test/benchmark --rps=$(BENCHMARK_RPS) --connections=$(BENCHMARK_CONNECTIONS) --duration=$(BENCHMARK_DURATION) .PHONY: install-benchmark-server install-benchmark-server: ## Install nighthawk server for benchmark test From 8add6f51644d92c320fdfbfbc30eb95c4d9bd47d Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Tue, 18 Jun 2024 19:34:04 +0800 Subject: [PATCH 12/30] fix github action unit problem Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 12 ++++++------ tools/make/kube.mk | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index d71d70bd064..1e8274ef09d 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -22,13 +22,13 @@ on: type: string required: false cpu_limits: - description: "The CPU resource limits for the envoy gateway. Default: 1000m." - default: '1000m' + description: "The CPU resource limits for the envoy gateway, in unit 'm'. Default: 1000m." + default: '1000' type: string required: false memory_limits: - description: "The memory resource limits for the envoy gateway. Default: 1024Mi." - default: '1024Mi' + description: "The memory resource limits for the envoy gateway, in unit 'Mi'. Default: 1024Mi." + default: '1024' type: string required: false @@ -48,6 +48,6 @@ jobs: BENCHMARK_RPS: ${{ github.event.inputs.rps || 1000 }} BENCHMARK_CONNECTIONS: ${{ github.event.inputs.connections || 100 }} BENCHMARK_DURATION: ${{ github.event.inputs.duration || 90 }} - BENCHMARK_CPU_LIMITS: ${{ github.event.inputs.cpu_limits || 1000m }} - BENCHMARK_MEMORY_LIMITS: ${{ github.event.inputs.memory_limits || 1024Mi }} + BENCHMARK_CPU_LIMITS: ${{ github.event.inputs.cpu_limits || 1000 }} + BENCHMARK_MEMORY_LIMITS: ${{ github.event.inputs.memory_limits || 1024 }} run: make benchmark diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 522f2fb2981..53ccce2c57f 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -11,8 +11,8 @@ GATEWAY_RELEASE_URL ?= https://github.com/kubernetes-sigs/gateway-api/releases/d WAIT_TIMEOUT ?= 15m BENCHMARK_TIMEOUT ?= 60 -BENCHMARK_CPU_LIMITS ?= 1000m -BENCHMARK_MEMORY_LIMITS ?= 1024Mi +BENCHMARK_CPU_LIMITS ?= 1000 # unit: 'm' +BENCHMARK_MEMORY_LIMITS ?= 1024 # unit: 'Mi' BENCHMARK_RPS ?= 1000 BENCHMARK_CONNECTIONS ?= 100 BENCHMARK_DURATION ?= 60 @@ -78,8 +78,8 @@ kube-deploy: manifests helm-generate ## Install Envoy Gateway into the Kubernete kube-deploy-for-benchmark-test: manifests helm-generate ## Install Envoy Gateway for benchmark test purpose only. @$(LOG_TARGET) helm install eg charts/gateway-helm --set deployment.envoyGateway.imagePullPolicy=$(IMAGE_PULL_POLICY) \ - --set deployment.envoyGateway.resources.limits.cpu=$(BENCHMARK_CPU_LIMITS) \ - --set deployment.envoyGateway.resources.limits.memory=$(BENCHMARK_MEMORY_LIMITS) \ + --set deployment.envoyGateway.resources.limits.cpu=$(BENCHMARK_CPU_LIMITS)m \ + --set deployment.envoyGateway.resources.limits.memory=$(BENCHMARK_MEMORY_LIMITS)Mi \ -n envoy-gateway-system --create-namespace --debug --timeout='$(WAIT_TIMEOUT)' --wait --wait-for-jobs .PHONY: kube-undeploy From a7eb9f4d2487ee444f6ccce7917a008a64b7ec9b Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Tue, 18 Jun 2024 21:06:02 +0800 Subject: [PATCH 13/30] add scale-down test case support Signed-off-by: shawnh2 --- test/benchmark/config/gatewayclass.yaml | 2 +- test/benchmark/suite/suite.go | 50 +++++++++++++++++++++--- test/benchmark/tests/scale_httproutes.go | 42 +++++++++++++++++--- tools/make/kube.mk | 2 +- 4 files changed, 82 insertions(+), 14 deletions(-) diff --git a/test/benchmark/config/gatewayclass.yaml b/test/benchmark/config/gatewayclass.yaml index 1a62f3ba569..fbecf76332d 100644 --- a/test/benchmark/config/gatewayclass.yaml +++ b/test/benchmark/config/gatewayclass.yaml @@ -25,7 +25,7 @@ spec: limits: memory: "1024Mi" cpu: "1000m" - request: + requests: memory: "256Mi" cpu: "500m" telemetry: diff --git a/test/benchmark/suite/suite.go b/test/benchmark/suite/suite.go index cbaa631650d..659b63e7cb6 100644 --- a/test/benchmark/suite/suite.go +++ b/test/benchmark/suite/suite.go @@ -217,20 +217,22 @@ func prepareBenchmarkClientRuntimeArgs(gatewayHostPort string, requestHeaders .. } // ScaleUpHTTPRoutes scales up HTTPRoutes that are all referenced to one Gateway according to -// the scale range: (a, b], which a <= b. The `afterCreation` is a callback function that only -// runs every time after one HTTPRoutes has been created successfully. +// the scale range: (a, b], which scales up from a to b with a <= b. // -// All scaled resources will be labeled with BenchmarkTestScaledKey. -func (b *BenchmarkTestSuite) ScaleUpHTTPRoutes(ctx context.Context, scaleRange [2]uint16, targetNameFormat, refGateway string, afterCreation func(route *gwapiv1.HTTPRoute)) error { +// The `afterCreation` is a callback function that only runs every time after one HTTPRoutes +// has been created successfully. +// +// All created scaled resources will be labeled with BenchmarkTestScaledKey. +func (b *BenchmarkTestSuite) ScaleUpHTTPRoutes(ctx context.Context, scaleRange [2]uint16, routeNameFormat, refGateway string, afterCreation func(*gwapiv1.HTTPRoute)) error { var i, begin, end uint16 begin, end = scaleRange[0], scaleRange[1] if begin > end { - return fmt.Errorf("got wrong scale range, %d is less than %d", end, begin) + return fmt.Errorf("got wrong scale range, %d is not greater than %d", end, begin) } for i = begin + 1; i <= end; i++ { - routeName := fmt.Sprintf(targetNameFormat, i) + routeName := fmt.Sprintf(routeNameFormat, i) newRoute := b.HTTPRouteTemplate.DeepCopy() newRoute.SetName(routeName) newRoute.SetLabels(b.scaledLabel) @@ -248,6 +250,42 @@ func (b *BenchmarkTestSuite) ScaleUpHTTPRoutes(ctx context.Context, scaleRange [ return nil } +// ScaleDownHTTPRoutes scales down HTTPRoutes that are all referenced to one Gateway according to +// the scale range: [a, b), which scales down from a to b with a > b. +// +// The `afterDeletion` is a callback function that only runs every time after one HTTPRoutes has +// been deleted successfully. +func (b *BenchmarkTestSuite) ScaleDownHTTPRoutes(ctx context.Context, scaleRange [2]uint16, routeNameFormat, refGateway string, afterDeletion func(*gwapiv1.HTTPRoute)) error { + var i, begin, end uint16 + begin, end = scaleRange[0], scaleRange[1] + + if begin <= end { + return fmt.Errorf("got wrong scale range, %d is not less than %d", end, begin) + } + + if end == 0 { + return fmt.Errorf("cannot scale routes down to zero") + } + + for i = begin; i > end; i-- { + routeName := fmt.Sprintf(routeNameFormat, i) + oldRoute := b.HTTPRouteTemplate.DeepCopy() + oldRoute.SetName(routeName) + oldRoute.SetLabels(b.scaledLabel) + oldRoute.Spec.ParentRefs[0].Name = gwapiv1.ObjectName(refGateway) + + if err := b.CleanupResource(ctx, oldRoute); err != nil { + return err + } + + if afterDeletion != nil { + afterDeletion(oldRoute) + } + } + + return nil +} + func (b *BenchmarkTestSuite) CreateResource(ctx context.Context, object client.Object) error { if err := b.Client.Create(ctx, object); err != nil { if !kerrors.IsAlreadyExists(err) { diff --git a/test/benchmark/tests/scale_httproutes.go b/test/benchmark/tests/scale_httproutes.go index 0d358335cd0..ef63535ee57 100644 --- a/test/benchmark/tests/scale_httproutes.go +++ b/test/benchmark/tests/scale_httproutes.go @@ -36,7 +36,6 @@ var ScaleHTTPRoutes = suite.BenchmarkTest{ requestHeaders = []string{ "Host: www.benchmark.com", } - routeNameFormat = "benchmark-route-%d" ) gatewayNN := types.NamespacedName{Name: "benchmark", Namespace: ns} @@ -45,18 +44,22 @@ var ScaleHTTPRoutes = suite.BenchmarkTest{ err = bSuite.CreateResource(ctx, gateway) require.NoError(t, err) + routeNameFormat := "benchmark-route-%d" + routeScales := []uint16{10, 50, 100, 300, 500} + routeScalesN := len(routeScales) + routeNNs := make([]types.NamespacedName, 0, routeScales[routeScalesN-1]) + bSuite.RegisterCleanup(t, ctx, gateway, &gwapiv1.HTTPRoute{}) t.Run("scaling up httproutes", func(t *testing.T) { - routeScales := []uint16{10, 20} - routeNNs := make([]types.NamespacedName, 0, routeScales[len(routeScales)-1]) - var start uint16 = 0 for _, scale := range routeScales { t.Run(fmt.Sprintf("scaling up httproutes to %d", scale), func(t *testing.T) { err = bSuite.ScaleUpHTTPRoutes(ctx, [2]uint16{start, scale}, routeNameFormat, gatewayNN.Name, func(route *gwapiv1.HTTPRoute) { routeNN := types.NamespacedName{Name: route.Name, Namespace: route.Namespace} routeNNs = append(routeNNs, routeNN) + + t.Logf("Create HTTPRoute: %s", routeNN.String()) }) require.NoError(t, err) start = scale @@ -72,8 +75,35 @@ var ScaleHTTPRoutes = suite.BenchmarkTest{ } }) - // TODO: implement scaling down httproutes + t.Run("scaling down httproutes", func(t *testing.T) { + var start = routeScales[routeScalesN-1] + + for i := routeScalesN - 2; i >= 0; i-- { + scale := routeScales[i] + + t.Run(fmt.Sprintf("scaling down httproutes to %d", scale), func(t *testing.T) { + err = bSuite.ScaleDownHTTPRoutes(ctx, [2]uint16{start, scale}, routeNameFormat, gatewayNN.Name, func(route *gwapiv1.HTTPRoute) { + routeNN := routeNNs[len(routeNNs)-1] + routeNNs = routeNNs[:len(routeNNs)-1] + + // Making sure we are deleting the right one route. + require.Equal(t, routeNN, + types.NamespacedName{Name: route.Name, Namespace: route.Namespace}) + + t.Logf("Delete HTTPRoute: %s", routeNN.String()) + }) + require.NoError(t, err) + start = scale - return + gatewayAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, bSuite.Client, bSuite.TimeoutConfig, + bSuite.ControllerName, kubernetes.NewGatewayRef(gatewayNN), routeNNs...) + + // Run benchmark test at different scale. + name := fmt.Sprintf("scale-down-httproutes-%d", scale) + err = bSuite.Benchmark(t, ctx, name, gatewayAddr, requestHeaders...) + require.NoError(t, err) + }) + } + }) }, } diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 53ccce2c57f..36f8609f83c 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -10,7 +10,7 @@ GATEWAY_RELEASE_URL ?= https://github.com/kubernetes-sigs/gateway-api/releases/d WAIT_TIMEOUT ?= 15m -BENCHMARK_TIMEOUT ?= 60 +BENCHMARK_TIMEOUT ?= 60m BENCHMARK_CPU_LIMITS ?= 1000 # unit: 'm' BENCHMARK_MEMORY_LIMITS ?= 1024 # unit: 'Mi' BENCHMARK_RPS ?= 1000 From 9314a81a00d2e4b79596e3dce784573bfd835ad6 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Wed, 19 Jun 2024 16:28:07 +0800 Subject: [PATCH 14/30] fix according to benchmark-result: - add uninstall commands and properly cleanup resources - increase timeout config - change local port of port forwarder to random assign Signed-off-by: shawnh2 --- test/benchmark/suite/report.go | 3 ++- test/benchmark/suite/suite.go | 46 +++++++++++++--------------------- tools/make/kube.mk | 8 ++++++ 3 files changed, 27 insertions(+), 30 deletions(-) diff --git a/test/benchmark/suite/report.go b/test/benchmark/suite/report.go index 7486f0883e2..e9a030f1480 100644 --- a/test/benchmark/suite/report.go +++ b/test/benchmark/suite/report.go @@ -25,6 +25,7 @@ import ( ) const ( + localMetricsPort = 0 controlPlaneMetricsPort = 19001 ) @@ -123,7 +124,7 @@ func (r *BenchmarkReport) getBenchmarkResultFromPodLogs(ctx context.Context, pod } func (r *BenchmarkReport) getMetricsFromPodPortForwarder(t *testing.T, pod *types.NamespacedName) error { - fw, err := kube.NewLocalPortForwarder(r.kubeClient, *pod, controlPlaneMetricsPort, controlPlaneMetricsPort) + fw, err := kube.NewLocalPortForwarder(r.kubeClient, *pod, localMetricsPort, controlPlaneMetricsPort) if err != nil { return fmt.Errorf("failed to build port forwarder for pod %s: %v", pod.String(), err) } diff --git a/test/benchmark/suite/suite.go b/test/benchmark/suite/suite.go index 659b63e7cb6..ab77417938e 100644 --- a/test/benchmark/suite/suite.go +++ b/test/benchmark/suite/suite.go @@ -17,7 +17,6 @@ import ( "time" batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" @@ -82,7 +81,9 @@ func NewBenchmarkTestSuite(client client.Client, options BenchmarkOptions, return nil, err } + // Reset some timeout config for the benchmark test. config.SetupTimeoutConfig(&timeoutConfig) + timeoutConfig.RouteMustHaveParents = 180 * time.Second // Prepare static options for benchmark client. staticArgs := prepareBenchmarkClientStaticArgs(options) @@ -111,7 +112,7 @@ func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { test.Test(t, b) - // TODO: generate a human readable benchmark report for each test. + // TODO: generate a human-readable benchmark report for each test. } } @@ -153,9 +154,11 @@ func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, } t.Logf("Job %s still not complete", name) + return false, nil }); err != nil { t.Errorf("Failed to run benchmark test: %v", err) + return err } @@ -274,7 +277,7 @@ func (b *BenchmarkTestSuite) ScaleDownHTTPRoutes(ctx context.Context, scaleRange oldRoute.SetLabels(b.scaledLabel) oldRoute.Spec.ParentRefs[0].Name = gwapiv1.ObjectName(refGateway) - if err := b.CleanupResource(ctx, oldRoute); err != nil { + if err := b.DeleteResource(ctx, oldRoute); err != nil { return err } @@ -297,19 +300,7 @@ func (b *BenchmarkTestSuite) CreateResource(ctx context.Context, object client.O return nil } -// RegisterCleanup registers cleanup functions for all benchmark test resources. -func (b *BenchmarkTestSuite) RegisterCleanup(t *testing.T, ctx context.Context, object, scaledObject client.Object) { - t.Cleanup(func() { - t.Logf("Start to cleanup benchmark test resources") - - _ = b.CleanupResource(ctx, object) - _ = b.CleanupScaledResources(ctx, scaledObject) - - t.Logf("Clean up complete!") - }) -} - -func (b *BenchmarkTestSuite) CleanupResource(ctx context.Context, object client.Object) error { +func (b *BenchmarkTestSuite) DeleteResource(ctx context.Context, object client.Object) error { if err := b.Client.Delete(ctx, object); err != nil { if !kerrors.IsNotFound(err) { return err @@ -320,8 +311,8 @@ func (b *BenchmarkTestSuite) CleanupResource(ctx context.Context, object client. return nil } -// CleanupScaledResources only cleanups all the resources under benchmark-test namespace. -func (b *BenchmarkTestSuite) CleanupScaledResources(ctx context.Context, object client.Object) error { +// DeleteScaledResources only cleanups all the resources under benchmark-test namespace. +func (b *BenchmarkTestSuite) DeleteScaledResources(ctx context.Context, object client.Object) error { if err := b.Client.DeleteAllOf(ctx, object, client.MatchingLabels{BenchmarkTestScaledKey: "true"}, client.InNamespace("benchmark-test")); err != nil { return err @@ -329,17 +320,14 @@ func (b *BenchmarkTestSuite) CleanupScaledResources(ctx context.Context, object return nil } -// CleanupBenchmarkClientJobs only cleanups all the jobs and its associated pods under benchmark-test namespace. -func (b *BenchmarkTestSuite) CleanupBenchmarkClientJobs(ctx context.Context, name string) error { - if err := b.Client.DeleteAllOf(ctx, &batchv1.Job{}, - client.MatchingLabels{BenchmarkTestClientKey: "true"}, client.InNamespace("benchmark-test")); err != nil { - return err - } +// RegisterCleanup registers cleanup functions for all benchmark test resources. +func (b *BenchmarkTestSuite) RegisterCleanup(t *testing.T, ctx context.Context, object, scaledObject client.Object) { + t.Cleanup(func() { + t.Logf("Start to cleanup benchmark test resources") - if err := b.Client.DeleteAllOf(ctx, &corev1.Pod{}, - client.MatchingLabels{"job-name": name}, client.InNamespace("benchmark-test")); err != nil { - return err - } + _ = b.DeleteResource(ctx, object) + _ = b.DeleteScaledResources(ctx, scaledObject) - return nil + t.Logf("Clean up complete!") + }) } diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 36f8609f83c..b1f9f83e6f7 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -174,6 +174,14 @@ install-benchmark-server: ## Install nighthawk server for benchmark test kubectl -n benchmark-test create configmap test-server-config --from-file=test/benchmark/config/nighthawk-test-server-config.yaml -o yaml kubectl apply -f test/benchmark/config/nighthawk-test-server.yaml +.PHONY: uninstall-benchmark-server +uninstall-benchmark-server: ## Uninstall nighthawk server for benchmark test + @$(LOG_TARGET) + kubectl delete job -n benchmark-test -l benchmark-test/client=true + kubectl delete -f test/benchmark/config/nighthawk-test-server.yaml + kubectl delete configmap test-server-config -n benchmark-test + kubectl delete namespace benchmark-test + .PHONY: install-e2e-telemetry install-e2e-telemetry: prepare-helm-repo install-fluent-bit install-loki install-tempo install-otel-collector install-prometheus @$(LOG_TARGET) From 0e70867439d2d919aa2acc19397d817bc15ed5d9 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Wed, 19 Jun 2024 16:55:24 +0800 Subject: [PATCH 15/30] increase memory to 2GiB to see the difference Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 1e8274ef09d..3d384e8ff22 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -28,7 +28,7 @@ on: required: false memory_limits: description: "The memory resource limits for the envoy gateway, in unit 'Mi'. Default: 1024Mi." - default: '1024' + default: '2048' type: string required: false From 32a7c3437c1085ff011587013824912baac050ff Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Wed, 19 Jun 2024 17:53:27 +0800 Subject: [PATCH 16/30] return report for every benchmark test run Signed-off-by: shawnh2 --- test/benchmark/suite/report.go | 7 ++++-- test/benchmark/suite/suite.go | 28 +++++++++++++----------- test/benchmark/suite/test.go | 2 +- test/benchmark/tests/scale_httproutes.go | 12 +++++++--- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/test/benchmark/suite/report.go b/test/benchmark/suite/report.go index e9a030f1480..be134926ff6 100644 --- a/test/benchmark/suite/report.go +++ b/test/benchmark/suite/report.go @@ -30,25 +30,28 @@ const ( ) type BenchmarkReport struct { + Name string RawResult []byte RawMetrics []byte kubeClient kube.CLIClient } -func NewBenchmarkReport() (*BenchmarkReport, error) { +func NewBenchmarkReport(name string) (*BenchmarkReport, error) { kubeClient, err := kube.NewCLIClient(options.DefaultConfigFlags.ToRawKubeConfigLoader()) if err != nil { return nil, err } return &BenchmarkReport{ + Name: name, kubeClient: kubeClient, }, nil } +// Print prints the raw report of one benchmark test. func (r *BenchmarkReport) Print(t *testing.T, name string) { - t.Logf("The report of benchmark test: %s", name) + t.Logf("The raw report of benchmark test: %s", name) t.Logf("=== Benchmark Result: \n\n %s \n\n", r.RawResult) t.Logf("=== Control-Plane Metrics: \n\n %s \n\n", r.RawMetrics) diff --git a/test/benchmark/suite/suite.go b/test/benchmark/suite/suite.go index ab77417938e..4841596e33a 100644 --- a/test/benchmark/suite/suite.go +++ b/test/benchmark/suite/suite.go @@ -37,7 +37,6 @@ type BenchmarkTestSuite struct { TimeoutConfig config.TimeoutConfig ControllerName string Options BenchmarkOptions - Reports []BenchmarkReport // Resources template for supported benchmark targets. GatewayTemplate *gwapiv1.Gateway @@ -110,9 +109,13 @@ func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { for _, test := range tests { t.Logf("Running benchmark test: %s", test.ShortName) - test.Test(t, b) + reports := test.Test(t, b) + if len(reports) == 0 { + continue + } - // TODO: generate a human-readable benchmark report for each test. + // TODO: Generate a human-readable benchmark report for each test. + t.Logf("Got %d reports for test: %s", len(reports), test.ShortName) } } @@ -121,17 +124,17 @@ func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { // TODO: currently running benchmark test via nighthawk_client, // consider switching to gRPC nighthawk-service for benchmark test. // ref: https://github.com/envoyproxy/nighthawk/blob/main/api/client/service.proto -func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, gatewayHostPort string, requestHeaders ...string) error { +func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, gatewayHostPort string, requestHeaders ...string) (*BenchmarkReport, error) { t.Logf("Running benchmark test: %s", name) jobNN, err := b.createBenchmarkClientJob(ctx, name, gatewayHostPort, requestHeaders...) if err != nil { - return err + return nil, err } duration, err := strconv.ParseInt(b.Options.Duration, 10, 64) if err != nil { - return err + return nil, err } // Wait from benchmark test job to complete. @@ -159,28 +162,27 @@ func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, }); err != nil { t.Errorf("Failed to run benchmark test: %v", err) - return err + return nil, err } t.Logf("Running benchmark test: %s successfully", name) - report, err := NewBenchmarkReport() + report, err := NewBenchmarkReport(name) if err != nil { - return err + return nil, err } // Get all the reports from this benchmark test run. if err = report.GetBenchmarkResult(t, ctx, jobNN); err != nil { - return err + return nil, err } if err = report.GetControlPlaneMetrics(t, ctx); err != nil { - return err + return nil, err } report.Print(t, name) - b.Reports = append(b.Reports, *report) - return nil + return report, nil } func (b *BenchmarkTestSuite) createBenchmarkClientJob(ctx context.Context, name, gatewayHostPort string, requestHeaders ...string) (*types.NamespacedName, error) { diff --git a/test/benchmark/suite/test.go b/test/benchmark/suite/test.go index 24d0ef8c1bb..17138dfad25 100644 --- a/test/benchmark/suite/test.go +++ b/test/benchmark/suite/test.go @@ -13,7 +13,7 @@ import "testing" type BenchmarkTest struct { ShortName string Description string - Test func(*testing.T, *BenchmarkTestSuite) + Test func(*testing.T, *BenchmarkTestSuite) []*BenchmarkReport } // BenchmarkOptions for nighthawk-client. diff --git a/test/benchmark/tests/scale_httproutes.go b/test/benchmark/tests/scale_httproutes.go index ef63535ee57..ca9c13f1828 100644 --- a/test/benchmark/tests/scale_httproutes.go +++ b/test/benchmark/tests/scale_httproutes.go @@ -28,7 +28,7 @@ func init() { var ScaleHTTPRoutes = suite.BenchmarkTest{ ShortName: "ScaleHTTPRoute", Description: "Fixed one Gateway and different scales of HTTPRoutes.", - Test: func(t *testing.T, bSuite *suite.BenchmarkTestSuite) { + Test: func(t *testing.T, bSuite *suite.BenchmarkTestSuite) (reports []*suite.BenchmarkReport) { var ( ctx = context.Background() ns = "benchmark-test" @@ -69,8 +69,10 @@ var ScaleHTTPRoutes = suite.BenchmarkTest{ // Run benchmark test at different scale. name := fmt.Sprintf("scale-up-httproutes-%d", scale) - err = bSuite.Benchmark(t, ctx, name, gatewayAddr, requestHeaders...) + report, err := bSuite.Benchmark(t, ctx, name, gatewayAddr, requestHeaders...) require.NoError(t, err) + + reports = append(reports, report) }) } }) @@ -100,10 +102,14 @@ var ScaleHTTPRoutes = suite.BenchmarkTest{ // Run benchmark test at different scale. name := fmt.Sprintf("scale-down-httproutes-%d", scale) - err = bSuite.Benchmark(t, ctx, name, gatewayAddr, requestHeaders...) + report, err := bSuite.Benchmark(t, ctx, name, gatewayAddr, requestHeaders...) require.NoError(t, err) + + reports = append(reports, report) }) } }) + + return }, } From 6d724a7b906ef8fa8f4bbf4dd35aac49380e839f Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Wed, 19 Jun 2024 18:01:14 +0800 Subject: [PATCH 17/30] right memory setup for github ci Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 3d384e8ff22..4b07a99621f 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -22,13 +22,13 @@ on: type: string required: false cpu_limits: - description: "The CPU resource limits for the envoy gateway, in unit 'm'. Default: 1000m." + description: "The CPU resource limits for the envoy gateway, in unit 'm'. Default: 1000." default: '1000' type: string required: false memory_limits: - description: "The memory resource limits for the envoy gateway, in unit 'Mi'. Default: 1024Mi." - default: '2048' + description: "The memory resource limits for the envoy gateway, in unit 'Mi'. Default: 1024." + default: '1024' type: string required: false @@ -49,5 +49,5 @@ jobs: BENCHMARK_CONNECTIONS: ${{ github.event.inputs.connections || 100 }} BENCHMARK_DURATION: ${{ github.event.inputs.duration || 90 }} BENCHMARK_CPU_LIMITS: ${{ github.event.inputs.cpu_limits || 1000 }} - BENCHMARK_MEMORY_LIMITS: ${{ github.event.inputs.memory_limits || 1024 }} + BENCHMARK_MEMORY_LIMITS: ${{ github.event.inputs.memory_limits || 2048 }} run: make benchmark From 8ac9073a14fbea3c0fd724d29344ab60987fcb89 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Thu, 20 Jun 2024 19:14:45 +0800 Subject: [PATCH 18/30] update report util methods Signed-off-by: shawnh2 --- test/benchmark/suite/report.go | 50 ++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/test/benchmark/suite/report.go b/test/benchmark/suite/report.go index be134926ff6..ec62be16ba5 100644 --- a/test/benchmark/suite/report.go +++ b/test/benchmark/suite/report.go @@ -30,9 +30,9 @@ const ( ) type BenchmarkReport struct { - Name string - RawResult []byte - RawMetrics []byte + Name string + RawResult []byte + RawCPMetrics []byte kubeClient kube.CLIClient } @@ -54,7 +54,7 @@ func (r *BenchmarkReport) Print(t *testing.T, name string) { t.Logf("The raw report of benchmark test: %s", name) t.Logf("=== Benchmark Result: \n\n %s \n\n", r.RawResult) - t.Logf("=== Control-Plane Metrics: \n\n %s \n\n", r.RawMetrics) + t.Logf("=== Control-Plane Metrics: \n\n %s \n\n", r.RawCPMetrics) } func (r *BenchmarkReport) GetBenchmarkResult(t *testing.T, ctx context.Context, job *types.NamespacedName) error { @@ -70,17 +70,21 @@ func (r *BenchmarkReport) GetBenchmarkResult(t *testing.T, ctx context.Context, } pod := &pods.Items[0] - if err = r.getBenchmarkResultFromPodLogs( + logs, err := r.getLogsFromPod( ctx, &types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, - ); err != nil { + ) + if err != nil { return err } + r.RawResult = logs + return nil } func (r *BenchmarkReport) GetControlPlaneMetrics(t *testing.T, ctx context.Context) error { - egPods, err := r.kubeClient.Kube().CoreV1().Pods("envoy-gateway-system").List(ctx, metav1.ListOptions{LabelSelector: "control-plane=envoy-gateway"}) + egPods, err := r.kubeClient.Kube().CoreV1().Pods("envoy-gateway-system"). + List(ctx, metav1.ListOptions{LabelSelector: "control-plane=envoy-gateway"}) if err != nil { return err } @@ -94,24 +98,26 @@ func (r *BenchmarkReport) GetControlPlaneMetrics(t *testing.T, ctx context.Conte } egPod := &egPods.Items[0] - if err = r.getMetricsFromPodPortForwarder( + metrics, err := r.getMetricsFromPortForwarder( t, &types.NamespacedName{Name: egPod.Name, Namespace: egPod.Namespace}, - ); err != nil { + ) + if err != nil { return err } + r.RawCPMetrics = metrics + return nil } -// getBenchmarkResultFromPodLogs scrapes the logs directly from the pod (default container) -// and save it as the raw result in benchmark report. -func (r *BenchmarkReport) getBenchmarkResultFromPodLogs(ctx context.Context, pod *types.NamespacedName) error { +// getLogsFromPod scrapes the logs directly from the pod (default container). +func (r *BenchmarkReport) getLogsFromPod(ctx context.Context, pod *types.NamespacedName) ([]byte, error) { podLogOpts := corev1.PodLogOptions{} req := r.kubeClient.Kube().CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts) podLogs, err := req.Stream(ctx) if err != nil { - return err + return nil, err } defer podLogs.Close() @@ -119,24 +125,26 @@ func (r *BenchmarkReport) getBenchmarkResultFromPodLogs(ctx context.Context, pod buf := new(bytes.Buffer) _, err = io.Copy(buf, podLogs) if err != nil { - return err + return nil, err } - r.RawResult = buf.Bytes() - return nil + return buf.Bytes(), nil } -func (r *BenchmarkReport) getMetricsFromPodPortForwarder(t *testing.T, pod *types.NamespacedName) error { +// getMetricsFromPortForwarder retrieves metrics from pod by request url `/metrics`. +func (r *BenchmarkReport) getMetricsFromPortForwarder(t *testing.T, pod *types.NamespacedName) ([]byte, error) { fw, err := kube.NewLocalPortForwarder(r.kubeClient, *pod, localMetricsPort, controlPlaneMetricsPort) if err != nil { - return fmt.Errorf("failed to build port forwarder for pod %s: %v", pod.String(), err) + return nil, fmt.Errorf("failed to build port forwarder for pod %s: %v", pod.String(), err) } if err = fw.Start(); err != nil { fw.Stop() - return fmt.Errorf("failed to start port forwarder for pod %s: %v", pod.String(), err) + + return nil, fmt.Errorf("failed to start port forwarder for pod %s: %v", pod.String(), err) } + var out []byte // Retrieving metrics from Pod. go func() { defer fw.Stop() @@ -154,10 +162,10 @@ func (r *BenchmarkReport) getMetricsFromPodPortForwarder(t *testing.T, pod *type return } - r.RawMetrics = metrics + out = metrics }() fw.WaitForStop() - return nil + return out, nil } From 526627a346983ab719e6dc22d8d8c452cd5b92ae Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Thu, 20 Jun 2024 20:22:41 +0800 Subject: [PATCH 19/30] add export benchmark report support Signed-off-by: shawnh2 --- go.mod | 2 +- test/benchmark/benchmark_test.go | 1 + test/benchmark/suite/flags.go | 9 +- test/benchmark/suite/render.go | 231 +++++++++++++++++++++++++++++++ test/benchmark/suite/suite.go | 27 +++- tools/make/kube.mk | 3 +- 6 files changed, 265 insertions(+), 8 deletions(-) create mode 100644 test/benchmark/suite/render.go diff --git a/go.mod b/go.mod index e3b7ca11d6b..68d1340bc45 100644 --- a/go.mod +++ b/go.mod @@ -164,7 +164,7 @@ require ( github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/client_model v0.6.1 github.com/prometheus/procfs v0.15.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect diff --git a/test/benchmark/benchmark_test.go b/test/benchmark/benchmark_test.go index 052bcf40a2c..7edbd215cec 100644 --- a/test/benchmark/benchmark_test.go +++ b/test/benchmark/benchmark_test.go @@ -45,6 +45,7 @@ func TestBenchmark(t *testing.T) { "config/gateway.yaml", "config/httproute.yaml", "config/nighthawk-client.yaml", + *suite.ReportSavePath, ) if err != nil { t.Fatalf("Failed to create BenchmarkTestSuite: %v", err) diff --git a/test/benchmark/suite/flags.go b/test/benchmark/suite/flags.go index 5b209a3392f..a07d2d4b010 100644 --- a/test/benchmark/suite/flags.go +++ b/test/benchmark/suite/flags.go @@ -11,8 +11,9 @@ package suite import "flag" var ( - RPS = flag.String("rps", "1000", "The target requests-per-second rate.") - Connections = flag.String("connections", "10", "The maximum allowed number of concurrent connections per event loop. HTTP/1 only.") - Duration = flag.String("duration", "60", "The number of seconds that the test should run.") - Concurrency = flag.String("concurrency", "auto", "The number of concurrent event loops that should be used.") + RPS = flag.String("rps", "1000", "The target requests-per-second rate.") + Connections = flag.String("connections", "10", "The maximum allowed number of concurrent connections per event loop. HTTP/1 only.") + Duration = flag.String("duration", "60", "The number of seconds that the test should run.") + Concurrency = flag.String("concurrency", "auto", "The number of concurrent event loops that should be used.") + ReportSavePath = flag.String("report-save-path", "", "The path where to save the benchmark test report.") ) diff --git a/test/benchmark/suite/render.go b/test/benchmark/suite/render.go new file mode 100644 index 00000000000..6d7717dd8f2 --- /dev/null +++ b/test/benchmark/suite/render.go @@ -0,0 +1,231 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build benchmark +// +build benchmark + +package suite + +import ( + "bytes" + "fmt" + "io" + "math" + "os" + "strconv" + "strings" + "text/tabwriter" + + prom "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" +) + +const ( + omitEmptyValue = "-" + benchmarkEnvPrefix = "BENCHMARK_" + + // Supported metric type. + metricTypeGauge = "gauge" + metricTypeCounter = "counter" + + // Supported metric unit. TODO: associate them with func map + metricUnitMiB = "MiB" + metricUnitSeconds = "Seconds" +) + +type ReportTableHeader struct { + Name string `json:"name"` + Metric *MetricEntry `json:"metric,omitempty"` + Result *ResultEntry `json:"result,omitempty"` +} + +type MetricEntry struct { + Name string `json:"name"` + Type string `json:"type"` + Unit string `json:"unit"` + Help string `json:"help,omitempty"` +} + +type ResultEntry struct { + // +} + +var ( + controlPlaneMetricHeaders = []*ReportTableHeader{ // TODO: convert and save this config with json + { + Name: "Benchmark Name", + }, + { + Name: "Envoy Gateway Memory", + Metric: &MetricEntry{ + Name: "process_resident_memory_bytes", + Type: metricTypeGauge, + Unit: metricUnitMiB, + }, + }, + { + Name: "Envoy Gateway Total CPU", + Metric: &MetricEntry{ + Name: "process_cpu_seconds_total", + Type: metricTypeCounter, + Unit: metricUnitSeconds, + }, + }, + } +) + +// RenderReport renders a report out of given list of benchmark report in Markdown format. +func RenderReport(writer io.Writer, name, description string, reports []*BenchmarkReport, titleLevel int) error { + writeSection(writer, name, titleLevel, description) + + writeSection(writer, "Results", titleLevel+1, "Click to see the full results.") + renderResultsTable(writer, reports) + + writeSection(writer, "Metrics", titleLevel+1, "") + err := renderMetricsTable(writer, controlPlaneMetricHeaders, reports) + if err != nil { + return err + } + + return nil +} + +// newMarkdownStyleTableWriter returns a tabwriter that write table in Markdown style. +func newMarkdownStyleTableWriter(writer io.Writer) *tabwriter.Writer { + return tabwriter.NewWriter(writer, 0, 0, 0, ' ', tabwriter.Debug) +} + +func renderEnvSettingsTable(writer io.Writer) { + table := newMarkdownStyleTableWriter(writer) + + headers := []string{"RPS", "Connections", "Duration", "CPU Limits", "Memory Limits"} + writeTableRow(table, headers, func(h string) string { + return h + }) + + writeTableDelimiter(table, len(headers)) + + writeTableRow(table, headers, func(h string) string { + if v, ok := os.LookupEnv(benchmarkEnvPrefix + strings.ToUpper(h)); ok { + return v + } + return omitEmptyValue + }) + + _ = table.Flush() +} + +func renderResultsTable(writer io.Writer, reports []*BenchmarkReport) { + // TODO: better processing these benchmark results. + for _, report := range reports { + writeCollapsibleSection(writer, report.Name, report.RawResult) + } +} + +func renderMetricsTable(writer io.Writer, headers []*ReportTableHeader, reports []*BenchmarkReport) error { + table := newMarkdownStyleTableWriter(writer) + + // Write metrics table header. + writeTableRow(table, headers, func(h *ReportTableHeader) string { // TODO: add footnote support + if h.Metric != nil && len(h.Metric.Unit) > 0 { + return fmt.Sprintf("%s (%s)", h.Name, h.Metric.Unit) + } + return h.Name + }) + + // Write metrics table delimiter. + writeTableDelimiter(table, len(headers)) + + // Write metrics table body. + for _, report := range reports { + mf, err := parseMetrics(report.RawCPMetrics) // TODO: move metrics outside, and add envoyproxy metrics support + if err != nil { + return err + } + + writeTableRow(table, headers, func(h *ReportTableHeader) string { + if h.Metric != nil { + if mv, ok := mf[h.Metric.Name]; ok { + // Store the help of metric for later usage. + h.Metric.Help = *mv.Help + + switch *mv.Type { + case prom.MetricType_GAUGE: + return strconv.FormatFloat(byteToMiB(*mv.Metric[0].Gauge.Value), 'f', -1, 64) + case prom.MetricType_COUNTER: + return strconv.FormatFloat(*mv.Metric[0].Counter.Value, 'f', -1, 64) + } + } + return omitEmptyValue + } + + // Despite metrics, we still got benchmark test name. + return report.Name + }) + } + + _ = table.Flush() + + return nil +} + +func byteToMiB(x float64) float64 { + return math.Round(x / (1024 * 1024)) +} + +// writeSection writes one section in Markdown style, content is optional. +func writeSection(writer io.Writer, title string, level int, content string) { + md := fmt.Sprintf("\n%s %s\n", strings.Repeat("#", level), title) + if len(content) > 0 { + md += fmt.Sprintf("\n%s\n", content) + } + _, _ = fmt.Fprintln(writer, md) +} + +// writeCollapsibleSection writes one collapsible section in Markdown style. +func writeCollapsibleSection(writer io.Writer, title string, content []byte) { + _, _ = fmt.Fprintln(writer, fmt.Sprintf(` +
+%s + +%s + +
`, title, fmt.Sprintf("```plaintext\n%s\n```", content))) +} + +// writeTableRow writes one row in Markdown table style. +func writeTableRow[T any](table *tabwriter.Writer, values []T, get func(T) string) { + row := "|" + for _, v := range values { + row += get(v) + "\t" + } + + _, _ = fmt.Fprintln(table, row) +} + +// writeTableDelimiter writes table delimiter in Markdown table style. +func writeTableDelimiter(table *tabwriter.Writer, n int) { + sep := "|" + for i := 0; i < n; i++ { + sep += "-\t" + } + + _, _ = fmt.Fprintln(table, sep) +} + +// parseMetrics parses input metrics that in Prometheus format. +func parseMetrics(metrics []byte) (map[string]*prom.MetricFamily, error) { + var ( + reader = bytes.NewReader(metrics) + parser expfmt.TextParser + ) + + mf, err := parser.TextToMetricFamilies(reader) + if err != nil { + return nil, err + } + + return mf, nil +} diff --git a/test/benchmark/suite/suite.go b/test/benchmark/suite/suite.go index 4841596e33a..e0262015dac 100644 --- a/test/benchmark/suite/suite.go +++ b/test/benchmark/suite/suite.go @@ -9,6 +9,7 @@ package suite import ( + "bytes" "context" "fmt" "os" @@ -37,6 +38,7 @@ type BenchmarkTestSuite struct { TimeoutConfig config.TimeoutConfig ControllerName string Options BenchmarkOptions + ReportSavePath string // Resources template for supported benchmark targets. GatewayTemplate *gwapiv1.Gateway @@ -48,7 +50,7 @@ type BenchmarkTestSuite struct { } func NewBenchmarkTestSuite(client client.Client, options BenchmarkOptions, - gatewayManifest, httpRouteManifest, benchmarkClientManifest string) (*BenchmarkTestSuite, error) { + gatewayManifest, httpRouteManifest, benchmarkClientManifest, reportPath string) (*BenchmarkTestSuite, error) { var ( gateway = new(gwapiv1.Gateway) httproute = new(gwapiv1.HTTPRoute) @@ -94,6 +96,7 @@ func NewBenchmarkTestSuite(client client.Client, options BenchmarkOptions, Options: options, TimeoutConfig: timeoutConfig, ControllerName: DefaultControllerName, + ReportSavePath: reportPath, GatewayTemplate: gateway, HTTPRouteTemplate: httproute, BenchmarkClientJob: benchmarkClient, @@ -106,6 +109,12 @@ func NewBenchmarkTestSuite(client client.Client, options BenchmarkOptions, func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { t.Logf("Running %d benchmark test", len(tests)) + buf := make([]byte, 0) + writer := bytes.NewBuffer(buf) + + writeSection(writer, "Benchmark Report", 1, "") + renderEnvSettingsTable(writer) + for _, test := range tests { t.Logf("Running benchmark test: %s", test.ShortName) @@ -114,8 +123,22 @@ func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { continue } - // TODO: Generate a human-readable benchmark report for each test. + // Generate a human-readable benchmark report for each test. t.Logf("Got %d reports for test: %s", len(reports), test.ShortName) + + if err := RenderReport(writer, "Test: "+test.ShortName, test.Description, reports, 2); err != nil { + t.Errorf("Error generating report for %s: %v", test.ShortName, err) + } + } + + if len(b.ReportSavePath) > 0 { + if err := os.WriteFile(b.ReportSavePath, writer.Bytes(), 0644); err != nil { + t.Errorf("Error writing report to path %s: %v", b.ReportSavePath, err) + } else { + t.Logf("Writing report to path %s successfully", b.ReportSavePath) + } + } else { + t.Log(fmt.Sprintf("%s", writer.Bytes())) } } diff --git a/tools/make/kube.mk b/tools/make/kube.mk index b1f9f83e6f7..11e48ed0817 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -16,6 +16,7 @@ BENCHMARK_MEMORY_LIMITS ?= 1024 # unit: 'Mi' BENCHMARK_RPS ?= 1000 BENCHMARK_CONNECTIONS ?= 100 BENCHMARK_DURATION ?= 60 +BENCHMARK_REPORT_PATH ?= benchmark_report.md FLUENT_BIT_CHART_VERSION ?= 0.30.4 OTEL_COLLECTOR_CHART_VERSION ?= 0.73.1 @@ -165,7 +166,7 @@ run-benchmark: install-benchmark-server ## Run benchmark tests kubectl wait --timeout=$(WAIT_TIMEOUT) -n benchmark-test deployment/nighthawk-test-server --for=condition=Available kubectl wait --timeout=$(WAIT_TIMEOUT) -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available kubectl apply -f test/benchmark/config/gatewayclass.yaml - go test -v -tags benchmark -timeout $(BENCHMARK_TIMEOUT) ./test/benchmark --rps=$(BENCHMARK_RPS) --connections=$(BENCHMARK_CONNECTIONS) --duration=$(BENCHMARK_DURATION) + go test -v -tags benchmark -timeout $(BENCHMARK_TIMEOUT) ./test/benchmark --rps=$(BENCHMARK_RPS) --connections=$(BENCHMARK_CONNECTIONS) --duration=$(BENCHMARK_DURATION) --report-save-path=$(BENCHMARK_REPORT_PATH) .PHONY: install-benchmark-server install-benchmark-server: ## Install nighthawk server for benchmark test From 4d874f2a77e78d6f5b1e40538a97db1311206747 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Thu, 20 Jun 2024 20:30:29 +0800 Subject: [PATCH 20/30] view benchmark report in github comment Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 4b07a99621f..0cbfe80239e 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -50,4 +50,21 @@ jobs: BENCHMARK_DURATION: ${{ github.event.inputs.duration || 90 }} BENCHMARK_CPU_LIMITS: ${{ github.event.inputs.cpu_limits || 1000 }} BENCHMARK_MEMORY_LIMITS: ${{ github.event.inputs.memory_limits || 2048 }} + BENCHMARK_REPORT_PATH: benchmark_report.md run: make benchmark + + - name: Read Benchmark results + id: result + uses: juliangruber/read-file-action@v1 + with: + path: ./benchmark_report.md + + - name: Comment + run: | + # Use GitHub API to create a comment on the PR + PR_NUMBER=${{ github.event.pull_request.number }} + COMMENT=${{ steps.result.outputs.content }} + GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} + COMMENT_URL="https://api.github.com/repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" + + curl -s -H "Authorization: token ${GITHUB_TOKEN}" -X POST $COMMENT_URL -d "{\"body\":\"$COMMENT\"}" From 891d2d4a59f9cf9a9e7a30e2e4b5f3c5f1058c65 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Fri, 21 Jun 2024 15:20:54 +0800 Subject: [PATCH 21/30] correct github comment ci Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 21 ++++++--------------- tools/make/kube.mk | 4 ++-- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 0cbfe80239e..eafab37eaef 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -32,6 +32,9 @@ on: type: string required: false +permissions: + pull-requests: write + jobs: benchmark-test: name: Benchmark Test @@ -50,21 +53,9 @@ jobs: BENCHMARK_DURATION: ${{ github.event.inputs.duration || 90 }} BENCHMARK_CPU_LIMITS: ${{ github.event.inputs.cpu_limits || 1000 }} BENCHMARK_MEMORY_LIMITS: ${{ github.event.inputs.memory_limits || 2048 }} - BENCHMARK_REPORT_PATH: benchmark_report.md run: make benchmark - - name: Read Benchmark results - id: result - uses: juliangruber/read-file-action@v1 + - name: Comment Benchmark results + uses: thollander/actions-comment-pull-request@v2 with: - path: ./benchmark_report.md - - - name: Comment - run: | - # Use GitHub API to create a comment on the PR - PR_NUMBER=${{ github.event.pull_request.number }} - COMMENT=${{ steps.result.outputs.content }} - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - COMMENT_URL="https://api.github.com/repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" - - curl -s -H "Authorization: token ${GITHUB_TOKEN}" -X POST $COMMENT_URL -d "{\"body\":\"$COMMENT\"}" + path: ./bin/benchmark/report.md diff --git a/tools/make/kube.mk b/tools/make/kube.mk index 11e48ed0817..9eeaeeb14dd 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -16,7 +16,6 @@ BENCHMARK_MEMORY_LIMITS ?= 1024 # unit: 'Mi' BENCHMARK_RPS ?= 1000 BENCHMARK_CONNECTIONS ?= 100 BENCHMARK_DURATION ?= 60 -BENCHMARK_REPORT_PATH ?= benchmark_report.md FLUENT_BIT_CHART_VERSION ?= 0.30.4 OTEL_COLLECTOR_CHART_VERSION ?= 0.73.1 @@ -163,10 +162,11 @@ endif .PHONY: run-benchmark run-benchmark: install-benchmark-server ## Run benchmark tests @$(LOG_TARGET) + mkdir -p $(OUTPUT_DIR)/benchmark kubectl wait --timeout=$(WAIT_TIMEOUT) -n benchmark-test deployment/nighthawk-test-server --for=condition=Available kubectl wait --timeout=$(WAIT_TIMEOUT) -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available kubectl apply -f test/benchmark/config/gatewayclass.yaml - go test -v -tags benchmark -timeout $(BENCHMARK_TIMEOUT) ./test/benchmark --rps=$(BENCHMARK_RPS) --connections=$(BENCHMARK_CONNECTIONS) --duration=$(BENCHMARK_DURATION) --report-save-path=$(BENCHMARK_REPORT_PATH) + go test -v -tags benchmark -timeout $(BENCHMARK_TIMEOUT) ./test/benchmark --rps=$(BENCHMARK_RPS) --connections=$(BENCHMARK_CONNECTIONS) --duration=$(BENCHMARK_DURATION) --report-save-path=$(OUTPUT_DIR)/benchmark/report.md .PHONY: install-benchmark-server install-benchmark-server: ## Install nighthawk server for benchmark test From a0e1497482fa5e4a786937fb6400087b5f8029f9 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Fri, 21 Jun 2024 15:23:42 +0800 Subject: [PATCH 22/30] fix env setting table Signed-off-by: shawnh2 --- test/benchmark/suite/render.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/test/benchmark/suite/render.go b/test/benchmark/suite/render.go index 6d7717dd8f2..08bd151ccb1 100644 --- a/test/benchmark/suite/render.go +++ b/test/benchmark/suite/render.go @@ -101,14 +101,16 @@ func renderEnvSettingsTable(writer io.Writer) { table := newMarkdownStyleTableWriter(writer) headers := []string{"RPS", "Connections", "Duration", "CPU Limits", "Memory Limits"} - writeTableRow(table, headers, func(h string) string { - return h + units := []string{"", "", "Seconds", "m", "MiB"} + writeTableRow(table, headers, func(i int, h string) string { + return fmt.Sprintf("%s (%s)", h, units[i]) }) writeTableDelimiter(table, len(headers)) - writeTableRow(table, headers, func(h string) string { - if v, ok := os.LookupEnv(benchmarkEnvPrefix + strings.ToUpper(h)); ok { + writeTableRow(table, headers, func(_ int, h string) string { + env := strings.Replace(strings.ToUpper(h), " ", "_", -1) + if v, ok := os.LookupEnv(benchmarkEnvPrefix + env); ok { return v } return omitEmptyValue @@ -128,7 +130,7 @@ func renderMetricsTable(writer io.Writer, headers []*ReportTableHeader, reports table := newMarkdownStyleTableWriter(writer) // Write metrics table header. - writeTableRow(table, headers, func(h *ReportTableHeader) string { // TODO: add footnote support + writeTableRow(table, headers, func(_ int, h *ReportTableHeader) string { // TODO: add footnote support if h.Metric != nil && len(h.Metric.Unit) > 0 { return fmt.Sprintf("%s (%s)", h.Name, h.Metric.Unit) } @@ -145,7 +147,7 @@ func renderMetricsTable(writer io.Writer, headers []*ReportTableHeader, reports return err } - writeTableRow(table, headers, func(h *ReportTableHeader) string { + writeTableRow(table, headers, func(_ int, h *ReportTableHeader) string { if h.Metric != nil { if mv, ok := mf[h.Metric.Name]; ok { // Store the help of metric for later usage. @@ -196,10 +198,10 @@ func writeCollapsibleSection(writer io.Writer, title string, content []byte) { } // writeTableRow writes one row in Markdown table style. -func writeTableRow[T any](table *tabwriter.Writer, values []T, get func(T) string) { +func writeTableRow[T any](table *tabwriter.Writer, values []T, get func(int, T) string) { row := "|" - for _, v := range values { - row += get(v) + "\t" + for i, v := range values { + row += get(i, v) + "\t" } _, _ = fmt.Fprintln(table, row) From d1a7cdec88d0aac436abbe513553c888567fa99c Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Fri, 21 Jun 2024 16:42:36 +0800 Subject: [PATCH 23/30] correct ci args Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 2 +- test/benchmark/suite/render.go | 7 +------ test/benchmark/suite/suite.go | 4 ++-- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index eafab37eaef..7a032137b2a 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -58,4 +58,4 @@ jobs: - name: Comment Benchmark results uses: thollander/actions-comment-pull-request@v2 with: - path: ./bin/benchmark/report.md + filePath: ./bin/benchmark/report.md diff --git a/test/benchmark/suite/render.go b/test/benchmark/suite/render.go index 08bd151ccb1..87dedfe563b 100644 --- a/test/benchmark/suite/render.go +++ b/test/benchmark/suite/render.go @@ -30,7 +30,7 @@ const ( metricTypeGauge = "gauge" metricTypeCounter = "counter" - // Supported metric unit. TODO: associate them with func map + // Supported metric unit. metricUnitMiB = "MiB" metricUnitSeconds = "Seconds" ) @@ -38,7 +38,6 @@ const ( type ReportTableHeader struct { Name string `json:"name"` Metric *MetricEntry `json:"metric,omitempty"` - Result *ResultEntry `json:"result,omitempty"` } type MetricEntry struct { @@ -48,10 +47,6 @@ type MetricEntry struct { Help string `json:"help,omitempty"` } -type ResultEntry struct { - // -} - var ( controlPlaneMetricHeaders = []*ReportTableHeader{ // TODO: convert and save this config with json { diff --git a/test/benchmark/suite/suite.go b/test/benchmark/suite/suite.go index e0262015dac..6bbd428ab42 100644 --- a/test/benchmark/suite/suite.go +++ b/test/benchmark/suite/suite.go @@ -133,9 +133,9 @@ func (b *BenchmarkTestSuite) Run(t *testing.T, tests []BenchmarkTest) { if len(b.ReportSavePath) > 0 { if err := os.WriteFile(b.ReportSavePath, writer.Bytes(), 0644); err != nil { - t.Errorf("Error writing report to path %s: %v", b.ReportSavePath, err) + t.Errorf("Error writing report to path '%s': %v", b.ReportSavePath, err) } else { - t.Logf("Writing report to path %s successfully", b.ReportSavePath) + t.Logf("Writing report to path '%s' successfully", b.ReportSavePath) } } else { t.Log(fmt.Sprintf("%s", writer.Bytes())) From 5322f0b07e7ceb3c26923d980c47b6f78c0205a1 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Fri, 21 Jun 2024 17:55:07 +0800 Subject: [PATCH 24/30] add benchmark report Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 5 - test/benchmark/benchmark_report.md | 910 +++++++++++++++++++++++++++++ test/benchmark/suite/render.go | 5 +- 3 files changed, 914 insertions(+), 6 deletions(-) create mode 100644 test/benchmark/benchmark_report.md diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 7a032137b2a..1507b41a425 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -54,8 +54,3 @@ jobs: BENCHMARK_CPU_LIMITS: ${{ github.event.inputs.cpu_limits || 1000 }} BENCHMARK_MEMORY_LIMITS: ${{ github.event.inputs.memory_limits || 2048 }} run: make benchmark - - - name: Comment Benchmark results - uses: thollander/actions-comment-pull-request@v2 - with: - filePath: ./bin/benchmark/report.md diff --git a/test/benchmark/benchmark_report.md b/test/benchmark/benchmark_report.md new file mode 100644 index 00000000000..4d21b70ebea --- /dev/null +++ b/test/benchmark/benchmark_report.md @@ -0,0 +1,910 @@ +# Benchmark Report + +|RPS ()|Connections ()|Duration (Seconds)|CPU Limits (m)|Memory Limits (MiB)| +|- |- |- |- |- | +|1000 |100 |90 |1000 |2048 | + +## Test: ScaleHTTPRoute + +Fixed one Gateway and different scales of HTTPRoutes. + + +### Results + +Click to see the full results. + + +
+scale-up-httproutes-10 + +```plaintext +[2024-06-21 09:22:45.839][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[09:22:45.840239][1][I] Detected 4 (v)CPUs with affinity.. +[09:22:45.840252][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[09:22:45.840255][1][I] Global targets: 400 connections and 4000 calls per second. +[09:22:45.840257][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[09:24:16.542128][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000 per second.) +[09:24:16.542378][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) +[09:24:16.542725][22][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.987788902321 per second.) +[09:24:16.542877][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) +[09:24:22.099712][22][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359873 samples) + min: 0s 000ms 313us | mean: 0s 000ms 519us | max: 0s 072ms 552us | pstdev: 0s 000ms 793us + + Percentile Count Value + 0.5 179946 0s 000ms 439us + 0.75 269912 0s 000ms 513us + 0.8 287903 0s 000ms 533us + 0.9 323889 0s 000ms 614us + 0.95 341884 0s 000ms 736us + 0.990625 356500 0s 001ms 559us + 0.99902344 359522 0s 007ms 985us + +Queueing and connection setup latency (359874 samples) + min: 0s 000ms 002us | mean: 0s 000ms 012us | max: 0s 063ms 952us | pstdev: 0s 000ms 131us + + Percentile Count Value + 0.5 180120 0s 000ms 010us + 0.75 270199 0s 000ms 011us + 0.8 288323 0s 000ms 011us + 0.9 324255 0s 000ms 011us + 0.95 341934 0s 000ms 012us + 0.990625 356501 0s 000ms 031us + 0.99902344 359523 0s 000ms 167us + +Request start to response end (359873 samples) + min: 0s 000ms 312us | mean: 0s 000ms 518us | max: 0s 072ms 552us | pstdev: 0s 000ms 793us + + Percentile Count Value + 0.5 179950 0s 000ms 438us + 0.75 269913 0s 000ms 512us + 0.8 287916 0s 000ms 533us + 0.9 323890 0s 000ms 614us + 0.95 341883 0s 000ms 736us + 0.990625 356500 0s 001ms 558us + 0.99902344 359522 0s 007ms 984us + +Response body size in bytes (359873 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359873 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Blocking. Results are skewed when significant numbers are reported here. (1 samples) + min: 0s 001ms 903us | mean: 0s 001ms 903us | max: 0s 001ms 903us | pstdev: 0s 000ms 000us + +Initiation to completion (359999 samples) + min: 0s 000ms 009us | mean: 0s 000ms 537us | max: 0s 072ms 634us | pstdev: 0s 000ms 841us + + Percentile Count Value + 0.5 180023 0s 000ms 455us + 0.75 270019 0s 000ms 530us + 0.8 288001 0s 000ms 551us + 0.9 324001 0s 000ms 634us + 0.95 342004 0s 000ms 755us + 0.990625 356625 0s 001ms 601us + 0.99902344 359648 0s 008ms 163us + +Counter Value Per second +benchmark.http_2xx 359873 3998.59 +benchmark.pool_overflow 126 1.40 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 102 1.13 +upstream_cx_rx_bytes_total 56500061 627778.28 +upstream_cx_total 102 1.13 +upstream_cx_tx_bytes_total 15474582 171939.75 +upstream_rq_pending_overflow 126 1.40 +upstream_rq_pending_total 102 1.13 +upstream_rq_total 359874 3998.60 + +[09:24:22.102587][1][I] Done. + +``` + +
+ +
+scale-up-httproutes-50 + +```plaintext +[2024-06-21 09:24:37.286][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[09:24:37.287391][1][I] Detected 4 (v)CPUs with affinity.. +[09:24:37.287403][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[09:24:37.287405][1][I] Global targets: 400 connections and 4000 calls per second. +[09:24:37.287406][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[09:26:07.989333][18][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) +[09:26:07.989702][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9986555573631 per second.) +[09:26:07.989935][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9988666679511 per second.) +[09:26:07.990125][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.988411116648 per second.) +[09:26:13.507511][23][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359812 samples) + min: 0s 000ms 309us | mean: 0s 000ms 509us | max: 0s 038ms 768us | pstdev: 0s 000ms 426us + + Percentile Count Value + 0.5 179913 0s 000ms 445us + 0.75 269869 0s 000ms 519us + 0.8 287867 0s 000ms 541us + 0.9 323837 0s 000ms 663us + 0.95 341824 0s 000ms 741us + 0.990625 356439 0s 001ms 300us + 0.99902344 359461 0s 005ms 292us + +Queueing and connection setup latency (359813 samples) + min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 029ms 871us | pstdev: 0s 000ms 075us + + Percentile Count Value + 0.5 180015 0s 000ms 010us + 0.75 270757 0s 000ms 011us + 0.8 287880 0s 000ms 011us + 0.9 323906 0s 000ms 011us + 0.95 341867 0s 000ms 012us + 0.990625 356440 0s 000ms 027us + 0.99902344 359462 0s 000ms 163us + +Request start to response end (359812 samples) + min: 0s 000ms 305us | mean: 0s 000ms 509us | max: 0s 038ms 768us | pstdev: 0s 000ms 426us + + Percentile Count Value + 0.5 179911 0s 000ms 445us + 0.75 269873 0s 000ms 519us + 0.8 287861 0s 000ms 540us + 0.9 323831 0s 000ms 662us + 0.95 341823 0s 000ms 741us + 0.990625 356439 0s 001ms 299us + 0.99902344 359461 0s 005ms 291us + +Response body size in bytes (359812 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359812 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Initiation to completion (359999 samples) + min: 0s 000ms 007us | mean: 0s 000ms 534us | max: 0s 038ms 785us | pstdev: 0s 000ms 618us + + Percentile Count Value + 0.5 180006 0s 000ms 462us + 0.75 270010 0s 000ms 537us + 0.8 288017 0s 000ms 558us + 0.9 324002 0s 000ms 682us + 0.95 342008 0s 000ms 760us + 0.990625 356625 0s 001ms 355us + 0.99902344 359648 0s 006ms 391us + +Counter Value Per second +benchmark.http_2xx 359812 3997.91 +benchmark.pool_overflow 187 2.08 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 56 0.62 +upstream_cx_rx_bytes_total 56490484 627671.58 +upstream_cx_total 56 0.62 +upstream_cx_tx_bytes_total 15471959 171910.53 +upstream_rq_pending_overflow 187 2.08 +upstream_rq_pending_total 56 0.62 +upstream_rq_total 359813 3997.92 + +[09:26:13.509310][1][I] Done. + +``` + +
+ +
+scale-up-httproutes-100 + +```plaintext +[2024-06-21 09:26:25.949][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[09:26:25.950098][1][I] Detected 4 (v)CPUs with affinity.. +[09:26:25.950110][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[09:26:25.950112][1][I] Global targets: 400 connections and 4000 calls per second. +[09:26:25.950114][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[09:27:56.651854][18][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) +[09:27:56.652350][21][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) +[09:27:56.652601][23][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000111111112 per second.) +[09:27:56.652103][19][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359853 samples) + min: 0s 000ms 314us | mean: 0s 000ms 516us | max: 0s 035ms 172us | pstdev: 0s 000ms 430us + + Percentile Count Value + 0.5 179943 0s 000ms 449us + 0.75 269895 0s 000ms 523us + 0.8 287893 0s 000ms 546us + 0.9 323868 0s 000ms 666us + 0.95 341865 0s 000ms 750us +[09:27:57.164977][1][I] Done. + 0.990625 356480 0s 001ms 430us + 0.99902344 359502 0s 005ms 762us + +Queueing and connection setup latency (359853 samples) + min: 0s 000ms 002us | mean: 0s 000ms 011us | max: 0s 026ms 753us | pstdev: 0s 000ms 060us + + Percentile Count Value + 0.5 180211 0s 000ms 010us + 0.75 270918 0s 000ms 011us + 0.8 288356 0s 000ms 011us + 0.9 323954 0s 000ms 011us + 0.95 341865 0s 000ms 012us + 0.990625 356481 0s 000ms 029us + 0.99902344 359502 0s 000ms 166us + +Request start to response end (359853 samples) + min: 0s 000ms 313us | mean: 0s 000ms 516us | max: 0s 035ms 172us | pstdev: 0s 000ms 430us + + Percentile Count Value + 0.5 179945 0s 000ms 448us + 0.75 269895 0s 000ms 523us + 0.8 287887 0s 000ms 545us + 0.9 323881 0s 000ms 666us + 0.95 341864 0s 000ms 749us + 0.990625 356480 0s 001ms 430us + 0.99902344 359502 0s 005ms 762us + +Response body size in bytes (359853 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359853 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Initiation to completion (360000 samples) + min: 0s 000ms 005us | mean: 0s 000ms 534us | max: 0s 035ms 190us | pstdev: 0s 000ms 447us + + Percentile Count Value + 0.5 180008 0s 000ms 466us + 0.75 270010 0s 000ms 540us + 0.8 288009 0s 000ms 563us + 0.9 324005 0s 000ms 686us + 0.95 342003 0s 000ms 769us + 0.990625 356625 0s 001ms 481us + 0.99902344 359649 0s 006ms 056us + +Counter Value Per second +benchmark.http_2xx 359853 3998.37 +benchmark.pool_overflow 147 1.63 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 64 0.71 +upstream_cx_rx_bytes_total 56496921 627743.57 +upstream_cx_total 64 0.71 +upstream_cx_tx_bytes_total 15473679 171929.77 +upstream_rq_pending_overflow 147 1.63 +upstream_rq_pending_total 64 0.71 +upstream_rq_total 359853 3998.37 + + +``` + +
+ +
+scale-up-httproutes-300 + +```plaintext +[2024-06-21 09:31:33.254][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[09:31:33.254741][1][I] Detected 4 (v)CPUs with affinity.. +[09:31:33.254754][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[09:31:33.254758][1][I] Global targets: 400 connections and 4000 calls per second. +[09:31:33.254760][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[09:33:03.956592][19][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) +[09:33:03.956838][20][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) +[09:33:03.957205][22][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.9876000159821 per second.) +[09:33:03.957340][25][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999777777783 per second.) +[09:33:09.465480][22][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359847 samples) + min: 0s 000ms 304us | mean: 0s 000ms 521us | max: 0s 024ms 524us | pstdev: 0s 000ms 424us + + Percentile Count Value + 0.5 179935 0s 000ms 454us + 0.75 269895 0s 000ms 527us + 0.8 287881 0s 000ms 548us + 0.9 323864 0s 000ms 662us + 0.95 341856 0s 000ms 754us + 0.990625 356474 0s 001ms 504us + 0.99902344 359496 0s 006ms 789us + +Queueing and connection setup latency (359848 samples) + min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 015ms 363us | pstdev: 0s 000ms 045us + + Percentile Count Value + 0.5 179968 0s 000ms 010us + 0.75 269895 0s 000ms 011us + 0.8 288448 0s 000ms 011us + 0.9 324002 0s 000ms 011us + 0.95 341858 0s 000ms 012us + 0.990625 356477 0s 000ms 029us + 0.99902344 359497 0s 000ms 167us + +Request start to response end (359847 samples) + min: 0s 000ms 304us | mean: 0s 000ms 521us | max: 0s 024ms 524us | pstdev: 0s 000ms 423us + + Percentile Count Value + 0.5 179945 0s 000ms 454us + 0.75 269914 0s 000ms 526us + 0.8 287893 0s 000ms 547us + 0.9 323864 0s 000ms 661us + 0.95 341856 0s 000ms 754us + 0.990625 356474 0s 001ms 503us + 0.99902344 359496 0s 006ms 786us + +Response body size in bytes (359847 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359847 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Initiation to completion (359999 samples) + min: 0s 000ms 006us | mean: 0s 000ms 538us | max: 0s 024ms 556us | pstdev: 0s 000ms 433us + + Percentile Count Value + 0.5 180015 0s 000ms 471us + 0.75 270008 0s 000ms 544us + 0.8 288021 0s 000ms 565us + 0.9 324004 0s 000ms 681us + 0.95 342002 0s 000ms 773us + 0.990625 356625 0s 001ms 545us + 0.99902344 359648 0s 006ms 943us + +Counter Value Per second +benchmark.http_2xx 359847 3998.30 +benchmark.pool_overflow 152 1.69 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 70 0.78 +upstream_cx_rx_bytes_total 56495979 627732.89 +upstream_cx_total 70 0.78 +upstream_cx_tx_bytes_total 15473464 171927.32 +upstream_rq_pending_overflow 152 1.69 +upstream_rq_pending_total 70 0.78 +upstream_rq_total 359848 3998.31 + +[09:33:09.468007][1][I] Done. + +``` + +
+ +
+scale-up-httproutes-500 + +```plaintext +[2024-06-21 09:38:56.778][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[09:38:56.779188][1][I] Detected 4 (v)CPUs with affinity.. +[09:38:56.779200][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[09:38:56.779202][1][I] Global targets: 400 connections and 4000 calls per second. +[09:38:56.779203][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[09:40:27.480937][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999777777783 per second.) +[09:40:27.481183][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) +[09:40:27.481532][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.99890000121 per second.) +[09:40:27.481728][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.988411116648 per second.) +[09:40:33.043298][23][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359807 samples) + min: 0s 000ms 310us | mean: 0s 000ms 560us | max: 0s 060ms 688us | pstdev: 0s 000ms 942us + + Percentile Count Value + 0.5 179905 0s 000ms 456us + 0.75 269868 0s 000ms 529us + 0.8 287867 0s 000ms 550us + 0.9 323830 0s 000ms 678us + 0.95 341818 0s 000ms 791us + 0.990625 356434 0s 002ms 103us + 0.99902344 359456 0s 012ms 998us + +Queueing and connection setup latency (359808 samples) + min: 0s 000ms 001us | mean: 0s 000ms 012us | max: 0s 027ms 772us | pstdev: 0s 000ms 104us + + Percentile Count Value + 0.5 180194 0s 000ms 010us + 0.75 270932 0s 000ms 011us + 0.8 288826 0s 000ms 011us + 0.9 324092 0s 000ms 011us + 0.95 341848 0s 000ms 012us + 0.990625 356435 0s 000ms 030us + 0.99902344 359457 0s 000ms 170us + +Request start to response end (359807 samples) + min: 0s 000ms 309us | mean: 0s 000ms 560us | max: 0s 060ms 688us | pstdev: 0s 000ms 942us + + Percentile Count Value + 0.5 179908 0s 000ms 456us + 0.75 269856 0s 000ms 529us + 0.8 287848 0s 000ms 550us + 0.9 323827 0s 000ms 677us + 0.95 341817 0s 000ms 791us + 0.990625 356434 0s 002ms 103us + 0.99902344 359456 0s 012ms 998us + +Response body size in bytes (359807 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359807 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Blocking. Results are skewed when significant numbers are reported here. (6 samples) + min: 0s 000ms 201us | mean: 0s 000ms 928us | max: 0s 002ms 801us | pstdev: 0s 000ms 880us + + Percentile Count Value + 0.5 3 0s 000ms 614us + 0.75 5 0s 000ms 914us + 0.8 5 0s 000ms 914us + +Initiation to completion (359999 samples) + min: 0s 000ms 006us | mean: 0s 000ms 581us | max: 0s 060ms 710us | pstdev: 0s 000ms 978us + + Percentile Count Value + 0.5 180010 0s 000ms 473us + 0.75 270021 0s 000ms 547us + 0.8 288019 0s 000ms 568us + 0.9 324004 0s 000ms 697us + 0.95 342000 0s 000ms 814us + 0.990625 356625 0s 002ms 210us + 0.99902344 359648 0s 014ms 801us + +Counter Value Per second +benchmark.http_2xx 359807 3997.85 +benchmark.pool_overflow 192 2.13 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 112 1.24 +upstream_cx_rx_bytes_total 56489699 627663.06 +upstream_cx_total 112 1.24 +upstream_cx_tx_bytes_total 15471744 171908.20 +upstream_rq_pending_overflow 192 2.13 +upstream_rq_pending_total 112 1.24 +upstream_rq_total 359808 3997.87 + +[09:40:33.045512][1][I] Done. + +``` + +
+ +
+scale-down-httproutes-300 + +```plaintext +[2024-06-21 09:41:00.346][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[09:41:00.347325][1][I] Detected 4 (v)CPUs with affinity.. +[09:41:00.347337][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[09:41:00.347339][1][I] Global targets: 400 connections and 4000 calls per second. +[09:41:00.347341][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[09:42:31.049165][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.999088889719 per second.) +[09:42:31.049603][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9997222222994 per second.) +[09:42:31.049727][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9981111146791 per second.) +[09:42:31.049828][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) +[09:42:31.683723][1][I] Done. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359812 samples) + min: 0s 000ms 302us | mean: 0s 001ms 445us | max: 0s 183ms 287us | pstdev: 0s 007ms 612us + + Percentile Count Value + 0.5 179921 0s 000ms 473us + 0.75 269867 0s 000ms 556us + 0.8 287850 0s 000ms 592us + 0.9 323832 0s 000ms 760us + 0.95 341822 0s 001ms 226us + 0.990625 356439 0s 043ms 878us + 0.99902344 359461 0s 095ms 641us + +Queueing and connection setup latency (359812 samples) + min: 0s 000ms 001us | mean: 0s 000ms 012us | max: 0s 027ms 984us | pstdev: 0s 000ms 113us + + Percentile Count Value + 0.5 180354 0s 000ms 010us + 0.75 270110 0s 000ms 011us + 0.8 288028 0s 000ms 011us + 0.9 324203 0s 000ms 011us + 0.95 341828 0s 000ms 012us + 0.990625 356439 0s 000ms 033us + 0.99902344 359461 0s 000ms 194us + +Request start to response end (359812 samples) + min: 0s 000ms 302us | mean: 0s 001ms 444us | max: 0s 183ms 287us | pstdev: 0s 007ms 612us + + Percentile Count Value + 0.5 179909 0s 000ms 472us + 0.75 269876 0s 000ms 555us + 0.8 287853 0s 000ms 592us + 0.9 323833 0s 000ms 759us + 0.95 341822 0s 001ms 226us + 0.990625 356439 0s 043ms 878us + 0.99902344 359461 0s 095ms 641us + +Response body size in bytes (359812 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359812 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Blocking. Results are skewed when significant numbers are reported here. (565 samples) + min: 0s 000ms 044us | mean: 0s 006ms 788us | max: 0s 095ms 117us | pstdev: 0s 015ms 888us + + Percentile Count Value + 0.5 283 0s 000ms 744us + 0.75 424 0s 002ms 919us + 0.8 452 0s 005ms 369us + 0.9 509 0s 018ms 845us + 0.95 537 0s 047ms 767us + 0.990625 560 0s 071ms 987us + +Initiation to completion (360000 samples) + min: 0s 000ms 007us | mean: 0s 001ms 468us | max: 0s 183ms 320us | pstdev: 0s 007ms 622us + + Percentile Count Value + 0.5 180014 0s 000ms 490us + 0.75 270017 0s 000ms 573us + 0.8 288000 0s 000ms 611us + 0.9 324002 0s 000ms 780us + 0.95 342000 0s 001ms 259us + 0.990625 356625 0s 043ms 876us + 0.99902344 359649 0s 095ms 735us + +Counter Value Per second +benchmark.http_2xx 359812 3997.91 +benchmark.pool_overflow 188 2.09 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 212 2.36 +upstream_cx_rx_bytes_total 56490484 627671.56 +upstream_cx_total 212 2.36 +upstream_cx_tx_bytes_total 15471916 171910.04 +upstream_rq_pending_overflow 188 2.09 +upstream_rq_pending_total 212 2.36 +upstream_rq_total 359812 3997.91 + + +``` + +
+ +
+scale-down-httproutes-100 + +```plaintext +[2024-06-21 09:42:53.953][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[09:42:53.954421][1][I] Detected 4 (v)CPUs with affinity.. +[09:42:53.954432][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[09:42:53.954435][1][I] Global targets: 400 connections and 4000 calls per second. +[09:42:53.954436][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[09:44:24.656476][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89998. (Completion rate was 999.9750111802468 per second.) +[09:44:24.656602][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89997. (Completion rate was 999.9653000474233 per second.) +[09:44:24.656822][21][I] Stopping after 90000 ms. Initiated: 89986 / Completed: 89977. (Completion rate was 999.7434335927504 per second.) +[09:44:24.657318][23][I] Stopping after 89999 ms. Initiated: 89997 / Completed: 89997. (Completion rate was 999.9674777513986 per second.) +[09:44:30.521193][18][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359688 samples) + min: 0s 000ms 308us | mean: 0s 009ms 727us | max: 0s 186ms 793us | pstdev: 0s 019ms 123us + + Percentile Count Value + 0.5 179844 0s 001ms 784us + 0.75 269768 0s 007ms 040us + 0.8 287752 0s 009ms 941us + 0.9 323723 0s 033ms 730us + 0.95 341704 0s 061ms 620us + 0.990625 356316 0s 083ms 664us + 0.99902344 359337 0s 110ms 288us + +Queueing and connection setup latency (359702 samples) + min: 0s 000ms 001us | mean: 0s 000ms 014us | max: 0s 049ms 557us | pstdev: 0s 000ms 258us + + Percentile Count Value + 0.5 180026 0s 000ms 008us + 0.75 269966 0s 000ms 010us + 0.8 287847 0s 000ms 010us + 0.9 323820 0s 000ms 011us + 0.95 341720 0s 000ms 018us + 0.990625 356330 0s 000ms 113us + 0.99902344 359351 0s 000ms 988us + +Request start to response end (359688 samples) + min: 0s 000ms 308us | mean: 0s 009ms 726us | max: 0s 186ms 793us | pstdev: 0s 019ms 123us + + Percentile Count Value + 0.5 179845 0s 001ms 783us + 0.75 269766 0s 007ms 039us + 0.8 287751 0s 009ms 940us + 0.9 323723 0s 033ms 730us + 0.95 341705 0s 061ms 620us + 0.990625 356316 0s 083ms 664us + 0.99902344 359337 0s 110ms 288us + +Response body size in bytes (359688 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359688 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Blocking. Results are skewed when significant numbers are reported here. (21695 samples) + min: 0s 000ms 035us | mean: 0s 004ms 952us | max: 0s 143ms 073us | pstdev: 0s 012ms 407us + + Percentile Count Value + 0.5 10848 0s 001ms 029us + 0.75 16272 0s 003ms 006us + 0.8 17356 0s 003ms 915us + 0.9 19526 0s 009ms 562us + 0.95 20611 0s 031ms 671us + 0.990625 21492 0s 065ms 878us + 0.99902344 21674 0s 083ms 513us + +Initiation to completion (359969 samples) + min: 0s 000ms 135us | mean: 0s 009ms 787us | max: 0s 186ms 834us | pstdev: 0s 019ms 151us + + Percentile Count Value + 0.5 179987 0s 001ms 820us + 0.75 269979 0s 007ms 143us + 0.8 287978 0s 010ms 083us + 0.9 323973 0s 033ms 902us + 0.95 341972 0s 061ms 700us + 0.990625 356596 0s 083ms 812us + 0.99902344 359618 0s 110ms 374us + +Counter Value Per second +benchmark.http_2xx 359688 3996.53 +benchmark.pool_overflow 281 3.12 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 119 1.32 +upstream_cx_rx_bytes_total 56471016 627455.05 +upstream_cx_total 119 1.32 +upstream_cx_tx_bytes_total 15467186 171857.44 +upstream_rq_pending_overflow 281 3.12 +upstream_rq_pending_total 119 1.32 +upstream_rq_total 359702 3996.68 + +[09:44:35.525681][19][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[09:44:40.533295][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[09:44:40.549139][1][I] Done. + +``` + +
+ +
+scale-down-httproutes-50 + +```plaintext +[2024-06-21 09:46:15.538][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[09:46:15.538712][1][I] Detected 4 (v)CPUs with affinity.. +[09:46:15.538725][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[09:46:15.538727][1][I] Global targets: 400 connections and 4000 calls per second. +[09:46:15.538729][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[09:47:46.240483][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) +[09:47:46.240994][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.988677780168 per second.) +[09:47:46.241223][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000 per second.) +[09:47:46.240809][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.987966677764 per second.) +[09:47:51.788782][19][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359809 samples) + min: 0s 000ms 298us | mean: 0s 000ms 544us | max: 0s 082ms 124us | pstdev: 0s 000ms 771us + + Percentile Count Value + 0.5 179920 0s 000ms 458us + 0.75 269873 0s 000ms 534us + 0.8 287849 0s 000ms 558us + 0.9 323834 0s 000ms 699us + 0.95 341819 0s 000ms 778us + 0.990625 356436 0s 001ms 748us + 0.99902344 359458 0s 009ms 125us + +Queueing and connection setup latency (359811 samples) + min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 019ms 383us | pstdev: 0s 000ms 048us + + Percentile Count Value + 0.5 180077 0s 000ms 010us + 0.75 271002 0s 000ms 011us + 0.8 288142 0s 000ms 011us + 0.9 323906 0s 000ms 011us + 0.95 341880 0s 000ms 012us + 0.990625 356438 0s 000ms 030us + 0.99902344 359460 0s 000ms 172us + +Request start to response end (359809 samples) + min: 0s 000ms 297us | mean: 0s 000ms 543us | max: 0s 082ms 124us | pstdev: 0s 000ms 771us + + Percentile Count Value + 0.5 179929 0s 000ms 458us + 0.75 269861 0s 000ms 534us + 0.8 287865 0s 000ms 558us + 0.9 323830 0s 000ms 698us + 0.95 341819 0s 000ms 778us + 0.990625 356436 0s 001ms 748us + 0.99902344 359458 0s 009ms 124us + +Response body size in bytes (359809 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359809 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Blocking. Results are skewed when significant numbers are reported here. (6 samples) + min: 0s 000ms 140us | mean: 0s 001ms 861us | max: 0s 003ms 722us | pstdev: 0s 001ms 492us + + Percentile Count Value + 0.5 3 0s 000ms 854us + 0.75 5 0s 003ms 630us + 0.8 5 0s 003ms 630us + +Initiation to completion (359998 samples) + min: 0s 000ms 007us | mean: 0s 000ms 562us | max: 0s 082ms 153us | pstdev: 0s 000ms 779us + + Percentile Count Value + 0.5 180001 0s 000ms 476us + 0.75 270017 0s 000ms 552us + 0.8 288016 0s 000ms 576us + 0.9 324003 0s 000ms 717us + 0.95 342000 0s 000ms 798us + 0.990625 356624 0s 001ms 806us + 0.99902344 359647 0s 009ms 598us + +Counter Value Per second +benchmark.http_2xx 359809 3997.88 +benchmark.pool_overflow 189 2.10 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 84 0.93 +upstream_cx_rx_bytes_total 56490013 627666.63 +upstream_cx_total 84 0.93 +upstream_cx_tx_bytes_total 15471873 171909.65 +upstream_rq_pending_overflow 189 2.10 +upstream_rq_pending_total 84 0.93 +upstream_rq_total 359811 3997.90 + +[09:47:56.790350][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[09:47:56.792949][1][I] Done. + +``` + +
+ +
+scale-down-httproutes-10 + +```plaintext +[2024-06-21 09:48:08.535][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[09:48:08.536097][1][I] Detected 4 (v)CPUs with affinity.. +[09:48:08.536111][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[09:48:08.536114][1][I] Global targets: 400 connections and 4000 calls per second. +[09:48:08.536116][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[09:49:39.238939][19][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000333333344 per second.) +[09:49:39.239235][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.988377783718 per second.) +[09:49:39.239520][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.987966677764 per second.) +[09:49:39.239658][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9892556709973 per second.) +[09:49:44.786959][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359744 samples) + min: 0s 000ms 300us | mean: 0s 000ms 552us | max: 0s 062ms 459us | pstdev: 0s 000ms 650us + + Percentile Count Value + 0.5 179903 0s 000ms 462us + 0.75 269816 0s 000ms 540us + 0.8 287814 0s 000ms 563us + 0.9 323771 0s 000ms 709us + 0.95 341759 0s 000ms 807us + 0.990625 356373 0s 001ms 984us + 0.99902344 359393 0s 008ms 610us + +Queueing and connection setup latency (359746 samples) + min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 017ms 278us | pstdev: 0s 000ms 043us + + Percentile Count Value + 0.5 179917 0s 000ms 010us + 0.75 270948 0s 000ms 011us + 0.8 288210 0s 000ms 011us + 0.9 323879 0s 000ms 011us + 0.95 341762 0s 000ms 012us + 0.990625 356374 0s 000ms 031us + 0.99902344 359395 0s 000ms 173us + +Request start to response end (359744 samples) + min: 0s 000ms 299us | mean: 0s 000ms 552us | max: 0s 062ms 459us | pstdev: 0s 000ms 650us + + Percentile Count Value + 0.5 179873 0s 000ms 461us + 0.75 269814 0s 000ms 540us + 0.8 287814 0s 000ms 562us + 0.9 323778 0s 000ms 709us + 0.95 341757 0s 000ms 807us + 0.990625 356372 0s 001ms 984us + 0.99902344 359393 0s 008ms 610us + +Response body size in bytes (359744 samples) + min: 10 | mean: 10 | max: 10 | pstdev: 0 + +Response header size in bytes (359744 samples) + min: 110 | mean: 110 | max: 110 | pstdev: 0 + +Blocking. Results are skewed when significant numbers are reported here. (9 samples) + min: 0s 000ms 088us | mean: 0s 000ms 567us | max: 0s 001ms 563us | pstdev: 0s 000ms 563us + + Percentile Count Value + 0.5 5 0s 000ms 257us + 0.75 7 0s 001ms 061us + 0.8 8 0s 001ms 402us + 0.9 9 0s 001ms 563us + +Initiation to completion (359998 samples) + min: 0s 000ms 006us | mean: 0s 000ms 570us | max: 0s 062ms 482us | pstdev: 0s 000ms 661us + + Percentile Count Value + 0.5 180035 0s 000ms 479us + 0.75 270019 0s 000ms 558us + 0.8 288010 0s 000ms 580us + 0.9 324007 0s 000ms 725us + 0.95 341999 0s 000ms 829us + 0.990625 356624 0s 002ms 038us + 0.99902344 359647 0s 008ms 940us + +Counter Value Per second +benchmark.http_2xx 359744 3997.14 +benchmark.pool_overflow 254 2.82 +cluster_manager.cluster_added 4 0.04 +default.total_match_count 4 0.04 +membership_change 4 0.04 +runtime.load_success 1 0.01 +runtime.override_dir_not_exists 1 0.01 +upstream_cx_http1_total 74 0.82 +upstream_cx_rx_bytes_total 56479808 627551.51 +upstream_cx_total 74 0.82 +upstream_cx_tx_bytes_total 15469078 171878.12 +upstream_rq_pending_overflow 254 2.82 +upstream_rq_pending_total 74 0.82 +upstream_rq_total 359746 3997.17 + +[09:49:49.788004][23][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[09:49:49.789745][1][I] Done. + +``` + +
+ +### Metrics + +|Benchmark Name |Envoy Gateway Memory (MiB)|Envoy Gateway Total CPU (Seconds)| +|- |- |- | +|scale-up-httproutes-10 |77 |0.35 | +|scale-up-httproutes-50 |90 |1.96 | +|scale-up-httproutes-100 |161 |9.97 | +|scale-up-httproutes-300 |883 |185.43 | +|scale-up-httproutes-500 |1473 |10.42 | +|scale-down-httproutes-300|697 |5.73 | +|scale-down-httproutes-100|463 |103.44 | +|scale-down-httproutes-50 |112 |161.11 | +|scale-down-httproutes-10 |116 |162.41 | diff --git a/test/benchmark/suite/render.go b/test/benchmark/suite/render.go index 87dedfe563b..146a897e1b2 100644 --- a/test/benchmark/suite/render.go +++ b/test/benchmark/suite/render.go @@ -98,7 +98,10 @@ func renderEnvSettingsTable(writer io.Writer) { headers := []string{"RPS", "Connections", "Duration", "CPU Limits", "Memory Limits"} units := []string{"", "", "Seconds", "m", "MiB"} writeTableRow(table, headers, func(i int, h string) string { - return fmt.Sprintf("%s (%s)", h, units[i]) + if len(units[i]) > 0 { + return fmt.Sprintf("%s (%s)", h, units[i]) + } + return h }) writeTableDelimiter(table, len(headers)) From f31f3f5c86c89649e80361808b35aaa697722289 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Fri, 21 Jun 2024 20:29:35 +0800 Subject: [PATCH 25/30] simplify the report Signed-off-by: shawnh2 --- test/benchmark/benchmark_report.md | 104 +---------------------------- 1 file changed, 3 insertions(+), 101 deletions(-) diff --git a/test/benchmark/benchmark_report.md b/test/benchmark/benchmark_report.md index 4d21b70ebea..35acb494132 100644 --- a/test/benchmark/benchmark_report.md +++ b/test/benchmark/benchmark_report.md @@ -1,8 +1,8 @@ # Benchmark Report -|RPS ()|Connections ()|Duration (Seconds)|CPU Limits (m)|Memory Limits (MiB)| -|- |- |- |- |- | -|1000 |100 |90 |1000 |2048 | +| RPS | Connections | Duration (Seconds) | CPU Limits (m) | Memory Limits (MiB) | +|------|-------------|--------------------|----------------|---------------------| +| 1000 | 100 | 90 | 1000 | 2048 | ## Test: ScaleHTTPRoute @@ -18,17 +18,6 @@ Click to see the full results. scale-up-httproutes-10 ```plaintext -[2024-06-21 09:22:45.839][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. -[09:22:45.840239][1][I] Detected 4 (v)CPUs with affinity.. -[09:22:45.840252][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. -[09:22:45.840255][1][I] Global targets: 400 connections and 4000 calls per second. -[09:22:45.840257][1][I] (Per-worker targets: 100 connections and 1000 calls per second) -[09:24:16.542128][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000 per second.) -[09:24:16.542378][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) -[09:24:16.542725][22][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.987788902321 per second.) -[09:24:16.542877][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) -[09:24:22.099712][22][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. -Nighthawk - A layer 7 protocol benchmarking tool. benchmark_http_client.latency_2xx (359873 samples) min: 0s 000ms 313us | mean: 0s 000ms 519us | max: 0s 072ms 552us | pstdev: 0s 000ms 793us @@ -113,17 +102,6 @@ upstream_rq_total 359874 3998.60 scale-up-httproutes-50 ```plaintext -[2024-06-21 09:24:37.286][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. -[09:24:37.287391][1][I] Detected 4 (v)CPUs with affinity.. -[09:24:37.287403][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. -[09:24:37.287405][1][I] Global targets: 400 connections and 4000 calls per second. -[09:24:37.287406][1][I] (Per-worker targets: 100 connections and 1000 calls per second) -[09:26:07.989333][18][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) -[09:26:07.989702][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9986555573631 per second.) -[09:26:07.989935][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9988666679511 per second.) -[09:26:07.990125][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.988411116648 per second.) -[09:26:13.507511][23][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. -Nighthawk - A layer 7 protocol benchmarking tool. benchmark_http_client.latency_2xx (359812 samples) min: 0s 000ms 309us | mean: 0s 000ms 509us | max: 0s 038ms 768us | pstdev: 0s 000ms 426us @@ -205,16 +183,6 @@ upstream_rq_total 359813 3997.92 scale-up-httproutes-100 ```plaintext -[2024-06-21 09:26:25.949][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. -[09:26:25.950098][1][I] Detected 4 (v)CPUs with affinity.. -[09:26:25.950110][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. -[09:26:25.950112][1][I] Global targets: 400 connections and 4000 calls per second. -[09:26:25.950114][1][I] (Per-worker targets: 100 connections and 1000 calls per second) -[09:27:56.651854][18][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) -[09:27:56.652350][21][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) -[09:27:56.652601][23][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000111111112 per second.) -[09:27:56.652103][19][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) -Nighthawk - A layer 7 protocol benchmarking tool. benchmark_http_client.latency_2xx (359853 samples) min: 0s 000ms 314us | mean: 0s 000ms 516us | max: 0s 035ms 172us | pstdev: 0s 000ms 430us @@ -296,17 +264,6 @@ upstream_rq_total 359853 3998.37 scale-up-httproutes-300 ```plaintext -[2024-06-21 09:31:33.254][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. -[09:31:33.254741][1][I] Detected 4 (v)CPUs with affinity.. -[09:31:33.254754][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. -[09:31:33.254758][1][I] Global targets: 400 connections and 4000 calls per second. -[09:31:33.254760][1][I] (Per-worker targets: 100 connections and 1000 calls per second) -[09:33:03.956592][19][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) -[09:33:03.956838][20][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) -[09:33:03.957205][22][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.9876000159821 per second.) -[09:33:03.957340][25][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999777777783 per second.) -[09:33:09.465480][22][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. -Nighthawk - A layer 7 protocol benchmarking tool. benchmark_http_client.latency_2xx (359847 samples) min: 0s 000ms 304us | mean: 0s 000ms 521us | max: 0s 024ms 524us | pstdev: 0s 000ms 424us @@ -388,17 +345,6 @@ upstream_rq_total 359848 3998.31 scale-up-httproutes-500 ```plaintext -[2024-06-21 09:38:56.778][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. -[09:38:56.779188][1][I] Detected 4 (v)CPUs with affinity.. -[09:38:56.779200][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. -[09:38:56.779202][1][I] Global targets: 400 connections and 4000 calls per second. -[09:38:56.779203][1][I] (Per-worker targets: 100 connections and 1000 calls per second) -[09:40:27.480937][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999777777783 per second.) -[09:40:27.481183][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) -[09:40:27.481532][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.99890000121 per second.) -[09:40:27.481728][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.988411116648 per second.) -[09:40:33.043298][23][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. -Nighthawk - A layer 7 protocol benchmarking tool. benchmark_http_client.latency_2xx (359807 samples) min: 0s 000ms 310us | mean: 0s 000ms 560us | max: 0s 060ms 688us | pstdev: 0s 000ms 942us @@ -488,17 +434,6 @@ upstream_rq_total 359808 3997.87 scale-down-httproutes-300 ```plaintext -[2024-06-21 09:41:00.346][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. -[09:41:00.347325][1][I] Detected 4 (v)CPUs with affinity.. -[09:41:00.347337][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. -[09:41:00.347339][1][I] Global targets: 400 connections and 4000 calls per second. -[09:41:00.347341][1][I] (Per-worker targets: 100 connections and 1000 calls per second) -[09:42:31.049165][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.999088889719 per second.) -[09:42:31.049603][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9997222222994 per second.) -[09:42:31.049727][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9981111146791 per second.) -[09:42:31.049828][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) -[09:42:31.683723][1][I] Done. -Nighthawk - A layer 7 protocol benchmarking tool. benchmark_http_client.latency_2xx (359812 samples) min: 0s 000ms 302us | mean: 0s 001ms 445us | max: 0s 183ms 287us | pstdev: 0s 007ms 612us @@ -590,17 +525,6 @@ upstream_rq_total 359812 3997.91 scale-down-httproutes-100 ```plaintext -[2024-06-21 09:42:53.953][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. -[09:42:53.954421][1][I] Detected 4 (v)CPUs with affinity.. -[09:42:53.954432][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. -[09:42:53.954435][1][I] Global targets: 400 connections and 4000 calls per second. -[09:42:53.954436][1][I] (Per-worker targets: 100 connections and 1000 calls per second) -[09:44:24.656476][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89998. (Completion rate was 999.9750111802468 per second.) -[09:44:24.656602][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89997. (Completion rate was 999.9653000474233 per second.) -[09:44:24.656822][21][I] Stopping after 90000 ms. Initiated: 89986 / Completed: 89977. (Completion rate was 999.7434335927504 per second.) -[09:44:24.657318][23][I] Stopping after 89999 ms. Initiated: 89997 / Completed: 89997. (Completion rate was 999.9674777513986 per second.) -[09:44:30.521193][18][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. -Nighthawk - A layer 7 protocol benchmarking tool. benchmark_http_client.latency_2xx (359688 samples) min: 0s 000ms 308us | mean: 0s 009ms 727us | max: 0s 186ms 793us | pstdev: 0s 019ms 123us @@ -696,17 +620,6 @@ upstream_rq_total 359702 3996.68 scale-down-httproutes-50 ```plaintext -[2024-06-21 09:46:15.538][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. -[09:46:15.538712][1][I] Detected 4 (v)CPUs with affinity.. -[09:46:15.538725][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. -[09:46:15.538727][1][I] Global targets: 400 connections and 4000 calls per second. -[09:46:15.538729][1][I] (Per-worker targets: 100 connections and 1000 calls per second) -[09:47:46.240483][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) -[09:47:46.240994][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.988677780168 per second.) -[09:47:46.241223][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000 per second.) -[09:47:46.240809][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.987966677764 per second.) -[09:47:51.788782][19][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. -Nighthawk - A layer 7 protocol benchmarking tool. benchmark_http_client.latency_2xx (359809 samples) min: 0s 000ms 298us | mean: 0s 000ms 544us | max: 0s 082ms 124us | pstdev: 0s 000ms 771us @@ -797,17 +710,6 @@ upstream_rq_total 359811 3997.90 scale-down-httproutes-10 ```plaintext -[2024-06-21 09:48:08.535][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. -[09:48:08.536097][1][I] Detected 4 (v)CPUs with affinity.. -[09:48:08.536111][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. -[09:48:08.536114][1][I] Global targets: 400 connections and 4000 calls per second. -[09:48:08.536116][1][I] (Per-worker targets: 100 connections and 1000 calls per second) -[09:49:39.238939][19][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000333333344 per second.) -[09:49:39.239235][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.988377783718 per second.) -[09:49:39.239520][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.987966677764 per second.) -[09:49:39.239658][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9892556709973 per second.) -[09:49:44.786959][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. -Nighthawk - A layer 7 protocol benchmarking tool. benchmark_http_client.latency_2xx (359744 samples) min: 0s 000ms 300us | mean: 0s 000ms 552us | max: 0s 062ms 459us | pstdev: 0s 000ms 650us From ed0732972ed91db4d19841c47a57576a08f022e5 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Tue, 25 Jun 2024 10:28:47 +0800 Subject: [PATCH 26/30] add pull_request_target to benchmark report commenter Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 1507b41a425..cd8a166b193 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -4,6 +4,10 @@ on: branches: - "main" - "release/v*" + pull_request_target: + branches: + - "main" + - "release/v*" workflow_dispatch: inputs: rps: @@ -54,3 +58,9 @@ jobs: BENCHMARK_CPU_LIMITS: ${{ github.event.inputs.cpu_limits || 1000 }} BENCHMARK_MEMORY_LIMITS: ${{ github.event.inputs.memory_limits || 2048 }} run: make benchmark + + - name: Comment Benchmark report + uses: thollander/actions-comment-pull-request@v2 + with: + filePath: ./bin/benchmark/report.md + comment_tag: benchmark-report From 3fad16c3033b4420884e582808031d7552a8a13e Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Tue, 25 Jun 2024 19:32:34 +0800 Subject: [PATCH 27/30] add envoyproxy metrics report and render support Signed-off-by: shawnh2 --- test/benchmark/suite/render.go | 212 ++++++++++++++++++++++++--------- test/benchmark/suite/report.go | 60 +++++++++- test/benchmark/suite/suite.go | 5 +- 3 files changed, 210 insertions(+), 67 deletions(-) diff --git a/test/benchmark/suite/render.go b/test/benchmark/suite/render.go index 146a897e1b2..e3c059685f1 100644 --- a/test/benchmark/suite/render.go +++ b/test/benchmark/suite/render.go @@ -20,6 +20,7 @@ import ( prom "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" + "k8s.io/apimachinery/pkg/util/sets" ) const ( @@ -31,55 +32,71 @@ const ( metricTypeCounter = "counter" // Supported metric unit. - metricUnitMiB = "MiB" - metricUnitSeconds = "Seconds" + metricUnitMiB = "MiB" + metricUnitSeconds = "Seconds" + metricUnitMilliCPU = "m" ) type ReportTableHeader struct { - Name string `json:"name"` - Metric *MetricEntry `json:"metric,omitempty"` + Name string + Metric *MetricEntry + + // Underlying name of one envoy-proxy, used by data-plane metrics. + ProxyName string } type MetricEntry struct { - Name string `json:"name"` - Type string `json:"type"` - Unit string `json:"unit"` - Help string `json:"help,omitempty"` + Name string + Type string + FromControlPlane bool + DisplayUnit string + ConvertUnit func(float64) float64 } -var ( - controlPlaneMetricHeaders = []*ReportTableHeader{ // TODO: convert and save this config with json +// RenderReport renders a report out of given list of benchmark report in Markdown format. +func RenderReport(writer io.Writer, name, description string, reports []*BenchmarkReport, titleLevel int) error { + headerSettings := []ReportTableHeader{ { Name: "Benchmark Name", }, { Name: "Envoy Gateway Memory", Metric: &MetricEntry{ - Name: "process_resident_memory_bytes", - Type: metricTypeGauge, - Unit: metricUnitMiB, + Name: "process_resident_memory_bytes", + Type: metricTypeGauge, + DisplayUnit: metricUnitMiB, + FromControlPlane: true, + ConvertUnit: byteToMiB, }, }, { Name: "Envoy Gateway Total CPU", Metric: &MetricEntry{ - Name: "process_cpu_seconds_total", - Type: metricTypeCounter, - Unit: metricUnitSeconds, + Name: "process_cpu_seconds_total", + Type: metricTypeCounter, + DisplayUnit: metricUnitSeconds, + FromControlPlane: true, + }, + }, + { + Name: "Envoy Proxy Memory", + Metric: &MetricEntry{ + Name: "envoy_server_memory_allocated", + Type: metricTypeGauge, + DisplayUnit: metricUnitMiB, + FromControlPlane: false, + ConvertUnit: byteToMiB, }, }, } -) -// RenderReport renders a report out of given list of benchmark report in Markdown format. -func RenderReport(writer io.Writer, name, description string, reports []*BenchmarkReport, titleLevel int) error { writeSection(writer, name, titleLevel, description) writeSection(writer, "Results", titleLevel+1, "Click to see the full results.") renderResultsTable(writer, reports) writeSection(writer, "Metrics", titleLevel+1, "") - err := renderMetricsTable(writer, controlPlaneMetricHeaders, reports) + err := renderMetricsTable(writer, headerSettings, reports) if err != nil { return err } @@ -93,21 +110,41 @@ func newMarkdownStyleTableWriter(writer io.Writer) *tabwriter.Writer { } func renderEnvSettingsTable(writer io.Writer) { + _, _ = fmt.Fprintln(writer, "Benchmark test settings:", "\n") + table := newMarkdownStyleTableWriter(writer) - headers := []string{"RPS", "Connections", "Duration", "CPU Limits", "Memory Limits"} - units := []string{"", "", "Seconds", "m", "MiB"} - writeTableRow(table, headers, func(i int, h string) string { - if len(units[i]) > 0 { - return fmt.Sprintf("%s (%s)", h, units[i]) - } - return h - }) + headers := []ReportTableHeader{ + { + Name: "RPS", + }, + { + Name: "Connections", + }, + { + Name: "Duration", + Metric: &MetricEntry{ + DisplayUnit: metricUnitSeconds, + }, + }, + { + Name: "CPU Limits", + Metric: &MetricEntry{ + DisplayUnit: metricUnitMilliCPU, + }, + }, + { + Name: "Memory Limits", + Metric: &MetricEntry{ + DisplayUnit: metricUnitMiB, + }, + }, + } - writeTableDelimiter(table, len(headers)) + renderMetricsTableHeader(table, headers) - writeTableRow(table, headers, func(_ int, h string) string { - env := strings.Replace(strings.ToUpper(h), " ", "_", -1) + writeTableRow(table, headers, func(_ int, h ReportTableHeader) string { + env := strings.Replace(strings.ToUpper(h.Name), " ", "_", -1) if v, ok := os.LookupEnv(benchmarkEnvPrefix + env); ok { return v } @@ -124,53 +161,90 @@ func renderResultsTable(writer io.Writer, reports []*BenchmarkReport) { } } -func renderMetricsTable(writer io.Writer, headers []*ReportTableHeader, reports []*BenchmarkReport) error { +func renderMetricsTable(writer io.Writer, headerSettings []ReportTableHeader, reports []*BenchmarkReport) error { table := newMarkdownStyleTableWriter(writer) - // Write metrics table header. - writeTableRow(table, headers, func(_ int, h *ReportTableHeader) string { // TODO: add footnote support - if h.Metric != nil && len(h.Metric.Unit) > 0 { - return fmt.Sprintf("%s (%s)", h.Name, h.Metric.Unit) + // Preprocess the table header for metrics table. + var headers []ReportTableHeader + // 1. Collect all the possible proxy names. + proxyNames := sets.NewString() + for _, report := range reports { + for name := range report.RawDPMetrics { + proxyNames.Insert(name) } - return h.Name - }) + } + // 2. Generate header names for data-plane proxies. + for _, hs := range headerSettings { + if hs.Metric != nil && !hs.Metric.FromControlPlane { + for i, proxyName := range proxyNames.List() { + names := strings.Split(proxyName, "-") + headers = append(headers, ReportTableHeader{ + Name: fmt.Sprintf("%s: %s[%d]", hs.Name, names[len(names)-1], i+1), + Metric: hs.Metric, + ProxyName: proxyName, + }) + } + } else { + // For control-plane metrics or plain header. + headers = append(headers, hs) + } + } - // Write metrics table delimiter. - writeTableDelimiter(table, len(headers)) + renderMetricsTableHeader(table, headers) - // Write metrics table body. for _, report := range reports { - mf, err := parseMetrics(report.RawCPMetrics) // TODO: move metrics outside, and add envoyproxy metrics support + mfCP, err := parseMetrics(report.RawCPMetrics) if err != nil { return err } - writeTableRow(table, headers, func(_ int, h *ReportTableHeader) string { - if h.Metric != nil { - if mv, ok := mf[h.Metric.Name]; ok { - // Store the help of metric for later usage. - h.Metric.Help = *mv.Help - - switch *mv.Type { - case prom.MetricType_GAUGE: - return strconv.FormatFloat(byteToMiB(*mv.Metric[0].Gauge.Value), 'f', -1, 64) - case prom.MetricType_COUNTER: - return strconv.FormatFloat(*mv.Metric[0].Counter.Value, 'f', -1, 64) - } + mfDPs := make(map[string]map[string]*prom.MetricFamily, len(report.RawDPMetrics)) + for dpName, dpMetrics := range report.RawDPMetrics { + mfDP, err := parseMetrics(dpMetrics) + if err != nil { + return err + } + mfDPs[dpName] = mfDP + } + + writeTableRow(table, headers, func(_ int, h ReportTableHeader) string { + if h.Metric == nil { + return report.Name + } + + if h.Metric.FromControlPlane { + return processMetricValue(mfCP, h) + } else { + if mfDP, ok := mfDPs[h.ProxyName]; ok { + return processMetricValue(mfDP, h) } - return omitEmptyValue } - // Despite metrics, we still got benchmark test name. - return report.Name + return omitEmptyValue }) } _ = table.Flush() + // Generate footnotes for envoy-proxy headers. + for i, proxyName := range proxyNames.List() { + _, _ = fmt.Fprintln(writer, fmt.Sprintf("%d.", i+1), proxyName) + } + return nil } +func renderMetricsTableHeader(table *tabwriter.Writer, headers []ReportTableHeader) { + writeTableRow(table, headers, func(_ int, h ReportTableHeader) string { + if h.Metric != nil && len(h.Metric.DisplayUnit) > 0 { + return fmt.Sprintf("%s (%s)", h.Name, h.Metric.DisplayUnit) + } + return h.Name + }) + + writeTableDelimiter(table, len(headers)) +} + func byteToMiB(x float64) float64 { return math.Round(x / (1024 * 1024)) } @@ -229,3 +303,27 @@ func parseMetrics(metrics []byte) (map[string]*prom.MetricFamily, error) { return mf, nil } + +// processMetricValue process one metric value according to the given header and metric families. +func processMetricValue(metricFamilies map[string]*prom.MetricFamily, header ReportTableHeader) string { + if mf, ok := metricFamilies[header.Metric.Name]; ok { + var value float64 + + switch header.Metric.Type { + case metricTypeGauge: + value = *mf.Metric[0].Gauge.Value + case metricTypeCounter: + value = *mf.Metric[0].Counter.Value + default: + return omitEmptyValue + } + + if header.Metric.ConvertUnit != nil { + value = header.Metric.ConvertUnit(value) + } + + return strconv.FormatFloat(value, 'f', -1, 64) + } + + return omitEmptyValue +} diff --git a/test/benchmark/suite/report.go b/test/benchmark/suite/report.go index ec62be16ba5..949fb5fa8d2 100644 --- a/test/benchmark/suite/report.go +++ b/test/benchmark/suite/report.go @@ -33,6 +33,7 @@ type BenchmarkReport struct { Name string RawResult []byte RawCPMetrics []byte + RawDPMetrics map[string][]byte kubeClient kube.CLIClient } @@ -44,8 +45,9 @@ func NewBenchmarkReport(name string) (*BenchmarkReport, error) { } return &BenchmarkReport{ - Name: name, - kubeClient: kubeClient, + Name: name, + RawDPMetrics: make(map[string][]byte), + kubeClient: kubeClient, }, nil } @@ -55,6 +57,26 @@ func (r *BenchmarkReport) Print(t *testing.T, name string) { t.Logf("=== Benchmark Result: \n\n %s \n\n", r.RawResult) t.Logf("=== Control-Plane Metrics: \n\n %s \n\n", r.RawCPMetrics) + + for dpName, dpMetrics := range r.RawDPMetrics { + t.Logf("=== Data-Plane Metrics for %s: \n\n %s \n\n", dpName, dpMetrics) + } +} + +func (r *BenchmarkReport) Collect(t *testing.T, ctx context.Context, job *types.NamespacedName) error { + if err := r.GetBenchmarkResult(t, ctx, job); err != nil { + return err + } + + if err := r.GetControlPlaneMetrics(t, ctx); err != nil { + return err + } + + if err := r.GetDataPlaneMetrics(t, ctx); err != nil { + return err + } + + return nil } func (r *BenchmarkReport) GetBenchmarkResult(t *testing.T, ctx context.Context, job *types.NamespacedName) error { @@ -99,7 +121,7 @@ func (r *BenchmarkReport) GetControlPlaneMetrics(t *testing.T, ctx context.Conte egPod := &egPods.Items[0] metrics, err := r.getMetricsFromPortForwarder( - t, &types.NamespacedName{Name: egPod.Name, Namespace: egPod.Namespace}, + t, &types.NamespacedName{Name: egPod.Name, Namespace: egPod.Namespace}, "/metrics", ) if err != nil { return err @@ -110,6 +132,32 @@ func (r *BenchmarkReport) GetControlPlaneMetrics(t *testing.T, ctx context.Conte return nil } +func (r *BenchmarkReport) GetDataPlaneMetrics(t *testing.T, ctx context.Context) error { + epPods, err := r.kubeClient.Kube().CoreV1().Pods("envoy-gateway-system"). + List(ctx, metav1.ListOptions{LabelSelector: "gateway.envoyproxy.io/owning-gateway-namespace=benchmark-test,gateway.envoyproxy.io/owning-gateway-name=benchmark"}) + if err != nil { + return err + } + + if len(epPods.Items) < 1 { + return fmt.Errorf("failed to get any pods for envoy-proxies") + } + + t.Logf("Got %d pod(s) from data-plane", len(epPods.Items)) + + for _, epPod := range epPods.Items { + podNN := &types.NamespacedName{Name: epPod.Name, Namespace: epPod.Namespace} + metrics, err := r.getMetricsFromPortForwarder(t, podNN, "/stats/prometheus") + if err != nil { + return err + } + + r.RawDPMetrics[podNN.String()] = metrics + } + + return nil +} + // getLogsFromPod scrapes the logs directly from the pod (default container). func (r *BenchmarkReport) getLogsFromPod(ctx context.Context, pod *types.NamespacedName) ([]byte, error) { podLogOpts := corev1.PodLogOptions{} @@ -131,8 +179,8 @@ func (r *BenchmarkReport) getLogsFromPod(ctx context.Context, pod *types.Namespa return buf.Bytes(), nil } -// getMetricsFromPortForwarder retrieves metrics from pod by request url `/metrics`. -func (r *BenchmarkReport) getMetricsFromPortForwarder(t *testing.T, pod *types.NamespacedName) ([]byte, error) { +// getMetricsFromPortForwarder retrieves metrics from pod by request url, like `/metrics`. +func (r *BenchmarkReport) getMetricsFromPortForwarder(t *testing.T, pod *types.NamespacedName, url string) ([]byte, error) { fw, err := kube.NewLocalPortForwarder(r.kubeClient, *pod, localMetricsPort, controlPlaneMetricsPort) if err != nil { return nil, fmt.Errorf("failed to build port forwarder for pod %s: %v", pod.String(), err) @@ -149,7 +197,7 @@ func (r *BenchmarkReport) getMetricsFromPortForwarder(t *testing.T, pod *types.N go func() { defer fw.Stop() - url := fmt.Sprintf("http://%s/metrics", fw.Address()) + url := fmt.Sprintf("http://%s%s", fw.Address(), url) resp, err := http.Get(url) if err != nil { t.Errorf("failed to request %s: %v", url, err) diff --git a/test/benchmark/suite/suite.go b/test/benchmark/suite/suite.go index 6bbd428ab42..919e019e63a 100644 --- a/test/benchmark/suite/suite.go +++ b/test/benchmark/suite/suite.go @@ -196,10 +196,7 @@ func (b *BenchmarkTestSuite) Benchmark(t *testing.T, ctx context.Context, name, } // Get all the reports from this benchmark test run. - if err = report.GetBenchmarkResult(t, ctx, jobNN); err != nil { - return nil, err - } - if err = report.GetControlPlaneMetrics(t, ctx); err != nil { + if err = report.Collect(t, ctx, jobNN); err != nil { return nil, err } From 19ae3d8e702d114d9bce5b6bbae00c6187ab092c Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Tue, 25 Jun 2024 22:15:43 +0800 Subject: [PATCH 28/30] grant benchmark-test job with write access Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index cd8a166b193..d826758287c 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -36,13 +36,12 @@ on: type: string required: false -permissions: - pull-requests: write - jobs: benchmark-test: name: Benchmark Test runs-on: ubuntu-latest + permissions: + pull-requests: write steps: - uses: actions/checkout@v2 From 970f5f700bcfa2057c4dc268cd33c05d2bd27d83 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Tue, 25 Jun 2024 22:58:05 +0800 Subject: [PATCH 29/30] upload latest benchmark report and remove commenter ci Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 12 - test/benchmark/benchmark_report.md | 1273 +++++++++++++++------------- 2 files changed, 693 insertions(+), 592 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index d826758287c..4b07a99621f 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -4,10 +4,6 @@ on: branches: - "main" - "release/v*" - pull_request_target: - branches: - - "main" - - "release/v*" workflow_dispatch: inputs: rps: @@ -40,8 +36,6 @@ jobs: benchmark-test: name: Benchmark Test runs-on: ubuntu-latest - permissions: - pull-requests: write steps: - uses: actions/checkout@v2 @@ -57,9 +51,3 @@ jobs: BENCHMARK_CPU_LIMITS: ${{ github.event.inputs.cpu_limits || 1000 }} BENCHMARK_MEMORY_LIMITS: ${{ github.event.inputs.memory_limits || 2048 }} run: make benchmark - - - name: Comment Benchmark report - uses: thollander/actions-comment-pull-request@v2 - with: - filePath: ./bin/benchmark/report.md - comment_tag: benchmark-report diff --git a/test/benchmark/benchmark_report.md b/test/benchmark/benchmark_report.md index 35acb494132..c62199e7b0c 100644 --- a/test/benchmark/benchmark_report.md +++ b/test/benchmark/benchmark_report.md @@ -1,8 +1,10 @@ # Benchmark Report -| RPS | Connections | Duration (Seconds) | CPU Limits (m) | Memory Limits (MiB) | -|------|-------------|--------------------|----------------|---------------------| -| 1000 | 100 | 90 | 1000 | 2048 | +Benchmark test settings: + +|RPS |Connections|Duration (Seconds)|CPU Limits (m)|Memory Limits (MiB)| +|- |- |- |- |- | +|1000|100 |90 |1000 |2048 | ## Test: ScaleHTTPRoute @@ -18,81 +20,94 @@ Click to see the full results. scale-up-httproutes-10 ```plaintext - -benchmark_http_client.latency_2xx (359873 samples) - min: 0s 000ms 313us | mean: 0s 000ms 519us | max: 0s 072ms 552us | pstdev: 0s 000ms 793us - - Percentile Count Value - 0.5 179946 0s 000ms 439us - 0.75 269912 0s 000ms 513us - 0.8 287903 0s 000ms 533us - 0.9 323889 0s 000ms 614us - 0.95 341884 0s 000ms 736us - 0.990625 356500 0s 001ms 559us - 0.99902344 359522 0s 007ms 985us - -Queueing and connection setup latency (359874 samples) - min: 0s 000ms 002us | mean: 0s 000ms 012us | max: 0s 063ms 952us | pstdev: 0s 000ms 131us - - Percentile Count Value - 0.5 180120 0s 000ms 010us - 0.75 270199 0s 000ms 011us - 0.8 288323 0s 000ms 011us - 0.9 324255 0s 000ms 011us - 0.95 341934 0s 000ms 012us - 0.990625 356501 0s 000ms 031us - 0.99902344 359523 0s 000ms 167us - -Request start to response end (359873 samples) - min: 0s 000ms 312us | mean: 0s 000ms 518us | max: 0s 072ms 552us | pstdev: 0s 000ms 793us - - Percentile Count Value - 0.5 179950 0s 000ms 438us - 0.75 269913 0s 000ms 512us - 0.8 287916 0s 000ms 533us - 0.9 323890 0s 000ms 614us - 0.95 341883 0s 000ms 736us - 0.990625 356500 0s 001ms 558us - 0.99902344 359522 0s 007ms 984us - -Response body size in bytes (359873 samples) +[2024-06-25 14:23:36.545][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:23:36.545677][1][I] Detected 4 (v)CPUs with affinity.. +[14:23:36.545689][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:23:36.545691][1][I] Global targets: 400 connections and 4000 calls per second. +[14:23:36.545692][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:25:07.247563][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9998444444686 per second.) +[14:25:07.248019][20][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9975333394178 per second.) +[14:25:07.248086][22][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.999577777956 per second.) +[14:25:07.248380][24][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9990666675377 per second.) +[14:25:07.818186][1][I] Done. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359802 samples) + min: 0s 000ms 290us | mean: 0s 000ms 544us | max: 0s 062ms 072us | pstdev: 0s 000ms 863us + + Percentile Count Value + 0.5 179926 0s 000ms 448us + 0.75 269854 0s 000ms 530us + 0.8 287861 0s 000ms 552us + 0.9 323828 0s 000ms 684us + 0.95 341812 0s 000ms 793us + 0.990625 356429 0s 001ms 863us + 0.99902344 359451 0s 009ms 407us + +Queueing and connection setup latency (359802 samples) + min: 0s 000ms 002us | mean: 0s 000ms 011us | max: 0s 023ms 417us | pstdev: 0s 000ms 100us + + Percentile Count Value + 0.5 179902 0s 000ms 010us + 0.75 269909 0s 000ms 010us + 0.8 288003 0s 000ms 011us + 0.9 323850 0s 000ms 011us + 0.95 341822 0s 000ms 012us + 0.990625 356429 0s 000ms 029us + 0.99902344 359451 0s 000ms 170us + +Request start to response end (359802 samples) + min: 0s 000ms 289us | mean: 0s 000ms 543us | max: 0s 062ms 072us | pstdev: 0s 000ms 863us + + Percentile Count Value + 0.5 179915 0s 000ms 448us + 0.75 269861 0s 000ms 530us + 0.8 287848 0s 000ms 552us + 0.9 323824 0s 000ms 683us + 0.95 341813 0s 000ms 792us + 0.990625 356429 0s 001ms 862us + 0.99902344 359451 0s 009ms 406us + +Response body size in bytes (359802 samples) min: 10 | mean: 10 | max: 10 | pstdev: 0 -Response header size in bytes (359873 samples) +Response header size in bytes (359802 samples) min: 110 | mean: 110 | max: 110 | pstdev: 0 -Blocking. Results are skewed when significant numbers are reported here. (1 samples) - min: 0s 001ms 903us | mean: 0s 001ms 903us | max: 0s 001ms 903us | pstdev: 0s 000ms 000us +Blocking. Results are skewed when significant numbers are reported here. (2 samples) + min: 0s 000ms 901us | mean: 0s 001ms 551us | max: 0s 002ms 202us | pstdev: 0s 000ms 650us -Initiation to completion (359999 samples) - min: 0s 000ms 009us | mean: 0s 000ms 537us | max: 0s 072ms 634us | pstdev: 0s 000ms 841us + Percentile Count Value + 0.5 1 0s 000ms 901us + +Initiation to completion (360000 samples) + min: 0s 000ms 006us | mean: 0s 000ms 562us | max: 0s 062ms 310us | pstdev: 0s 000ms 883us Percentile Count Value - 0.5 180023 0s 000ms 455us - 0.75 270019 0s 000ms 530us - 0.8 288001 0s 000ms 551us - 0.9 324001 0s 000ms 634us - 0.95 342004 0s 000ms 755us - 0.990625 356625 0s 001ms 601us - 0.99902344 359648 0s 008ms 163us + 0.5 180016 0s 000ms 465us + 0.75 270003 0s 000ms 547us + 0.8 288024 0s 000ms 570us + 0.9 324002 0s 000ms 702us + 0.95 342002 0s 000ms 815us + 0.990625 356625 0s 001ms 912us + 0.99902344 359649 0s 009ms 617us Counter Value Per second -benchmark.http_2xx 359873 3998.59 -benchmark.pool_overflow 126 1.40 +benchmark.http_2xx 359802 3997.80 +benchmark.pool_overflow 198 2.20 cluster_manager.cluster_added 4 0.04 default.total_match_count 4 0.04 membership_change 4 0.04 runtime.load_success 1 0.01 runtime.override_dir_not_exists 1 0.01 -upstream_cx_http1_total 102 1.13 -upstream_cx_rx_bytes_total 56500061 627778.28 -upstream_cx_total 102 1.13 -upstream_cx_tx_bytes_total 15474582 171939.75 -upstream_rq_pending_overflow 126 1.40 -upstream_rq_pending_total 102 1.13 -upstream_rq_total 359874 3998.60 +upstream_cx_http1_total 108 1.20 +upstream_cx_rx_bytes_total 56488914 627653.97 +upstream_cx_total 108 1.20 +upstream_cx_tx_bytes_total 15471486 171905.23 +upstream_rq_pending_overflow 198 2.20 +upstream_rq_pending_total 108 1.20 +upstream_rq_total 359802 3997.80 -[09:24:22.102587][1][I] Done. ``` @@ -102,78 +117,90 @@ upstream_rq_total 359874 3998.60 scale-up-httproutes-50 ```plaintext - -benchmark_http_client.latency_2xx (359812 samples) - min: 0s 000ms 309us | mean: 0s 000ms 509us | max: 0s 038ms 768us | pstdev: 0s 000ms 426us - - Percentile Count Value - 0.5 179913 0s 000ms 445us - 0.75 269869 0s 000ms 519us - 0.8 287867 0s 000ms 541us - 0.9 323837 0s 000ms 663us - 0.95 341824 0s 000ms 741us - 0.990625 356439 0s 001ms 300us - 0.99902344 359461 0s 005ms 292us - -Queueing and connection setup latency (359813 samples) - min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 029ms 871us | pstdev: 0s 000ms 075us - - Percentile Count Value - 0.5 180015 0s 000ms 010us - 0.75 270757 0s 000ms 011us - 0.8 287880 0s 000ms 011us - 0.9 323906 0s 000ms 011us - 0.95 341867 0s 000ms 012us - 0.990625 356440 0s 000ms 027us - 0.99902344 359462 0s 000ms 163us - -Request start to response end (359812 samples) - min: 0s 000ms 305us | mean: 0s 000ms 509us | max: 0s 038ms 768us | pstdev: 0s 000ms 426us - - Percentile Count Value - 0.5 179911 0s 000ms 445us - 0.75 269873 0s 000ms 519us - 0.8 287861 0s 000ms 540us - 0.9 323831 0s 000ms 662us - 0.95 341823 0s 000ms 741us - 0.990625 356439 0s 001ms 299us - 0.99902344 359461 0s 005ms 291us - -Response body size in bytes (359812 samples) +[2024-06-25 14:25:18.533][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:25:18.533824][1][I] Detected 4 (v)CPUs with affinity.. +[14:25:18.533833][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:25:18.533836][1][I] Global targets: 400 connections and 4000 calls per second. +[14:25:18.533837][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:26:49.235731][18][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) +[14:26:49.235977][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000 per second.) +[14:26:49.236240][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.9888111119815 per second.) +[14:26:49.236565][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.9879333448638 per second.) +[14:26:54.772502][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359912 samples) + min: 0s 000ms 303us | mean: 0s 000ms 508us | max: 0s 031ms 259us | pstdev: 0s 000ms 449us + + Percentile Count Value + 0.5 179967 0s 000ms 437us + 0.75 269938 0s 000ms 509us + 0.8 287950 0s 000ms 533us + 0.9 323925 0s 000ms 643us + 0.95 341919 0s 000ms 735us + 0.990625 356538 0s 001ms 526us + 0.99902344 359561 0s 006ms 589us + +Queueing and connection setup latency (359914 samples) + min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 023ms 449us | pstdev: 0s 000ms 080us + + Percentile Count Value + 0.5 180198 0s 000ms 010us + 0.75 270795 0s 000ms 011us + 0.8 288165 0s 000ms 011us + 0.9 324019 0s 000ms 011us + 0.95 341989 0s 000ms 011us + 0.990625 356540 0s 000ms 025us + 0.99902344 359564 0s 000ms 159us + +Request start to response end (359912 samples) + min: 0s 000ms 302us | mean: 0s 000ms 508us | max: 0s 031ms 258us | pstdev: 0s 000ms 448us + + Percentile Count Value + 0.5 179970 0s 000ms 437us + 0.75 269934 0s 000ms 509us + 0.8 287943 0s 000ms 533us + 0.9 323921 0s 000ms 643us + 0.95 341917 0s 000ms 735us + 0.990625 356538 0s 001ms 525us + 0.99902344 359561 0s 006ms 589us + +Response body size in bytes (359912 samples) min: 10 | mean: 10 | max: 10 | pstdev: 0 -Response header size in bytes (359812 samples) +Response header size in bytes (359912 samples) min: 110 | mean: 110 | max: 110 | pstdev: 0 -Initiation to completion (359999 samples) - min: 0s 000ms 007us | mean: 0s 000ms 534us | max: 0s 038ms 785us | pstdev: 0s 000ms 618us +Initiation to completion (359998 samples) + min: 0s 000ms 006us | mean: 0s 000ms 525us | max: 0s 032ms 263us | pstdev: 0s 000ms 461us Percentile Count Value - 0.5 180006 0s 000ms 462us - 0.75 270010 0s 000ms 537us - 0.8 288017 0s 000ms 558us - 0.9 324002 0s 000ms 682us - 0.95 342008 0s 000ms 760us - 0.990625 356625 0s 001ms 355us - 0.99902344 359648 0s 006ms 391us + 0.5 180024 0s 000ms 453us + 0.75 270017 0s 000ms 526us + 0.8 288004 0s 000ms 550us + 0.9 323999 0s 000ms 663us + 0.95 341999 0s 000ms 753us + 0.990625 356624 0s 001ms 571us + 0.99902344 359647 0s 006ms 655us Counter Value Per second -benchmark.http_2xx 359812 3997.91 -benchmark.pool_overflow 187 2.08 +benchmark.http_2xx 359912 3999.02 +benchmark.pool_overflow 86 0.96 cluster_manager.cluster_added 4 0.04 default.total_match_count 4 0.04 membership_change 4 0.04 runtime.load_success 1 0.01 runtime.override_dir_not_exists 1 0.01 -upstream_cx_http1_total 56 0.62 -upstream_cx_rx_bytes_total 56490484 627671.58 -upstream_cx_total 56 0.62 -upstream_cx_tx_bytes_total 15471959 171910.53 -upstream_rq_pending_overflow 187 2.08 -upstream_rq_pending_total 56 0.62 -upstream_rq_total 359813 3997.92 +upstream_cx_http1_total 61 0.68 +upstream_cx_rx_bytes_total 56506184 627846.33 +upstream_cx_total 61 0.68 +upstream_cx_tx_bytes_total 15476302 171958.87 +upstream_rq_pending_overflow 86 0.96 +upstream_rq_pending_total 61 0.68 +upstream_rq_total 359914 3999.04 -[09:26:13.509310][1][I] Done. +[14:26:59.773726][23][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[14:26:59.775576][1][I] Done. ``` @@ -183,78 +210,90 @@ upstream_rq_total 359813 3997.92 scale-up-httproutes-100 ```plaintext - -benchmark_http_client.latency_2xx (359853 samples) - min: 0s 000ms 314us | mean: 0s 000ms 516us | max: 0s 035ms 172us | pstdev: 0s 000ms 430us - - Percentile Count Value - 0.5 179943 0s 000ms 449us - 0.75 269895 0s 000ms 523us - 0.8 287893 0s 000ms 546us - 0.9 323868 0s 000ms 666us - 0.95 341865 0s 000ms 750us -[09:27:57.164977][1][I] Done. - 0.990625 356480 0s 001ms 430us - 0.99902344 359502 0s 005ms 762us - -Queueing and connection setup latency (359853 samples) - min: 0s 000ms 002us | mean: 0s 000ms 011us | max: 0s 026ms 753us | pstdev: 0s 000ms 060us - - Percentile Count Value - 0.5 180211 0s 000ms 010us - 0.75 270918 0s 000ms 011us - 0.8 288356 0s 000ms 011us - 0.9 323954 0s 000ms 011us - 0.95 341865 0s 000ms 012us - 0.990625 356481 0s 000ms 029us - 0.99902344 359502 0s 000ms 166us - -Request start to response end (359853 samples) - min: 0s 000ms 313us | mean: 0s 000ms 516us | max: 0s 035ms 172us | pstdev: 0s 000ms 430us - - Percentile Count Value - 0.5 179945 0s 000ms 448us - 0.75 269895 0s 000ms 523us - 0.8 287887 0s 000ms 545us - 0.9 323881 0s 000ms 666us - 0.95 341864 0s 000ms 749us - 0.990625 356480 0s 001ms 430us - 0.99902344 359502 0s 005ms 762us - -Response body size in bytes (359853 samples) +[2024-06-25 14:27:18.024][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:27:18.024969][1][I] Detected 4 (v)CPUs with affinity.. +[14:27:18.024981][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:27:18.024984][1][I] Global targets: 400 connections and 4000 calls per second. +[14:27:18.024985][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:28:48.726723][18][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000111111112 per second.) +[14:28:48.726970][19][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) +[14:28:48.727250][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.9885555593705 per second.) +[14:28:48.727579][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.9878444571402 per second.) +[14:28:54.260999][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359869 samples) + min: 0s 000ms 301us | mean: 0s 000ms 514us | max: 0s 029ms 035us | pstdev: 0s 000ms 431us + + Percentile Count Value + 0.5 179965 0s 000ms 444us + 0.75 269904 0s 000ms 518us + 0.8 287910 0s 000ms 540us + 0.9 323888 0s 000ms 641us + 0.95 341879 0s 000ms 744us + 0.990625 356497 0s 001ms 522us + 0.99902344 359518 0s 006ms 553us + +Queueing and connection setup latency (359871 samples) + min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 027ms 366us | pstdev: 0s 000ms 058us + + Percentile Count Value + 0.5 179998 0s 000ms 010us + 0.75 271128 0s 000ms 011us + 0.8 288699 0s 000ms 011us + 0.9 324073 0s 000ms 011us + 0.95 341895 0s 000ms 012us + 0.990625 356498 0s 000ms 029us + 0.99902344 359520 0s 000ms 166us + +Request start to response end (359869 samples) + min: 0s 000ms 301us | mean: 0s 000ms 513us | max: 0s 029ms 035us | pstdev: 0s 000ms 431us + + Percentile Count Value + 0.5 179935 0s 000ms 444us + 0.75 269906 0s 000ms 517us + 0.8 287896 0s 000ms 539us + 0.9 323883 0s 000ms 641us + 0.95 341876 0s 000ms 743us + 0.990625 356496 0s 001ms 521us + 0.99902344 359518 0s 006ms 552us + +Response body size in bytes (359869 samples) min: 10 | mean: 10 | max: 10 | pstdev: 0 -Response header size in bytes (359853 samples) +Response header size in bytes (359869 samples) min: 110 | mean: 110 | max: 110 | pstdev: 0 -Initiation to completion (360000 samples) - min: 0s 000ms 005us | mean: 0s 000ms 534us | max: 0s 035ms 190us | pstdev: 0s 000ms 447us +Initiation to completion (359998 samples) + min: 0s 000ms 007us | mean: 0s 000ms 532us | max: 0s 029ms 055us | pstdev: 0s 000ms 451us Percentile Count Value - 0.5 180008 0s 000ms 466us - 0.75 270010 0s 000ms 540us - 0.8 288009 0s 000ms 563us - 0.9 324005 0s 000ms 686us - 0.95 342003 0s 000ms 769us - 0.990625 356625 0s 001ms 481us - 0.99902344 359649 0s 006ms 056us + 0.5 180023 0s 000ms 461us + 0.75 270008 0s 000ms 535us + 0.8 288030 0s 000ms 557us + 0.9 324004 0s 000ms 661us + 0.95 341999 0s 000ms 763us + 0.990625 356624 0s 001ms 570us + 0.99902344 359647 0s 007ms 023us Counter Value Per second -benchmark.http_2xx 359853 3998.37 -benchmark.pool_overflow 147 1.63 +benchmark.http_2xx 359869 3998.54 +benchmark.pool_overflow 129 1.43 cluster_manager.cluster_added 4 0.04 default.total_match_count 4 0.04 membership_change 4 0.04 runtime.load_success 1 0.01 runtime.override_dir_not_exists 1 0.01 -upstream_cx_http1_total 64 0.71 -upstream_cx_rx_bytes_total 56496921 627743.57 -upstream_cx_total 64 0.71 -upstream_cx_tx_bytes_total 15473679 171929.77 -upstream_rq_pending_overflow 147 1.63 -upstream_rq_pending_total 64 0.71 -upstream_rq_total 359853 3998.37 +upstream_cx_http1_total 63 0.70 +upstream_cx_rx_bytes_total 56499433 627771.26 +upstream_cx_total 63 0.70 +upstream_cx_tx_bytes_total 15474453 171938.31 +upstream_rq_pending_overflow 129 1.43 +upstream_rq_pending_total 63 0.70 +upstream_rq_total 359871 3998.57 +[14:28:59.262084][23][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[14:28:59.263795][1][I] Done. ``` @@ -264,78 +303,88 @@ upstream_rq_total 359853 3998.37 scale-up-httproutes-300 ```plaintext - -benchmark_http_client.latency_2xx (359847 samples) - min: 0s 000ms 304us | mean: 0s 000ms 521us | max: 0s 024ms 524us | pstdev: 0s 000ms 424us - - Percentile Count Value - 0.5 179935 0s 000ms 454us - 0.75 269895 0s 000ms 527us - 0.8 287881 0s 000ms 548us - 0.9 323864 0s 000ms 662us - 0.95 341856 0s 000ms 754us - 0.990625 356474 0s 001ms 504us - 0.99902344 359496 0s 006ms 789us - -Queueing and connection setup latency (359848 samples) - min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 015ms 363us | pstdev: 0s 000ms 045us - - Percentile Count Value - 0.5 179968 0s 000ms 010us - 0.75 269895 0s 000ms 011us - 0.8 288448 0s 000ms 011us - 0.9 324002 0s 000ms 011us - 0.95 341858 0s 000ms 012us - 0.990625 356477 0s 000ms 029us - 0.99902344 359497 0s 000ms 167us - -Request start to response end (359847 samples) - min: 0s 000ms 304us | mean: 0s 000ms 521us | max: 0s 024ms 524us | pstdev: 0s 000ms 423us - - Percentile Count Value - 0.5 179945 0s 000ms 454us - 0.75 269914 0s 000ms 526us - 0.8 287893 0s 000ms 547us - 0.9 323864 0s 000ms 661us - 0.95 341856 0s 000ms 754us - 0.990625 356474 0s 001ms 503us - 0.99902344 359496 0s 006ms 786us - -Response body size in bytes (359847 samples) +[2024-06-25 14:32:23.491][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:32:23.491963][1][I] Detected 4 (v)CPUs with affinity.. +[14:32:23.491978][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:32:23.491980][1][I] Global targets: 400 connections and 4000 calls per second. +[14:32:23.491981][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:33:54.193887][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) +[14:33:54.194133][19][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000222222228 per second.) +[14:33:54.194387][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999888888891 per second.) +[14:33:54.194633][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000 per second.) +[14:33:54.712898][1][I] Done. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359845 samples) + min: 0s 000ms 302us | mean: 0s 000ms 507us | max: 0s 019ms 987us | pstdev: 0s 000ms 380us + + Percentile Count Value + 0.5 179931 0s 000ms 440us + 0.75 269886 0s 000ms 512us + 0.8 287876 0s 000ms 533us + 0.9 323864 0s 000ms 637us + 0.95 341854 0s 000ms 734us + 0.990625 356472 0s 001ms 469us + 0.99902344 359494 0s 006ms 259us + +Queueing and connection setup latency (359845 samples) + min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 017ms 634us | pstdev: 0s 000ms 043us + + Percentile Count Value + 0.5 180160 0s 000ms 010us + 0.75 270791 0s 000ms 010us + 0.8 287918 0s 000ms 011us + 0.9 324365 0s 000ms 011us + 0.95 341940 0s 000ms 011us + 0.990625 356473 0s 000ms 027us + 0.99902344 359494 0s 000ms 167us + +Request start to response end (359845 samples) + min: 0s 000ms 302us | mean: 0s 000ms 506us | max: 0s 019ms 987us | pstdev: 0s 000ms 380us + + Percentile Count Value + 0.5 179923 0s 000ms 440us + 0.75 269893 0s 000ms 511us + 0.8 287905 0s 000ms 533us + 0.9 323861 0s 000ms 636us + 0.95 341854 0s 000ms 733us + 0.990625 356472 0s 001ms 468us + 0.99902344 359494 0s 006ms 259us + +Response body size in bytes (359845 samples) min: 10 | mean: 10 | max: 10 | pstdev: 0 -Response header size in bytes (359847 samples) +Response header size in bytes (359845 samples) min: 110 | mean: 110 | max: 110 | pstdev: 0 -Initiation to completion (359999 samples) - min: 0s 000ms 006us | mean: 0s 000ms 538us | max: 0s 024ms 556us | pstdev: 0s 000ms 433us +Initiation to completion (360000 samples) + min: 0s 000ms 007us | mean: 0s 000ms 525us | max: 0s 020ms 035us | pstdev: 0s 000ms 409us Percentile Count Value - 0.5 180015 0s 000ms 471us - 0.75 270008 0s 000ms 544us - 0.8 288021 0s 000ms 565us - 0.9 324004 0s 000ms 681us - 0.95 342002 0s 000ms 773us - 0.990625 356625 0s 001ms 545us - 0.99902344 359648 0s 006ms 943us + 0.5 180011 0s 000ms 456us + 0.75 270028 0s 000ms 529us + 0.8 288004 0s 000ms 550us + 0.9 324006 0s 000ms 656us + 0.95 342005 0s 000ms 751us + 0.990625 356625 0s 001ms 525us + 0.99902344 359649 0s 006ms 569us Counter Value Per second -benchmark.http_2xx 359847 3998.30 -benchmark.pool_overflow 152 1.69 +benchmark.http_2xx 359845 3998.28 +benchmark.pool_overflow 155 1.72 cluster_manager.cluster_added 4 0.04 default.total_match_count 4 0.04 membership_change 4 0.04 runtime.load_success 1 0.01 runtime.override_dir_not_exists 1 0.01 -upstream_cx_http1_total 70 0.78 -upstream_cx_rx_bytes_total 56495979 627732.89 -upstream_cx_total 70 0.78 -upstream_cx_tx_bytes_total 15473464 171927.32 -upstream_rq_pending_overflow 152 1.69 -upstream_rq_pending_total 70 0.78 -upstream_rq_total 359848 3998.31 +upstream_cx_http1_total 62 0.69 +upstream_cx_rx_bytes_total 56495665 627729.61 +upstream_cx_total 62 0.69 +upstream_cx_tx_bytes_total 15473335 171925.94 +upstream_rq_pending_overflow 155 1.72 +upstream_rq_pending_total 62 0.69 +upstream_rq_total 359845 3998.28 -[09:33:09.468007][1][I] Done. ``` @@ -345,86 +394,92 @@ upstream_rq_total 359848 3998.31 scale-up-httproutes-500 ```plaintext - -benchmark_http_client.latency_2xx (359807 samples) - min: 0s 000ms 310us | mean: 0s 000ms 560us | max: 0s 060ms 688us | pstdev: 0s 000ms 942us +[2024-06-25 14:38:43.691][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:38:43.691938][1][I] Detected 4 (v)CPUs with affinity.. +[14:38:43.691951][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:38:43.691953][1][I] Global targets: 400 connections and 4000 calls per second. +[14:38:43.691954][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:40:14.393764][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.9885666703507 per second.) +[14:40:14.393981][19][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000555555587 per second.) +[14:40:14.394312][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9992666672044 per second.) +[14:40:14.394484][23][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 1000.0000333333344 per second.) +[14:40:19.954533][18][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359743 samples) + min: 0s 000ms 311us | mean: 0s 000ms 566us | max: 0s 064ms 673us | pstdev: 0s 001ms 003us + + Percentile Count Value + 0.5 179890 0s 000ms 441us + 0.75 269813 0s 000ms 519us + 0.8 287802 0s 000ms 544us + 0.9 323771 0s 000ms 684us + 0.95 341757 0s 000ms 790us + 0.990625 356372 0s 002ms 465us + 0.99902344 359392 0s 016ms 404us + +Queueing and connection setup latency (359744 samples) + min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 040ms 116us | pstdev: 0s 000ms 093us Percentile Count Value - 0.5 179905 0s 000ms 456us - 0.75 269868 0s 000ms 529us - 0.8 287867 0s 000ms 550us - 0.9 323830 0s 000ms 678us - 0.95 341818 0s 000ms 791us - 0.990625 356434 0s 002ms 103us - 0.99902344 359456 0s 012ms 998us - -Queueing and connection setup latency (359808 samples) - min: 0s 000ms 001us | mean: 0s 000ms 012us | max: 0s 027ms 772us | pstdev: 0s 000ms 104us - - Percentile Count Value - 0.5 180194 0s 000ms 010us - 0.75 270932 0s 000ms 011us - 0.8 288826 0s 000ms 011us - 0.9 324092 0s 000ms 011us - 0.95 341848 0s 000ms 012us - 0.990625 356435 0s 000ms 030us - 0.99902344 359457 0s 000ms 170us - -Request start to response end (359807 samples) - min: 0s 000ms 309us | mean: 0s 000ms 560us | max: 0s 060ms 688us | pstdev: 0s 000ms 942us - - Percentile Count Value - 0.5 179908 0s 000ms 456us - 0.75 269856 0s 000ms 529us - 0.8 287848 0s 000ms 550us - 0.9 323827 0s 000ms 677us - 0.95 341817 0s 000ms 791us - 0.990625 356434 0s 002ms 103us - 0.99902344 359456 0s 012ms 998us - -Response body size in bytes (359807 samples) + 0.5 180015 0s 000ms 010us + 0.75 269817 0s 000ms 011us + 0.8 288758 0s 000ms 011us + 0.9 324218 0s 000ms 011us + 0.95 341791 0s 000ms 012us + 0.990625 356372 0s 000ms 029us + 0.99902344 359393 0s 000ms 172us + +Request start to response end (359743 samples) + min: 0s 000ms 311us | mean: 0s 000ms 566us | max: 0s 064ms 673us | pstdev: 0s 001ms 003us + + Percentile Count Value + 0.5 179874 0s 000ms 441us + 0.75 269820 0s 000ms 519us + 0.8 287800 0s 000ms 544us + 0.9 323772 0s 000ms 684us + 0.95 341758 0s 000ms 790us + 0.990625 356372 0s 002ms 464us + 0.99902344 359392 0s 016ms 404us + +Response body size in bytes (359743 samples) min: 10 | mean: 10 | max: 10 | pstdev: 0 -Response header size in bytes (359807 samples) +Response header size in bytes (359743 samples) min: 110 | mean: 110 | max: 110 | pstdev: 0 -Blocking. Results are skewed when significant numbers are reported here. (6 samples) - min: 0s 000ms 201us | mean: 0s 000ms 928us | max: 0s 002ms 801us | pstdev: 0s 000ms 880us - - Percentile Count Value - 0.5 3 0s 000ms 614us - 0.75 5 0s 000ms 914us - 0.8 5 0s 000ms 914us +Blocking. Results are skewed when significant numbers are reported here. (1 samples) + min: 0s 001ms 331us | mean: 0s 001ms 331us | max: 0s 001ms 331us | pstdev: 0s 000ms 000us Initiation to completion (359999 samples) - min: 0s 000ms 006us | mean: 0s 000ms 581us | max: 0s 060ms 710us | pstdev: 0s 000ms 978us + min: 0s 000ms 005us | mean: 0s 000ms 593us | max: 0s 064ms 710us | pstdev: 0s 001ms 145us Percentile Count Value - 0.5 180010 0s 000ms 473us - 0.75 270021 0s 000ms 547us - 0.8 288019 0s 000ms 568us - 0.9 324004 0s 000ms 697us - 0.95 342000 0s 000ms 814us - 0.990625 356625 0s 002ms 210us - 0.99902344 359648 0s 014ms 801us + 0.5 180029 0s 000ms 458us + 0.75 270014 0s 000ms 537us + 0.8 288011 0s 000ms 562us + 0.9 324008 0s 000ms 703us + 0.95 342000 0s 000ms 813us + 0.990625 356625 0s 002ms 640us + 0.99902344 359648 0s 018ms 319us Counter Value Per second -benchmark.http_2xx 359807 3997.85 -benchmark.pool_overflow 192 2.13 +benchmark.http_2xx 359743 3997.14 +benchmark.pool_overflow 256 2.84 cluster_manager.cluster_added 4 0.04 default.total_match_count 4 0.04 membership_change 4 0.04 runtime.load_success 1 0.01 runtime.override_dir_not_exists 1 0.01 -upstream_cx_http1_total 112 1.24 -upstream_cx_rx_bytes_total 56489699 627663.06 -upstream_cx_total 112 1.24 -upstream_cx_tx_bytes_total 15471744 171908.20 -upstream_rq_pending_overflow 192 2.13 -upstream_rq_pending_total 112 1.24 -upstream_rq_total 359808 3997.87 +upstream_cx_http1_total 96 1.07 +upstream_cx_rx_bytes_total 56479651 627551.52 +upstream_cx_total 96 1.07 +upstream_cx_tx_bytes_total 15468992 171877.65 +upstream_rq_pending_overflow 256 2.84 +upstream_rq_pending_total 96 1.07 +upstream_rq_total 359744 3997.15 -[09:40:33.045512][1][I] Done. +[14:40:19.962562][1][I] Done. ``` @@ -434,88 +489,101 @@ upstream_rq_total 359808 3997.87 scale-down-httproutes-300 ```plaintext - -benchmark_http_client.latency_2xx (359812 samples) - min: 0s 000ms 302us | mean: 0s 001ms 445us | max: 0s 183ms 287us | pstdev: 0s 007ms 612us - - Percentile Count Value - 0.5 179921 0s 000ms 473us - 0.75 269867 0s 000ms 556us - 0.8 287850 0s 000ms 592us - 0.9 323832 0s 000ms 760us - 0.95 341822 0s 001ms 226us - 0.990625 356439 0s 043ms 878us - 0.99902344 359461 0s 095ms 641us - -Queueing and connection setup latency (359812 samples) - min: 0s 000ms 001us | mean: 0s 000ms 012us | max: 0s 027ms 984us | pstdev: 0s 000ms 113us - - Percentile Count Value - 0.5 180354 0s 000ms 010us - 0.75 270110 0s 000ms 011us - 0.8 288028 0s 000ms 011us - 0.9 324203 0s 000ms 011us - 0.95 341828 0s 000ms 012us - 0.990625 356439 0s 000ms 033us - 0.99902344 359461 0s 000ms 194us - -Request start to response end (359812 samples) - min: 0s 000ms 302us | mean: 0s 001ms 444us | max: 0s 183ms 287us | pstdev: 0s 007ms 612us - - Percentile Count Value - 0.5 179909 0s 000ms 472us - 0.75 269876 0s 000ms 555us - 0.8 287853 0s 000ms 592us - 0.9 323833 0s 000ms 759us - 0.95 341822 0s 001ms 226us - 0.990625 356439 0s 043ms 878us - 0.99902344 359461 0s 095ms 641us - -Response body size in bytes (359812 samples) +[2024-06-25 14:40:47.629][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:40:47.629688][1][I] Detected 4 (v)CPUs with affinity.. +[14:40:47.629700][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:40:47.629703][1][I] Global targets: 400 connections and 4000 calls per second. +[14:40:47.629704][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:42:18.331523][18][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9990222231783 per second.) +[14:42:18.331681][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9999777777783 per second.) +[14:42:18.331935][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 89999. (Completion rate was 999.988855555927 per second.) +[14:42:18.332255][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.999166667361 per second.) +[14:42:23.961708][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359722 samples) + min: 0s 000ms 303us | mean: 0s 001ms 203us | max: 0s 121ms 311us | pstdev: 0s 006ms 029us + + Percentile Count Value + 0.5 179885 0s 000ms 456us + 0.75 269810 0s 000ms 543us + 0.8 287781 0s 000ms 570us + 0.9 323750 0s 000ms 749us + 0.95 341736 0s 001ms 344us + 0.990625 356350 0s 016ms 743us + 0.99902344 359371 0s 083ms 783us + +Queueing and connection setup latency (359723 samples) + min: 0s 000ms 001us | mean: 0s 000ms 012us | max: 0s 041ms 897us | pstdev: 0s 000ms 131us + + Percentile Count Value + 0.5 179921 0s 000ms 010us + 0.75 270953 0s 000ms 011us + 0.8 288381 0s 000ms 011us + 0.9 323867 0s 000ms 011us + 0.95 341759 0s 000ms 012us + 0.990625 356351 0s 000ms 032us + 0.99902344 359372 0s 000ms 206us + +Request start to response end (359722 samples) + min: 0s 000ms 302us | mean: 0s 001ms 202us | max: 0s 121ms 311us | pstdev: 0s 006ms 029us + + Percentile Count Value + 0.5 179877 0s 000ms 456us + 0.75 269810 0s 000ms 542us + 0.8 287783 0s 000ms 570us + 0.9 323752 0s 000ms 748us + 0.95 341736 0s 001ms 344us + 0.990625 356350 0s 016ms 742us + 0.99902344 359371 0s 083ms 783us + +Response body size in bytes (359722 samples) min: 10 | mean: 10 | max: 10 | pstdev: 0 -Response header size in bytes (359812 samples) +Response header size in bytes (359722 samples) min: 110 | mean: 110 | max: 110 | pstdev: 0 -Blocking. Results are skewed when significant numbers are reported here. (565 samples) - min: 0s 000ms 044us | mean: 0s 006ms 788us | max: 0s 095ms 117us | pstdev: 0s 015ms 888us +Blocking. Results are skewed when significant numbers are reported here. (1022 samples) + min: 0s 000ms 045us | mean: 0s 006ms 403us | max: 0s 099ms 049us | pstdev: 0s 015ms 130us Percentile Count Value - 0.5 283 0s 000ms 744us - 0.75 424 0s 002ms 919us - 0.8 452 0s 005ms 369us - 0.9 509 0s 018ms 845us - 0.95 537 0s 047ms 767us - 0.990625 560 0s 071ms 987us + 0.5 511 0s 001ms 012us + 0.75 767 0s 003ms 262us + 0.8 818 0s 004ms 782us + 0.9 920 0s 015ms 482us + 0.95 971 0s 047ms 316us + 0.990625 1013 0s 075ms 657us + 0.99902344 1022 0s 099ms 049us -Initiation to completion (360000 samples) - min: 0s 000ms 007us | mean: 0s 001ms 468us | max: 0s 183ms 320us | pstdev: 0s 007ms 622us +Initiation to completion (359999 samples) + min: 0s 000ms 002us | mean: 0s 001ms 224us | max: 0s 121ms 405us | pstdev: 0s 006ms 041us Percentile Count Value - 0.5 180014 0s 000ms 490us - 0.75 270017 0s 000ms 573us - 0.8 288000 0s 000ms 611us - 0.9 324002 0s 000ms 780us - 0.95 342000 0s 001ms 259us - 0.990625 356625 0s 043ms 876us - 0.99902344 359649 0s 095ms 735us + 0.5 180002 0s 000ms 473us + 0.75 270009 0s 000ms 560us + 0.8 288002 0s 000ms 589us + 0.9 324002 0s 000ms 770us + 0.95 342001 0s 001ms 390us + 0.990625 356625 0s 017ms 278us + 0.99902344 359648 0s 083ms 849us Counter Value Per second -benchmark.http_2xx 359812 3997.91 -benchmark.pool_overflow 188 2.09 +benchmark.http_2xx 359722 3996.91 +benchmark.pool_overflow 277 3.08 cluster_manager.cluster_added 4 0.04 default.total_match_count 4 0.04 membership_change 4 0.04 runtime.load_success 1 0.01 runtime.override_dir_not_exists 1 0.01 -upstream_cx_http1_total 212 2.36 -upstream_cx_rx_bytes_total 56490484 627671.56 -upstream_cx_total 212 2.36 -upstream_cx_tx_bytes_total 15471916 171910.04 -upstream_rq_pending_overflow 188 2.09 -upstream_rq_pending_total 212 2.36 -upstream_rq_total 359812 3997.91 +upstream_cx_http1_total 123 1.37 +upstream_cx_rx_bytes_total 56476354 627514.75 +upstream_cx_total 123 1.37 +upstream_cx_tx_bytes_total 15468089 171867.57 +upstream_rq_pending_overflow 277 3.08 +upstream_rq_pending_total 123 1.37 +upstream_rq_total 359723 3996.92 +[14:42:23.965353][1][I] Done. ``` @@ -525,92 +593,102 @@ upstream_rq_total 359812 3997.91 scale-down-httproutes-100 ```plaintext - -benchmark_http_client.latency_2xx (359688 samples) - min: 0s 000ms 308us | mean: 0s 009ms 727us | max: 0s 186ms 793us | pstdev: 0s 019ms 123us - - Percentile Count Value - 0.5 179844 0s 001ms 784us - 0.75 269768 0s 007ms 040us - 0.8 287752 0s 009ms 941us - 0.9 323723 0s 033ms 730us - 0.95 341704 0s 061ms 620us - 0.990625 356316 0s 083ms 664us - 0.99902344 359337 0s 110ms 288us - -Queueing and connection setup latency (359702 samples) - min: 0s 000ms 001us | mean: 0s 000ms 014us | max: 0s 049ms 557us | pstdev: 0s 000ms 258us - - Percentile Count Value - 0.5 180026 0s 000ms 008us - 0.75 269966 0s 000ms 010us - 0.8 287847 0s 000ms 010us - 0.9 323820 0s 000ms 011us - 0.95 341720 0s 000ms 018us - 0.990625 356330 0s 000ms 113us - 0.99902344 359351 0s 000ms 988us - -Request start to response end (359688 samples) - min: 0s 000ms 308us | mean: 0s 009ms 726us | max: 0s 186ms 793us | pstdev: 0s 019ms 123us - - Percentile Count Value - 0.5 179845 0s 001ms 783us - 0.75 269766 0s 007ms 039us - 0.8 287751 0s 009ms 940us - 0.9 323723 0s 033ms 730us - 0.95 341705 0s 061ms 620us - 0.990625 356316 0s 083ms 664us - 0.99902344 359337 0s 110ms 288us - -Response body size in bytes (359688 samples) +[2024-06-25 14:42:41.429][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:42:41.430321][1][I] Detected 4 (v)CPUs with affinity.. +[14:42:41.430334][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:42:41.430336][1][I] Global targets: 400 connections and 4000 calls per second. +[14:42:41.430338][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:44:12.132884][18][I] Stopping after 89999 ms. Initiated: 90000 / Completed: 89996. (Completion rate was 999.9555777767906 per second.) +[14:44:12.133405][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.999777777827 per second.) +[14:44:12.133681][23][I] Stopping after 90000 ms. Initiated: 89995 / Completed: 89995. (Completion rate was 999.9439111410252 per second.) +[14:44:12.138437][19][I] Stopping after 90005 ms. Initiated: 89979 / Completed: 89958. (Completion rate was 999.474453182769 per second.) +[14:44:17.976650][18][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (359649 samples) + min: 0s 000ms 297us | mean: 0s 009ms 493us | max: 0s 163ms 553us | pstdev: 0s 017ms 560us + + Percentile Count Value + 0.5 179826 0s 002ms 042us + 0.75 269740 0s 008ms 322us + 0.8 287721 0s 011ms 606us + 0.9 323685 0s 030ms 779us + 0.95 341667 0s 054ms 808us + 0.990625 356278 0s 079ms 679us + 0.99902344 359299 0s 106ms 098us + +Queueing and connection setup latency (359674 samples) + min: 0s 000ms 001us | mean: 0s 000ms 015us | max: 0s 033ms 947us | pstdev: 0s 000ms 269us + + Percentile Count Value + 0.5 179931 0s 000ms 008us + 0.75 269789 0s 000ms 010us + 0.8 287838 0s 000ms 011us + 0.9 323819 0s 000ms 011us + 0.95 341697 0s 000ms 015us + 0.990625 356303 0s 000ms 059us + 0.99902344 359323 0s 001ms 303us + +Request start to response end (359649 samples) + min: 0s 000ms 296us | mean: 0s 009ms 493us | max: 0s 163ms 553us | pstdev: 0s 017ms 560us + + Percentile Count Value + 0.5 179829 0s 002ms 041us + 0.75 269740 0s 008ms 321us + 0.8 287722 0s 011ms 606us + 0.9 323687 0s 030ms 779us + 0.95 341667 0s 054ms 808us + 0.990625 356278 0s 079ms 679us + 0.99902344 359299 0s 106ms 098us + +Response body size in bytes (359649 samples) min: 10 | mean: 10 | max: 10 | pstdev: 0 -Response header size in bytes (359688 samples) +Response header size in bytes (359649 samples) min: 110 | mean: 110 | max: 110 | pstdev: 0 -Blocking. Results are skewed when significant numbers are reported here. (21695 samples) - min: 0s 000ms 035us | mean: 0s 004ms 952us | max: 0s 143ms 073us | pstdev: 0s 012ms 407us +Blocking. Results are skewed when significant numbers are reported here. (19052 samples) + min: 0s 000ms 039us | mean: 0s 006ms 142us | max: 0s 146ms 767us | pstdev: 0s 012ms 746us Percentile Count Value - 0.5 10848 0s 001ms 029us - 0.75 16272 0s 003ms 006us - 0.8 17356 0s 003ms 915us - 0.9 19526 0s 009ms 562us - 0.95 20611 0s 031ms 671us - 0.990625 21492 0s 065ms 878us - 0.99902344 21674 0s 083ms 513us + 0.5 9526 0s 001ms 273us + 0.75 14289 0s 004ms 540us + 0.8 15242 0s 006ms 354us + 0.9 17147 0s 018ms 240us + 0.95 18100 0s 036ms 243us + 0.990625 18874 0s 063ms 096us + 0.99902344 19034 0s 087ms 937us -Initiation to completion (359969 samples) - min: 0s 000ms 135us | mean: 0s 009ms 787us | max: 0s 186ms 834us | pstdev: 0s 019ms 151us +Initiation to completion (359949 samples) + min: 0s 000ms 006us | mean: 0s 009ms 592us | max: 0s 163ms 594us | pstdev: 0s 017ms 643us Percentile Count Value - 0.5 179987 0s 001ms 820us - 0.75 269979 0s 007ms 143us - 0.8 287978 0s 010ms 083us - 0.9 323973 0s 033ms 902us - 0.95 341972 0s 061ms 700us - 0.990625 356596 0s 083ms 812us - 0.99902344 359618 0s 110ms 374us + 0.5 179976 0s 002ms 093us + 0.75 269962 0s 008ms 477us + 0.8 287962 0s 011ms 824us + 0.9 323955 0s 030ms 991us + 0.95 341957 0s 055ms 142us + 0.990625 356578 0s 080ms 228us + 0.99902344 359598 0s 106ms 164us Counter Value Per second -benchmark.http_2xx 359688 3996.53 -benchmark.pool_overflow 281 3.12 +benchmark.http_2xx 359649 3996.04 +benchmark.pool_overflow 300 3.33 cluster_manager.cluster_added 4 0.04 default.total_match_count 4 0.04 membership_change 4 0.04 runtime.load_success 1 0.01 runtime.override_dir_not_exists 1 0.01 -upstream_cx_http1_total 119 1.32 -upstream_cx_rx_bytes_total 56471016 627455.05 -upstream_cx_total 119 1.32 -upstream_cx_tx_bytes_total 15467186 171857.44 -upstream_rq_pending_overflow 281 3.12 -upstream_rq_pending_total 119 1.32 -upstream_rq_total 359702 3996.68 - -[09:44:35.525681][19][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. -[09:44:40.533295][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. -[09:44:40.549139][1][I] Done. +upstream_cx_http1_total 100 1.11 +upstream_cx_rx_bytes_total 56464893 627378.34 +upstream_cx_total 100 1.11 +upstream_cx_tx_bytes_total 15465939 171841.20 +upstream_rq_pending_overflow 300 3.33 +upstream_rq_pending_total 100 1.11 +upstream_rq_total 359674 3996.32 + +[14:44:22.981221][19][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[14:44:22.985214][1][I] Done. ``` @@ -620,87 +698,120 @@ upstream_rq_total 359702 3996.68 scale-down-httproutes-50 ```plaintext - -benchmark_http_client.latency_2xx (359809 samples) - min: 0s 000ms 298us | mean: 0s 000ms 544us | max: 0s 082ms 124us | pstdev: 0s 000ms 771us - - Percentile Count Value - 0.5 179920 0s 000ms 458us - 0.75 269873 0s 000ms 534us - 0.8 287849 0s 000ms 558us - 0.9 323834 0s 000ms 699us - 0.95 341819 0s 000ms 778us - 0.990625 356436 0s 001ms 748us - 0.99902344 359458 0s 009ms 125us - -Queueing and connection setup latency (359811 samples) - min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 019ms 383us | pstdev: 0s 000ms 048us - - Percentile Count Value - 0.5 180077 0s 000ms 010us - 0.75 271002 0s 000ms 011us - 0.8 288142 0s 000ms 011us - 0.9 323906 0s 000ms 011us - 0.95 341880 0s 000ms 012us - 0.990625 356438 0s 000ms 030us - 0.99902344 359460 0s 000ms 172us - -Request start to response end (359809 samples) - min: 0s 000ms 297us | mean: 0s 000ms 543us | max: 0s 082ms 124us | pstdev: 0s 000ms 771us - - Percentile Count Value - 0.5 179929 0s 000ms 458us - 0.75 269861 0s 000ms 534us - 0.8 287865 0s 000ms 558us - 0.9 323830 0s 000ms 698us - 0.95 341819 0s 000ms 778us - 0.990625 356436 0s 001ms 748us - 0.99902344 359458 0s 009ms 124us - -Response body size in bytes (359809 samples) - min: 10 | mean: 10 | max: 10 | pstdev: 0 - -Response header size in bytes (359809 samples) - min: 110 | mean: 110 | max: 110 | pstdev: 0 - -Blocking. Results are skewed when significant numbers are reported here. (6 samples) - min: 0s 000ms 140us | mean: 0s 001ms 861us | max: 0s 003ms 722us | pstdev: 0s 001ms 492us - - Percentile Count Value - 0.5 3 0s 000ms 854us - 0.75 5 0s 003ms 630us - 0.8 5 0s 003ms 630us - -Initiation to completion (359998 samples) - min: 0s 000ms 007us | mean: 0s 000ms 562us | max: 0s 082ms 153us | pstdev: 0s 000ms 779us - - Percentile Count Value - 0.5 180001 0s 000ms 476us - 0.75 270017 0s 000ms 552us - 0.8 288016 0s 000ms 576us - 0.9 324003 0s 000ms 717us - 0.95 342000 0s 000ms 798us - 0.990625 356624 0s 001ms 806us - 0.99902344 359647 0s 009ms 598us +[2024-06-25 14:44:35.399][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:44:35.400594][1][I] Detected 4 (v)CPUs with affinity.. +[14:44:35.400604][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:44:35.400607][1][I] Global targets: 400 connections and 4000 calls per second. +[14:44:35.400609][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:45:50.873708][19][E] Exiting due to failing termination predicate +[14:45:50.873747][19][I] Stopping after 74767 ms. Initiated: 74767 / Completed: 74746. (Completion rate was 999.716774109537 per second.) +[14:45:50.874918][18][E] Exiting due to failing termination predicate +[14:45:50.875120][18][I] Stopping after 74769 ms. Initiated: 74770 / Completed: 74749. (Completion rate was 999.7209036132042 per second.) +[14:45:50.878184][20][E] Exiting due to failing termination predicate +[14:45:50.878203][20][I] Stopping after 74772 ms. Initiated: 74772 / Completed: 74750. (Completion rate was 999.6972288218956 per second.) +[14:45:50.880264][21][E] Exiting due to failing termination predicate +[14:45:50.880286][21][I] Stopping after 74773 ms. Initiated: 74774 / Completed: 74747. (Completion rate was 999.6424011911454 per second.) +[14:45:51.840758][1][E] Terminated early because of a failure predicate. +[14:45:51.840780][1][I] Check the output for problematic counter values. The default Nighthawk failure predicates report failure if (1) Nighthawk could not connect to the target (see 'benchmark.pool_connection_failure' counter; check the address and port number, and try explicitly setting --address-family v4 or v6, especially when using DNS; instead of localhost try 127.0.0.1 or ::1 explicitly), (2) the protocol was not supported by the target (see 'benchmark.stream_resets' counter; check http/https in the URI, --h2), (3) the target returned a 4xx or 5xx HTTP response code (see 'benchmark.http_4xx' and 'benchmark.http_5xx' counters; check the URI path and the server config), or (4) a custom gRPC RequestSource failed. --failure-predicate can be used to relax expectations. +[14:45:56.841540][18][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +Nighthawk - A layer 7 protocol benchmarking tool. + +benchmark_http_client.latency_2xx (298738 samples) + min: 0s 000ms 307us | mean: 0s 011ms 515us | max: 0s 187ms 269us | pstdev: 0s 019ms 783us + + Percentile Count Value + 0.5 149369 0s 002ms 416us + 0.75 224054 0s 011ms 230us + 0.8 238992 0s 016ms 571us + 0.9 268865 0s 041ms 666us + 0.95 283803 0s 059ms 402us + 0.990625 295939 0s 085ms 483us + 0.99902344 298447 0s 120ms 631us + +benchmark_http_client.latency_5xx (7 samples) + min: 0s 000ms 210us | mean: 0s 000ms 884us | max: 0s 001ms 337us | pstdev: 0s 000ms 404us + + Percentile Count Value + 0.5 4 0s 000ms 683us + 0.75 6 0s 001ms 309us + 0.8 6 0s 001ms 309us + +Queueing and connection setup latency (298836 samples) + min: 0s 000ms 001us | mean: 0s 000ms 016us | max: 0s 075ms 845us | pstdev: 0s 000ms 315us + + Percentile Count Value + 0.5 149531 0s 000ms 008us + 0.75 224310 0s 000ms 010us + 0.8 239171 0s 000ms 010us + 0.9 268973 0s 000ms 011us + 0.95 283896 0s 000ms 016us + 0.990625 296035 0s 000ms 061us + 0.99902344 298545 0s 001ms 244us + +Request start to response end (298745 samples) + min: 0s 000ms 210us | mean: 0s 011ms 514us | max: 0s 187ms 269us | pstdev: 0s 019ms 783us + + Percentile Count Value + 0.5 149373 0s 002ms 415us + 0.75 224059 0s 011ms 229us + 0.8 238996 0s 016ms 569us + 0.9 268871 0s 041ms 664us + 0.95 283810 0s 059ms 402us + 0.990625 295946 0s 085ms 483us + 0.99902344 298454 0s 120ms 631us + +Response body size in bytes (298745 samples) + min: 0 | mean: 9.999765686455003 | max: 10 | pstdev: 0.048405377254271256 + +Response header size in bytes (298745 samples) + min: 58 | mean: 109.99878156956602 | max: 110 | pstdev: 0.25170796172221055 + +Blocking. Results are skewed when significant numbers are reported here. (8534 samples) + min: 0s 000ms 040us | mean: 0s 006ms 742us | max: 0s 101ms 433us | pstdev: 0s 012ms 626us + + Percentile Count Value + 0.5 4267 0s 001ms 369us + 0.75 6401 0s 005ms 750us + 0.8 6828 0s 008ms 406us + 0.9 7681 0s 022ms 739us + 0.95 8108 0s 037ms 554us + 0.990625 8454 0s 058ms 765us + 0.99902344 8526 0s 078ms 221us + +Initiation to completion (298992 samples) + min: 0s 000ms 005us | mean: 0s 011ms 618us | max: 0s 187ms 277us | pstdev: 0s 019ms 857us + + Percentile Count Value + 0.5 149496 0s 002ms 465us + 0.75 224244 0s 011ms 490us + 0.8 239195 0s 016ms 867us + 0.9 269093 0s 041ms 924us + 0.95 284043 0s 059ms 586us + 0.990625 296189 0s 085ms 766us + 0.99902344 298701 0s 120ms 954us Counter Value Per second -benchmark.http_2xx 359809 3997.88 -benchmark.pool_overflow 189 2.10 -cluster_manager.cluster_added 4 0.04 -default.total_match_count 4 0.04 -membership_change 4 0.04 +benchmark.http_2xx 298738 3995.38 +benchmark.http_5xx 7 0.09 +benchmark.pool_overflow 247 3.30 +cluster_manager.cluster_added 4 0.05 +default.total_match_count 4 0.05 +membership_change 4 0.05 runtime.load_success 1 0.01 runtime.override_dir_not_exists 1 0.01 -upstream_cx_http1_total 84 0.93 -upstream_cx_rx_bytes_total 56490013 627666.63 -upstream_cx_total 84 0.93 -upstream_cx_tx_bytes_total 15471873 171909.65 -upstream_rq_pending_overflow 189 2.10 -upstream_rq_pending_total 84 0.93 -upstream_rq_total 359811 3997.90 - -[09:47:56.790350][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. -[09:47:56.792949][1][I] Done. +sequencer.failed_terminations 4 0.05 +upstream_cx_http1_total 153 2.05 +upstream_cx_rx_bytes_total 46902510 627283.31 +upstream_cx_total 153 2.05 +upstream_cx_tx_bytes_total 12849948 171857.71 +upstream_rq_pending_overflow 247 3.30 +upstream_rq_pending_total 153 2.05 +upstream_rq_total 298836 3996.69 + +[14:46:01.842997][19][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[14:46:06.845068][20][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[14:46:11.847277][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. +[14:46:11.850206][1][E] An error ocurred. ``` @@ -710,88 +821,89 @@ upstream_rq_total 359811 3997.90 scale-down-httproutes-10 ```plaintext +[2024-06-25 14:48:06.510][1][warning][misc] [external/envoy/source/common/protobuf/message_validator_impl.cc:21] Deprecated field: type envoy.config.core.v3.HeaderValueOption Using deprecated option 'envoy.config.core.v3.HeaderValueOption.append' from file base.proto. This configuration will be removed from Envoy soon. Please see https://www.envoyproxy.io/docs/envoy/latest/version_history/version_history for details. If continued use of this field is absolutely necessary, see https://www.envoyproxy.io/docs/envoy/latest/configuration/operations/runtime#using-runtime-overrides-for-deprecated-features for how to apply a temporary and highly discouraged override. +[14:48:06.510881][1][I] Detected 4 (v)CPUs with affinity.. +[14:48:06.510893][1][I] Starting 4 threads / event loops. Time limit: 90 seconds. +[14:48:06.510895][1][I] Global targets: 400 connections and 4000 calls per second. +[14:48:06.510897][1][I] (Per-worker targets: 100 connections and 1000 calls per second) +[14:49:37.212970][19][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9998000000401 per second.) +[14:49:37.213444][21][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9973000072899 per second.) +[14:49:37.213541][23][I] Stopping after 90000 ms. Initiated: 90000 / Completed: 90000. (Completion rate was 999.9989777788228 per second.) +[14:49:37.213971][18][I] Stopping after 90001 ms. Initiated: 89999 / Completed: 89996. (Completion rate was 999.9414452707166 per second.) +Nighthawk - A layer 7 protocol benchmarking tool. -benchmark_http_client.latency_2xx (359744 samples) - min: 0s 000ms 300us | mean: 0s 000ms 552us | max: 0s 062ms 459us | pstdev: 0s 000ms 650us +benchmark_http_client.latency_2xx (359809 samples) + min: 0s 000ms 297us | mean: 0s 000ms 525us | max: 0s 048ms 130us | pstdev: 0s 000ms 466us +[14:49:42.728878][18][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. Percentile Count Value - 0.5 179903 0s 000ms 462us - 0.75 269816 0s 000ms 540us - 0.8 287814 0s 000ms 563us - 0.9 323771 0s 000ms 709us - 0.95 341759 0s 000ms 807us - 0.990625 356373 0s 001ms 984us - 0.99902344 359393 0s 008ms 610us + 0.5 179925 0s 000ms 450us + 0.75 269871 0s 000ms 525us + 0.8 287855 0s 000ms 549us + 0.9 323830 0s 000ms 661us + 0.95 341819 0s 000ms 762us + 0.990625 356436 0s 001ms 585us + 0.99902344 359458 0s 007ms 084us -Queueing and connection setup latency (359746 samples) - min: 0s 000ms 001us | mean: 0s 000ms 011us | max: 0s 017ms 278us | pstdev: 0s 000ms 043us +Queueing and connection setup latency (359812 samples) + min: 0s 000ms 002us | mean: 0s 000ms 012us | max: 0s 034ms 091us | pstdev: 0s 000ms 067us Percentile Count Value - 0.5 179917 0s 000ms 010us - 0.75 270948 0s 000ms 011us - 0.8 288210 0s 000ms 011us - 0.9 323879 0s 000ms 011us - 0.95 341762 0s 000ms 012us - 0.990625 356374 0s 000ms 031us - 0.99902344 359395 0s 000ms 173us + 0.5 180343 0s 000ms 010us + 0.75 269955 0s 000ms 011us + 0.8 288513 0s 000ms 011us + 0.9 323857 0s 000ms 012us + 0.95 341824 0s 000ms 023us + 0.990625 356439 0s 000ms 050us + 0.99902344 359461 0s 000ms 189us -Request start to response end (359744 samples) - min: 0s 000ms 299us | mean: 0s 000ms 552us | max: 0s 062ms 459us | pstdev: 0s 000ms 650us +Request start to response end (359809 samples) + min: 0s 000ms 297us | mean: 0s 000ms 524us | max: 0s 048ms 130us | pstdev: 0s 000ms 466us Percentile Count Value - 0.5 179873 0s 000ms 461us - 0.75 269814 0s 000ms 540us - 0.8 287814 0s 000ms 562us - 0.9 323778 0s 000ms 709us - 0.95 341757 0s 000ms 807us - 0.990625 356372 0s 001ms 984us - 0.99902344 359393 0s 008ms 610us + 0.5 179906 0s 000ms 450us + 0.75 269861 0s 000ms 524us + 0.8 287864 0s 000ms 549us + 0.9 323829 0s 000ms 661us + 0.95 341820 0s 000ms 761us + 0.990625 356436 0s 001ms 585us + 0.99902344 359458 0s 007ms 083us -Response body size in bytes (359744 samples) +Response body size in bytes (359809 samples) min: 10 | mean: 10 | max: 10 | pstdev: 0 -Response header size in bytes (359744 samples) +Response header size in bytes (359809 samples) min: 110 | mean: 110 | max: 110 | pstdev: 0 -Blocking. Results are skewed when significant numbers are reported here. (9 samples) - min: 0s 000ms 088us | mean: 0s 000ms 567us | max: 0s 001ms 563us | pstdev: 0s 000ms 563us +Initiation to completion (359996 samples) + min: 0s 000ms 005us | mean: 0s 000ms 543us | max: 0s 048ms 146us | pstdev: 0s 000ms 476us Percentile Count Value - 0.5 5 0s 000ms 257us - 0.75 7 0s 001ms 061us - 0.8 8 0s 001ms 402us - 0.9 9 0s 001ms 563us - -Initiation to completion (359998 samples) - min: 0s 000ms 006us | mean: 0s 000ms 570us | max: 0s 062ms 482us | pstdev: 0s 000ms 661us - - Percentile Count Value - 0.5 180035 0s 000ms 479us - 0.75 270019 0s 000ms 558us - 0.8 288010 0s 000ms 580us - 0.9 324007 0s 000ms 725us - 0.95 341999 0s 000ms 829us - 0.990625 356624 0s 002ms 038us - 0.99902344 359647 0s 008ms 940us + 0.5 179999 0s 000ms 468us + 0.75 269997 0s 000ms 543us + 0.8 288002 0s 000ms 568us + 0.9 324002 0s 000ms 683us + 0.95 341998 0s 000ms 783us + 0.990625 356622 0s 001ms 632us + 0.99902344 359645 0s 007ms 247us Counter Value Per second -benchmark.http_2xx 359744 3997.14 -benchmark.pool_overflow 254 2.82 +benchmark.http_2xx 359809 3997.86 +benchmark.pool_overflow 187 2.08 cluster_manager.cluster_added 4 0.04 default.total_match_count 4 0.04 membership_change 4 0.04 runtime.load_success 1 0.01 runtime.override_dir_not_exists 1 0.01 -upstream_cx_http1_total 74 0.82 -upstream_cx_rx_bytes_total 56479808 627551.51 -upstream_cx_total 74 0.82 -upstream_cx_tx_bytes_total 15469078 171878.12 -upstream_rq_pending_overflow 254 2.82 -upstream_rq_pending_total 74 0.82 -upstream_rq_total 359746 3997.17 +upstream_cx_http1_total 66 0.73 +upstream_cx_rx_bytes_total 56490013 627663.98 +upstream_cx_total 66 0.73 +upstream_cx_tx_bytes_total 15471916 171909.40 +upstream_rq_pending_overflow 187 2.08 +upstream_rq_pending_total 66 0.73 +upstream_rq_total 359812 3997.89 -[09:49:49.788004][23][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. -[09:49:49.789745][1][I] Done. +[14:49:42.732798][1][I] Done. ``` @@ -799,14 +911,15 @@ upstream_rq_total 359746 3997.17 ### Metrics -|Benchmark Name |Envoy Gateway Memory (MiB)|Envoy Gateway Total CPU (Seconds)| -|- |- |- | -|scale-up-httproutes-10 |77 |0.35 | -|scale-up-httproutes-50 |90 |1.96 | -|scale-up-httproutes-100 |161 |9.97 | -|scale-up-httproutes-300 |883 |185.43 | -|scale-up-httproutes-500 |1473 |10.42 | -|scale-down-httproutes-300|697 |5.73 | -|scale-down-httproutes-100|463 |103.44 | -|scale-down-httproutes-50 |112 |161.11 | -|scale-down-httproutes-10 |116 |162.41 | +|Benchmark Name |Envoy Gateway Memory (MiB)|Envoy Gateway Total CPU (Seconds)|Envoy Proxy Memory: k5s2s[1] (MiB)| +|- |- |- |- | +|scale-up-httproutes-10 |76 |0.34 |7 | +|scale-up-httproutes-50 |114 |1.94 |12 | +|scale-up-httproutes-100 |195 |10.85 |20 | +|scale-up-httproutes-300 |1124 |176.36 |81 | +|scale-up-httproutes-500 |1588 |16.69 |190 | +|scale-down-httproutes-300|661 |6.04 |87 | +|scale-down-httproutes-100|679 |104.73 |95 | +|scale-down-httproutes-50 |143 |172.84 |53 | +|scale-down-httproutes-10 |118 |174.16 |18 | +1. envoy-gateway-system/envoy-benchmark-test-benchmark-0520098c-7668c94dd5-k5s2s From 3dc4472ccdad5e814d85511437ec427bdfc93075 Mon Sep 17 00:00:00 2001 From: shawnh2 Date: Thu, 27 Jun 2024 08:52:02 +0800 Subject: [PATCH 30/30] fix lint and address comments Signed-off-by: shawnh2 --- .github/workflows/benchmark.yaml | 9 ++++++--- test/benchmark/benchmark_report.md | 2 +- tools/make/kube.mk | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 4b07a99621f..9f87ca4cbc3 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -7,8 +7,8 @@ on: workflow_dispatch: inputs: rps: - description: "The target requests-per-second rate. Default: 1000" - default: '1000' + description: "The target requests-per-second rate. Default: 10000" + default: '10000' type: string required: false connections: @@ -45,9 +45,12 @@ jobs: env: KIND_NODE_TAG: v1.28.0 IMAGE_PULL_POLICY: IfNotPresent - BENCHMARK_RPS: ${{ github.event.inputs.rps || 1000 }} + BENCHMARK_RPS: ${{ github.event.inputs.rps || 10000 }} BENCHMARK_CONNECTIONS: ${{ github.event.inputs.connections || 100 }} BENCHMARK_DURATION: ${{ github.event.inputs.duration || 90 }} BENCHMARK_CPU_LIMITS: ${{ github.event.inputs.cpu_limits || 1000 }} BENCHMARK_MEMORY_LIMITS: ${{ github.event.inputs.memory_limits || 2048 }} run: make benchmark + + - name: Read Benchmark report + run: cat test/benchmark/benchmark_report.md diff --git a/test/benchmark/benchmark_report.md b/test/benchmark/benchmark_report.md index c62199e7b0c..4d6d57da1a0 100644 --- a/test/benchmark/benchmark_report.md +++ b/test/benchmark/benchmark_report.md @@ -811,7 +811,7 @@ upstream_rq_total 298836 3996.69 [14:46:01.842997][19][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. [14:46:06.845068][20][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. [14:46:11.847277][21][I] Wait for the connection pool drain timed out, proceeding to hard shutdown. -[14:46:11.850206][1][E] An error ocurred. +[14:46:11.850206][1][E] An error occurred. ``` diff --git a/tools/make/kube.mk b/tools/make/kube.mk index a1562815312..d4dd8831110 100644 --- a/tools/make/kube.mk +++ b/tools/make/kube.mk @@ -13,7 +13,7 @@ WAIT_TIMEOUT ?= 15m BENCHMARK_TIMEOUT ?= 60m BENCHMARK_CPU_LIMITS ?= 1000 # unit: 'm' BENCHMARK_MEMORY_LIMITS ?= 1024 # unit: 'Mi' -BENCHMARK_RPS ?= 1000 +BENCHMARK_RPS ?= 10000 BENCHMARK_CONNECTIONS ?= 100 BENCHMARK_DURATION ?= 60 @@ -163,7 +163,7 @@ run-benchmark: install-benchmark-server ## Run benchmark tests kubectl wait --timeout=$(WAIT_TIMEOUT) -n benchmark-test deployment/nighthawk-test-server --for=condition=Available kubectl wait --timeout=$(WAIT_TIMEOUT) -n envoy-gateway-system deployment/envoy-gateway --for=condition=Available kubectl apply -f test/benchmark/config/gatewayclass.yaml - go test -v -tags benchmark -timeout $(BENCHMARK_TIMEOUT) ./test/benchmark --rps=$(BENCHMARK_RPS) --connections=$(BENCHMARK_CONNECTIONS) --duration=$(BENCHMARK_DURATION) --report-save-path=$(OUTPUT_DIR)/benchmark/report.md + go test -v -tags benchmark -timeout $(BENCHMARK_TIMEOUT) ./test/benchmark --rps=$(BENCHMARK_RPS) --connections=$(BENCHMARK_CONNECTIONS) --duration=$(BENCHMARK_DURATION) --report-save-path=benchmark_report.md .PHONY: install-benchmark-server install-benchmark-server: ## Install nighthawk server for benchmark test