-
Notifications
You must be signed in to change notification settings - Fork 366
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
e2e: add test for BackendTrafficPolicy retry (#2738)
* e2e: add test for BackendTrafficPolicy retry Signed-off-by: zirain <[email protected]> * lint Signed-off-by: zirain <[email protected]> --------- Signed-off-by: zirain <[email protected]> Co-authored-by: Xunzhuo <[email protected]>
- Loading branch information
Showing
6 changed files
with
259 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| # To simplify the deployment, disable non-essential components | ||
| alertmanager: | ||
| enabled: false | ||
| prometheus-pushgateway: | ||
| enabled: false | ||
| kube-state-metrics: | ||
| enabled: false | ||
| prometheus-node-exporter: | ||
| enabled: false | ||
| server: | ||
| fullnameOverride: prometheus | ||
| persistentVolume: | ||
| enabled: false | ||
| readinessProbeInitialDelay: 0 | ||
| global: | ||
| # Speed up scraping a bit from the default | ||
| scrape_interval: 15s | ||
| service: | ||
| # use LoadBalancer to expose prometheus | ||
| type: LoadBalancer | ||
| # use dockerhub | ||
| image: | ||
| repository: prom/prometheus | ||
| securityContext: null | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| apiVersion: gateway.envoyproxy.io/v1alpha1 | ||
| kind: BackendTrafficPolicy | ||
| metadata: | ||
| name: retry-policy | ||
| namespace: gateway-conformance-infra | ||
| spec: | ||
| targetRef: | ||
| group: gateway.networking.k8s.io | ||
| kind: HTTPRoute | ||
| name: retry-route | ||
| namespace: gateway-conformance-infra | ||
| retry: | ||
| numRetries: 5 | ||
| perRetry: | ||
| backOff: | ||
| baseInterval: 100ms | ||
| maxInterval: 10s | ||
| timeout: 250ms | ||
| retryOn: | ||
| httpStatusCodes: | ||
| - 500 | ||
| triggers: | ||
| - connect-failure | ||
| - retriable-status-codes | ||
| --- | ||
| apiVersion: gateway.networking.k8s.io/v1 | ||
| kind: HTTPRoute | ||
| metadata: | ||
| name: retry-route | ||
| namespace: gateway-conformance-infra | ||
| spec: | ||
| parentRefs: | ||
| - name: same-namespace | ||
| rules: | ||
| - matches: | ||
| - path: | ||
| type: PathPrefix | ||
| value: /status | ||
| backendRefs: | ||
| - name: infra-backend-v1 | ||
| port: 8080 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| // Copyright Envoy Gateway Authors | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // The full text of the Apache license is available in the LICENSE file at | ||
| // the root of the repo. | ||
|
|
||
| //go:build e2e | ||
| // +build e2e | ||
|
|
||
| package tests | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "testing" | ||
| "time" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| "k8s.io/apimachinery/pkg/types" | ||
| "sigs.k8s.io/gateway-api/conformance/utils/http" | ||
| "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" | ||
| "sigs.k8s.io/gateway-api/conformance/utils/suite" | ||
|
|
||
| "github.com/envoyproxy/gateway/test/e2e/utils/prometheus" | ||
| ) | ||
|
|
||
| func init() { | ||
| ConformanceTests = append(ConformanceTests, RetryTest) | ||
|
|
||
| } | ||
|
|
||
| var RetryTest = suite.ConformanceTest{ | ||
| ShortName: "Retry", | ||
| Description: "Test that the BackendTrafficPolicy API implementation supports retry", | ||
| Manifests: []string{"testdata/retry.yaml"}, | ||
| Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { | ||
| t.Run("retry-on-500", func(t *testing.T) { | ||
| ns := "gateway-conformance-infra" | ||
| routeNN := types.NamespacedName{Name: "retry-route", Namespace: ns} | ||
| gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns} | ||
| gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) | ||
|
|
||
| expectedResponse := http.ExpectedResponse{ | ||
| Request: http.Request{ | ||
| Path: "/status/500", | ||
| }, | ||
| Response: http.Response{ | ||
| StatusCode: 500, | ||
| }, | ||
| Namespace: ns, | ||
| } | ||
|
|
||
| promAddr, err := prometheus.Address(suite.Client, types.NamespacedName{Name: "prometheus", Namespace: "monitoring"}) | ||
| require.NoError(t, err) | ||
| promQL := fmt.Sprintf(`envoy_cluster_upstream_rq_retry{envoy_cluster_name="httproute/%s/%s/rule/0"}`, routeNN.Namespace, routeNN.Name) | ||
|
|
||
| before := float64(0) | ||
| v, err := prometheus.QuerySum(promAddr, promQL) | ||
| if err == nil { | ||
| before = v | ||
| } | ||
| t.Logf("query count %s before: %v", promQL, before) | ||
|
|
||
| req := http.MakeRequest(t, &expectedResponse, gwAddr, "HTTP", "http") | ||
| cReq, cResp, err := suite.RoundTripper.CaptureRoundTrip(req) | ||
| if err != nil { | ||
| t.Errorf("failed to get expected response: %v", err) | ||
| } | ||
|
|
||
| if err := http.CompareRequest(t, &req, cReq, cResp, expectedResponse); err != nil { | ||
| t.Errorf("failed to compare request and response: %v", err) | ||
| } | ||
|
|
||
| http.AwaitConvergence(t, | ||
| suite.TimeoutConfig.RequiredConsecutiveSuccesses, | ||
| suite.TimeoutConfig.MaxTimeToConsistency, | ||
| func(_ time.Duration) bool { | ||
| // check retry stats from Prometheus | ||
| v, err := prometheus.QuerySum(promAddr, promQL) | ||
| if err != nil { | ||
| return false | ||
| } | ||
| t.Logf("query count %s after: %v", promQL, v) | ||
|
|
||
| delta := int64(v - before) | ||
| // numRetries is 5, so delta mod 5 equals 0 | ||
| return delta > 0 && delta%5 == 0 | ||
| }) | ||
| }) | ||
| }, | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| // Copyright Envoy Gateway Authors | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // The full text of the Apache license is available in the LICENSE file at | ||
| // the root of the repo. | ||
|
|
||
| //go:build e2e | ||
| // +build e2e | ||
|
|
||
| package prometheus | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "time" | ||
|
|
||
| prom "github.com/prometheus/client_golang/api" | ||
| prompapiv1 "github.com/prometheus/client_golang/api/prometheus/v1" | ||
| "github.com/prometheus/common/model" | ||
| corev1 "k8s.io/api/core/v1" | ||
| "k8s.io/apimachinery/pkg/types" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
| ) | ||
|
|
||
| func Address(c client.Client, nn types.NamespacedName) (string, error) { | ||
| svc := &corev1.Service{} | ||
| if err := c.Get(context.TODO(), nn, svc); err != nil { | ||
| return "", fmt.Errorf("failed to get service: %w", err) | ||
| } | ||
| for _, ing := range svc.Status.LoadBalancer.Ingress { | ||
| if ing.IP != "" { | ||
| return fmt.Sprintf("http://%s", ing.IP), nil | ||
| } | ||
| } | ||
|
|
||
| return "", fmt.Errorf("no ingress found") | ||
| } | ||
|
|
||
| func RawQuery(address string, promQL string) (model.Value, error) { | ||
| c, err := prom.NewClient(prom.Config{Address: address}) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| v, _, err := prompapiv1.NewAPI(c).Query(context.Background(), promQL, time.Now()) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| switch v.Type() { | ||
| case model.ValScalar, model.ValString: | ||
| return v, nil | ||
| case model.ValVector: | ||
| value := v.(model.Vector) | ||
| if len(value) == 0 { | ||
| return nil, fmt.Errorf("value not found (query: %v)", promQL) | ||
| } | ||
| return v, nil | ||
|
|
||
| default: | ||
| return nil, fmt.Errorf("unhandled value type: %v", v.Type()) | ||
| } | ||
| } | ||
|
|
||
| func QuerySum(address string, promQL string) (float64, error) { | ||
| val, err := RawQuery(address, promQL) | ||
| if err != nil { | ||
| return 0, err | ||
| } | ||
| got, err := sum(val) | ||
| if err != nil { | ||
| return 0, fmt.Errorf("could not find metric value: %w", err) | ||
| } | ||
| return got, nil | ||
| } | ||
|
|
||
| func sum(val model.Value) (float64, error) { | ||
| if val.Type() != model.ValVector { | ||
| return 0, fmt.Errorf("value not a model.Vector; was %s", val.Type().String()) | ||
| } | ||
|
|
||
| value := val.(model.Vector) | ||
|
|
||
| valueCount := 0.0 | ||
| for _, sample := range value { | ||
| valueCount += float64(sample.Value) | ||
| } | ||
|
|
||
| if valueCount > 0.0 { | ||
| return valueCount, nil | ||
| } | ||
| return 0, fmt.Errorf("value not found") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters