From b6f43068f28ce76d7170e42d6c8241e41cdb1e5b Mon Sep 17 00:00:00 2001 From: Guy Daich Date: Thu, 21 Dec 2023 16:58:30 -0600 Subject: [PATCH] feat(translator): Implement BTP CircuitBreaker API (#2330) * Implement BTP CircuitBreaker API Signed-off-by: Guy Daich * Fix testdate gen Signed-off-by: Guy Daich * review fixes Signed-off-by: Guy Daich * small fix, rebase Signed-off-by: Guy Daich * gen fix Signed-off-by: Guy Daich --------- Signed-off-by: Guy Daich --- internal/gatewayapi/backendtrafficpolicy.go | 68 ++++ .../gatewayapi/backendtrafficpolicy_test.go | 52 +++ ...cpolicy-with-circuitbreakers-error.in.yaml | 44 +++ ...policy-with-circuitbreakers-error.out.yaml | 141 +++++++++ ...trafficpolicy-with-circuitbreakers.in.yaml | 95 ++++++ ...rafficpolicy-with-circuitbreakers.out.yaml | 297 ++++++++++++++++++ internal/ir/xds.go | 15 + internal/ir/zz_generated.deepcopy.go | 35 +++ internal/xds/translator/cluster.go | 47 ++- .../testdata/in/xds-ir/circuit-breaker.yaml | 19 ++ .../out/xds-ir/circuit-breaker.clusters.yaml | 19 ++ .../out/xds-ir/circuit-breaker.endpoints.yaml | 12 + .../out/xds-ir/circuit-breaker.listeners.yaml | 33 ++ .../out/xds-ir/circuit-breaker.routes.yaml | 12 + internal/xds/translator/translator.go | 13 +- internal/xds/translator/translator_test.go | 3 + test/e2e/testdata/circuitbreaker.yaml | 32 ++ test/e2e/tests/circuitbreaker.go | 59 ++++ 18 files changed, 984 insertions(+), 12 deletions(-) create mode 100644 internal/gatewayapi/backendtrafficpolicy_test.go create mode 100644 internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers-error.in.yaml create mode 100755 internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers-error.out.yaml create mode 100644 internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers.in.yaml create mode 100755 internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers.out.yaml create mode 100644 internal/xds/translator/testdata/in/xds-ir/circuit-breaker.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/circuit-breaker.clusters.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/circuit-breaker.endpoints.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/circuit-breaker.listeners.yaml create mode 100644 internal/xds/translator/testdata/out/xds-ir/circuit-breaker.routes.yaml create mode 100644 test/e2e/testdata/circuitbreaker.yaml create mode 100644 test/e2e/tests/circuitbreaker.go diff --git a/internal/gatewayapi/backendtrafficpolicy.go b/internal/gatewayapi/backendtrafficpolicy.go index 0b2a964ce76..95c1157019f 100644 --- a/internal/gatewayapi/backendtrafficpolicy.go +++ b/internal/gatewayapi/backendtrafficpolicy.go @@ -244,6 +244,7 @@ func (t *Translator) translateBackendTrafficPolicyForRoute(policy *egv1a1.Backen rl *ir.RateLimit lb *ir.LoadBalancer pp *ir.ProxyProtocol + cb *ir.CircuitBreaker ) // Build IR @@ -256,6 +257,10 @@ func (t *Translator) translateBackendTrafficPolicyForRoute(policy *egv1a1.Backen if policy.Spec.ProxyProtocol != nil { pp = t.buildProxyProtocol(policy) } + if policy.Spec.CircuitBreaker != nil { + cb = t.buildCircuitBreaker(policy) + } + // Apply IR to all relevant routes prefix := irRoutePrefix(route) for _, ir := range xdsIR { @@ -266,6 +271,7 @@ func (t *Translator) translateBackendTrafficPolicyForRoute(policy *egv1a1.Backen r.RateLimit = rl r.LoadBalancer = lb r.ProxyProtocol = pp + r.CircuitBreaker = cb } } } @@ -278,6 +284,7 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(policy *egv1a1.Back rl *ir.RateLimit lb *ir.LoadBalancer pp *ir.ProxyProtocol + cb *ir.CircuitBreaker ) // Build IR @@ -290,6 +297,9 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(policy *egv1a1.Back if policy.Spec.ProxyProtocol != nil { pp = t.buildProxyProtocol(policy) } + if policy.Spec.CircuitBreaker != nil { + cb = t.buildCircuitBreaker(policy) + } // Apply IR to all the routes within the specific Gateway // If the feature is already set, then skip it, since it must be have // set by a policy attaching to the route @@ -309,6 +319,9 @@ func (t *Translator) translateBackendTrafficPolicyForGateway(policy *egv1a1.Back if r.ProxyProtocol == nil { r.ProxyProtocol = pp } + if r.CircuitBreaker == nil { + r.CircuitBreaker = cb + } } } @@ -643,3 +656,58 @@ func ratelimitUnitToDuration(unit egv1a1.RateLimitUnit) int64 { } return seconds } + +func (t *Translator) buildCircuitBreaker(policy *egv1a1.BackendTrafficPolicy) *ir.CircuitBreaker { + var cb *ir.CircuitBreaker + pcb := policy.Spec.CircuitBreaker + + if pcb != nil { + cb = &ir.CircuitBreaker{} + + if pcb.MaxConnections != nil { + if ui32, ok := int64ToUint32(*pcb.MaxConnections); ok { + cb.MaxConnections = &ui32 + } else { + setCircuitBreakerPolicyErrorCondition(policy, fmt.Sprintf("invalid MaxConnections value %d", *pcb.MaxConnections)) + return nil + } + } + + if pcb.MaxParallelRequests != nil { + if ui32, ok := int64ToUint32(*pcb.MaxParallelRequests); ok { + cb.MaxParallelRequests = &ui32 + } else { + setCircuitBreakerPolicyErrorCondition(policy, fmt.Sprintf("invalid MaxParallelRequests value %d", *pcb.MaxParallelRequests)) + return nil + } + } + + if pcb.MaxPendingRequests != nil { + if ui32, ok := int64ToUint32(*pcb.MaxPendingRequests); ok { + cb.MaxPendingRequests = &ui32 + } else { + setCircuitBreakerPolicyErrorCondition(policy, fmt.Sprintf("invalid MaxPendingRequests value %d", *pcb.MaxPendingRequests)) + return nil + } + } + } + + return cb +} + +func setCircuitBreakerPolicyErrorCondition(policy *egv1a1.BackendTrafficPolicy, errMsg string) { + message := fmt.Sprintf("Unable to translate Circuit Breaker: %s", errMsg) + status.SetBackendTrafficPolicyCondition(policy, + gwv1a2.PolicyConditionAccepted, + metav1.ConditionFalse, + gwv1a2.PolicyReasonInvalid, + message, + ) +} + +func int64ToUint32(in int64) (uint32, bool) { + if in >= 0 && in <= math.MaxUint32 { + return uint32(in), true + } + return 0, false +} diff --git a/internal/gatewayapi/backendtrafficpolicy_test.go b/internal/gatewayapi/backendtrafficpolicy_test.go new file mode 100644 index 00000000000..df943a2032b --- /dev/null +++ b/internal/gatewayapi/backendtrafficpolicy_test.go @@ -0,0 +1,52 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package gatewayapi + +import ( + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestInt64ToUint32(t *testing.T) { + type testCase struct { + Name string + In int64 + Out uint32 + Success bool + } + + testCases := []testCase{ + { + Name: "valid", + In: 1024, + Out: 1024, + Success: true, + }, + { + Name: "invalid-underflow", + In: -1, + Out: 0, + Success: false, + }, + { + Name: "invalid-overflow", + In: math.MaxUint32 + 1, + Out: 0, + Success: false, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.Name, func(t *testing.T) { + out, success := int64ToUint32(tc.In) + require.Equal(t, tc.Out, out) + require.Equal(t, tc.Success, success) + }) + } +} diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers-error.in.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers-error.in.yaml new file mode 100644 index 00000000000..d762170ad92 --- /dev/null +++ b/internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers-error.in.yaml @@ -0,0 +1,44 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: GRPCRoute + metadata: + namespace: default + name: grpcroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: envoy-gateway + name: policy-for-gateway + spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + circuitBreaker: + maxConnections: -1 diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers-error.out.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers-error.out.yaml new file mode 100755 index 00000000000..4e3acbd5931 --- /dev/null +++ b/internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers-error.out.yaml @@ -0,0 +1,141 @@ +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-gateway + namespace: envoy-gateway + spec: + circuitBreaker: + maxConnections: -1 + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + status: + conditions: + - lastTransitionTime: null + message: 'Unable to translate Circuit Breaker: invalid MaxConnections value + -1' + reason: Invalid + status: "False" + type: Accepted +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: GRPCRoute + metadata: + creationTimestamp: null + name: grpcroute-1 + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: true + name: envoy-gateway/gateway-1/http + port: 10080 + routes: + - backendWeights: + invalid: 0 + valid: 0 + destination: + name: grpcroute/default/grpcroute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: GRPC + weight: 1 + hostname: '*' + name: grpcroute/default/grpcroute-1/rule/0/match/-1/* diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers.in.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers.in.yaml new file mode 100644 index 00000000000..babae4b3650 --- /dev/null +++ b/internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers.in.yaml @@ -0,0 +1,95 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-2 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: GRPCRoute + metadata: + namespace: default + name: grpcroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - namespace: envoy-gateway + name: gateway-2 + sectionName: http + rules: + - matches: + - path: + value: "/" + backendRefs: + - name: service-1 + port: 8080 +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: envoy-gateway + name: policy-for-gateway + spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + circuitBreaker: + maxConnections: 2048 + maxPendingRequests: 1 + maxParallelRequests: 4294967295 +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + namespace: default + name: policy-for-route + spec: + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-1 + namespace: default + circuitBreaker: + maxConnections: 42 + maxPendingRequests: 42 + maxParallelRequests: 42 diff --git a/internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers.out.yaml b/internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers.out.yaml new file mode 100755 index 00000000000..cea46c84bc3 --- /dev/null +++ b/internal/gatewayapi/testdata/backendtrafficpolicy-with-circuitbreakers.out.yaml @@ -0,0 +1,297 @@ +backendTrafficPolicies: +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-route + namespace: default + spec: + circuitBreaker: + maxConnections: 42 + maxParallelRequests: 42 + maxPendingRequests: 42 + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: httproute-1 + namespace: default + status: + conditions: + - lastTransitionTime: null + message: BackendTrafficPolicy has been accepted. + reason: Accepted + status: "True" + type: Accepted +- apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: BackendTrafficPolicy + metadata: + creationTimestamp: null + name: policy-for-gateway + namespace: envoy-gateway + spec: + circuitBreaker: + maxConnections: 2048 + maxParallelRequests: 4294967295 + maxPendingRequests: 1 + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + status: + conditions: + - lastTransitionTime: null + message: BackendTrafficPolicy has been accepted. + reason: Accepted + status: "True" + type: Accepted +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-2 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 1 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: GRPCRoute + metadata: + creationTimestamp: null + name: grpcroute-1 + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-1 + namespace: default + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - name: gateway-2 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + matches: + - path: + value: / + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-2 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-1: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 + envoy-gateway/gateway-2: + proxy: + listeners: + - address: null + name: envoy-gateway/gateway-2/http + ports: + - containerPort: 10080 + name: http + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-2 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-2 +xdsIR: + envoy-gateway/gateway-1: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: true + name: envoy-gateway/gateway-1/http + port: 10080 + routes: + - backendWeights: + invalid: 0 + valid: 0 + circuitBreaker: + maxConnections: 2048 + maxParallelRequests: 4294967295 + maxPendingRequests: 1 + destination: + name: grpcroute/default/grpcroute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: GRPC + weight: 1 + hostname: '*' + name: grpcroute/default/grpcroute-1/rule/0/match/-1/* + envoy-gateway/gateway-2: + accessLog: + text: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: false + name: envoy-gateway/gateway-2/http + port: 10080 + routes: + - backendWeights: + invalid: 0 + valid: 0 + circuitBreaker: + maxConnections: 42 + maxParallelRequests: 42 + maxPendingRequests: 42 + destination: + name: httproute/default/httproute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + protocol: HTTP + weight: 1 + hostname: gateway.envoyproxy.io + name: httproute/default/httproute-1/rule/0/match/0/gateway_envoyproxy_io + pathMatch: + distinct: false + name: "" + prefix: / diff --git a/internal/ir/xds.go b/internal/ir/xds.go index b9f214f1f38..ea45bc11744 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -306,6 +306,8 @@ type HTTPRoute struct { BasicAuth *BasicAuth `json:"basicAuth,omitempty" yaml:"basicAuth,omitempty"` // ExtensionRefs holds unstructured resources that were introduced by an extension and used on the HTTPRoute as extensionRef filters ExtensionRefs []*UnstructuredRef `json:"extensionRefs,omitempty" yaml:"extensionRefs,omitempty"` + // Circuit Breaker Settings + CircuitBreaker *CircuitBreaker `json:"circuitBreaker,omitempty" yaml:"circuitBreaker,omitempty"` } // UnstructuredRef holds unstructured data for an arbitrary k8s resource introduced by an extension @@ -1146,3 +1148,16 @@ type SlowStart struct { // Window defines the duration of the warm up period for newly added host. Window *metav1.Duration `json:"window" yaml:"window"` } + +// Backend CircuitBreaker settings for the DEFAULT routing priority +// +k8s:deepcopy-gen=true +type CircuitBreaker struct { + // The maximum number of connections that Envoy will establish. + MaxConnections *uint32 `json:"maxConnections,omitempty" yaml:"maxConnections,omitempty"` + + // The maximum number of pending requests that Envoy will queue. + MaxPendingRequests *uint32 `json:"maxPendingRequests,omitempty" yaml:"maxPendingRequests,omitempty"` + + // The maximum number of parallel requests that Envoy will make. + MaxParallelRequests *uint32 `json:"maxParallelRequests,omitempty" yaml:"maxParallelRequests,omitempty"` +} diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index d8520326f66..88cc71bf30b 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -143,6 +143,36 @@ func (in *CORS) DeepCopy() *CORS { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CircuitBreaker) DeepCopyInto(out *CircuitBreaker) { + *out = *in + if in.MaxConnections != nil { + in, out := &in.MaxConnections, &out.MaxConnections + *out = new(uint32) + **out = **in + } + if in.MaxPendingRequests != nil { + in, out := &in.MaxPendingRequests, &out.MaxPendingRequests + *out = new(uint32) + **out = **in + } + if in.MaxParallelRequests != nil { + in, out := &in.MaxParallelRequests, &out.MaxParallelRequests + *out = new(uint32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CircuitBreaker. +func (in *CircuitBreaker) DeepCopy() *CircuitBreaker { + if in == nil { + return nil + } + out := new(CircuitBreaker) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConsistentHash) DeepCopyInto(out *ConsistentHash) { *out = *in @@ -517,6 +547,11 @@ func (in *HTTPRoute) DeepCopyInto(out *HTTPRoute) { } } } + if in.CircuitBreaker != nil { + in, out := &in.CircuitBreaker, &out.CircuitBreaker + *out = new(CircuitBreaker) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HTTPRoute. diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go index d7aa3d21aff..91acdc04a66 100644 --- a/internal/xds/translator/cluster.go +++ b/internal/xds/translator/cluster.go @@ -31,12 +31,13 @@ const ( ) type xdsClusterArgs struct { - name string - settings []*ir.DestinationSetting - tSocket *corev3.TransportSocket - endpointType EndpointType - loadBalancer *ir.LoadBalancer - proxyProtocol *ir.ProxyProtocol + name string + settings []*ir.DestinationSetting + tSocket *corev3.TransportSocket + endpointType EndpointType + loadBalancer *ir.LoadBalancer + proxyProtocol *ir.ProxyProtocol + circuitBreaker *ir.CircuitBreaker } type EndpointType int @@ -130,9 +131,43 @@ func buildXdsCluster(args *xdsClusterArgs) *clusterv3.Cluster { cluster.LbPolicy = clusterv3.Cluster_MAGLEV } + if args.circuitBreaker != nil { + cluster.CircuitBreakers = buildXdsClusterCircuitBreaker(args.circuitBreaker) + } + return cluster } +func buildXdsClusterCircuitBreaker(circuitBreaker *ir.CircuitBreaker) *clusterv3.CircuitBreakers { + cbt := &clusterv3.CircuitBreakers_Thresholds{ + Priority: corev3.RoutingPriority_DEFAULT, + } + + if circuitBreaker.MaxConnections != nil { + cbt.MaxConnections = &wrapperspb.UInt32Value{ + Value: *circuitBreaker.MaxConnections, + } + } + + if circuitBreaker.MaxPendingRequests != nil { + cbt.MaxPendingRequests = &wrapperspb.UInt32Value{ + Value: *circuitBreaker.MaxPendingRequests, + } + } + + if circuitBreaker.MaxParallelRequests != nil { + cbt.MaxRequests = &wrapperspb.UInt32Value{ + Value: *circuitBreaker.MaxParallelRequests, + } + } + + ecb := &clusterv3.CircuitBreakers{ + Thresholds: []*clusterv3.CircuitBreakers_Thresholds{cbt}, + } + + return ecb +} + func buildXdsClusterLoadAssignment(clusterName string, destSettings []*ir.DestinationSetting) *endpointv3.ClusterLoadAssignment { localities := make([]*endpointv3.LocalityLbEndpoints, 0, len(destSettings)) for i, ds := range destSettings { diff --git a/internal/xds/translator/testdata/in/xds-ir/circuit-breaker.yaml b/internal/xds/translator/testdata/in/xds-ir/circuit-breaker.yaml new file mode 100644 index 00000000000..bd61285403d --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/circuit-breaker.yaml @@ -0,0 +1,19 @@ +http: +- name: "first-listener" + address: "0.0.0.0" + port: 10080 + hostnames: + - "*" + routes: + - name: "first-route" + hostname: "*" + circuitBreaker: + maxConnections: 1 + maxPendingRequests: 1 + maxParallelRequests: 1 + destination: + name: "first-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 diff --git a/internal/xds/translator/testdata/out/xds-ir/circuit-breaker.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/circuit-breaker.clusters.yaml new file mode 100644 index 00000000000..59a43e91a45 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/circuit-breaker.clusters.yaml @@ -0,0 +1,19 @@ +- circuitBreakers: + thresholds: + - maxConnections: 1 + maxPendingRequests: 1 + maxRequests: 1 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_ONLY + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: first-route-dest + lbPolicy: LEAST_REQUEST + name: first-route-dest + outlierDetection: {} + perConnectionBufferLimitBytes: 32768 + type: EDS diff --git a/internal/xds/translator/testdata/out/xds-ir/circuit-breaker.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/circuit-breaker.endpoints.yaml new file mode 100644 index 00000000000..3b3f2d09076 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/circuit-breaker.endpoints.yaml @@ -0,0 +1,12 @@ +- clusterName: first-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: first-route-dest/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/circuit-breaker.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/circuit-breaker.listeners.yaml new file mode 100644 index 00000000000..73ee1b42ef6 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/circuit-breaker.listeners.yaml @@ -0,0 +1,33 @@ +- address: + socketAddress: + address: 0.0.0.0 + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: first-listener + statPrefix: http + upgradeConfigs: + - upgradeType: websocket + useRemoteAddress: true + name: first-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/circuit-breaker.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/circuit-breaker.routes.yaml new file mode 100644 index 00000000000..2734c7cc42a --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/circuit-breaker.routes.yaml @@ -0,0 +1,12 @@ +- ignorePortInHostMatching: true + name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener/* + routes: + - match: + prefix: / + name: first-route + route: + cluster: first-route-dest diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go index c9f510f3236..f73f3a0b7f3 100644 --- a/internal/xds/translator/translator.go +++ b/internal/xds/translator/translator.go @@ -498,12 +498,13 @@ func processXdsCluster(tCtx *types.ResourceVersionTable, httpRoute *ir.HTTPRoute } if err := addXdsCluster(tCtx, &xdsClusterArgs{ - name: httpRoute.Destination.Name, - settings: httpRoute.Destination.Settings, - tSocket: nil, - endpointType: endpointType, - loadBalancer: httpRoute.LoadBalancer, - proxyProtocol: httpRoute.ProxyProtocol, + name: httpRoute.Destination.Name, + settings: httpRoute.Destination.Settings, + tSocket: nil, + endpointType: endpointType, + loadBalancer: httpRoute.LoadBalancer, + proxyProtocol: httpRoute.ProxyProtocol, + circuitBreaker: httpRoute.CircuitBreaker, }); err != nil && !errors.Is(err, ErrXdsClusterExists) { return err } diff --git a/internal/xds/translator/translator_test.go b/internal/xds/translator/translator_test.go index ac455679cb8..80e6ca91432 100644 --- a/internal/xds/translator/translator_test.go +++ b/internal/xds/translator/translator_test.go @@ -218,6 +218,9 @@ func TestTranslateXds(t *testing.T) { { name: "local-ratelimit", }, + { + name: "circuit-breaker", + }, } for _, tc := range testCases { diff --git a/test/e2e/testdata/circuitbreaker.yaml b/test/e2e/testdata/circuitbreaker.yaml new file mode 100644 index 00000000000..12557e7ef1a --- /dev/null +++ b/test/e2e/testdata/circuitbreaker.yaml @@ -0,0 +1,32 @@ +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: BackendTrafficPolicy +metadata: + name: circuitbreaker-example + namespace: gateway-conformance-infra +spec: + targetRef: + group: gateway.networking.k8s.io + kind: HTTPRoute + name: http-with-circuitbreaker + namespace: gateway-conformance-infra + circuitBreaker: + maxConnections: 0 + maxRequests: 0 + maxPendingRequests: 0 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: http-with-circuitbreaker + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: same-namespace + rules: + - matches: + - path: + type: PathPrefix + value: /circuitbreaker + backendRefs: + - name: infra-backend-v1 + port: 8080 diff --git a/test/e2e/tests/circuitbreaker.go b/test/e2e/tests/circuitbreaker.go new file mode 100644 index 00000000000..0bce74ba68f --- /dev/null +++ b/test/e2e/tests/circuitbreaker.go @@ -0,0 +1,59 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build e2e +// +build e2e + +package tests + +import ( + "testing" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/gateway-api/conformance/utils/http" + "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" + "sigs.k8s.io/gateway-api/conformance/utils/suite" +) + +func init() { + ConformanceTests = append(ConformanceTests, CircuitBreakerTest) +} + +var CircuitBreakerTest = suite.ConformanceTest{ + ShortName: "CircuitBreaker", + Description: "Deny All Requests", + Manifests: []string{"testdata/circuitbreaker.yaml"}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + t.Run("Deny All Requests", func(t *testing.T) { + ns := "gateway-conformance-infra" + routeNN := types.NamespacedName{Name: "http-with-circuitbreaker", Namespace: ns} + gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns} + gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) + // expect overflow since the policy applies a "closed" circuit breaker + expectedResponse := http.ExpectedResponse{ + Request: http.Request{ + Path: "/circuitbreaker", + }, + Response: http.Response{ + StatusCode: 503, + Headers: map[string]string{ + "x-envoy-overloaded": "true", + }, + }, + Namespace: ns, + } + + req := http.MakeRequest(t, &expectedResponse, gwAddr, "HTTP", "http") + cReq, cResp, err := suite.RoundTripper.CaptureRoundTrip(req) + if err != nil { + t.Errorf("failed to get expected response: %v", err) + } + + if err := http.CompareRequest(t, &req, cReq, cResp, expectedResponse); err != nil { + t.Errorf("failed to compare request and response: %v", err) + } + }) + }, +}