From 4ee9727209bbfbcdfafd498c38bf24700de7df71 Mon Sep 17 00:00:00 2001 From: Dino Fizzotti Date: Wed, 27 Sep 2023 16:34:59 +0100 Subject: [PATCH] Adding envoy cluster max connections property to external service --- Dockerfile | 2 +- Makefile | 2 +- api/v1/externalservice_types.go | 5 + api/v1/zz_generated.deepcopy.go | 5 + .../egress.monzo.com_externalservices.yaml | 7 ++ controllers/configmap.go | 15 +++ controllers/configmap_test.go | 111 +++++++++++++++++- 7 files changed, 143 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 82cd3d97..8c230807 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.20 as builder +FROM golang@sha256:d6df8b2e22c9c87fde828b18e0d0d5707bfe03034a49c0bde72ff5a1f5ebb5fe as builder WORKDIR /workspace # Copy the Go Modules manifests diff --git a/Makefile b/Makefile index ec8c8dcd..09b8139f 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ ifeq (, $(shell which controller-gen)) CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ cd $$CONTROLLER_GEN_TMP_DIR ;\ go mod init tmp ;\ - go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.4 ;\ + go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.0 ;\ rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ } CONTROLLER_GEN=$(GOBIN)/controller-gen diff --git a/api/v1/externalservice_types.go b/api/v1/externalservice_types.go index 21537fd5..1a3fafca 100644 --- a/api/v1/externalservice_types.go +++ b/api/v1/externalservice_types.go @@ -57,6 +57,11 @@ type ExternalServiceSpec struct { // When set allows overwriting the A records of the DNS being overridden. // +optional IpOverride []string `json:"ipOverride,omitempty"` + + // The maximum number of connections that Envoy will establish to all hosts in an upstream cluster (defaults to 1024). + // If this circuit breaker overflows the upstream_cx_overflow counter for the cluster will increment. + // +optional + EnvoyClusterMaxConnections *uint32 `json:"envoyClusterMaxConnections,omitempty"` } type ExternalServicePort struct { diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 21930184..3c597f35 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -139,6 +139,11 @@ func (in *ExternalServiceSpec) DeepCopyInto(out *ExternalServiceSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.EnvoyClusterMaxConnections != nil { + in, out := &in.EnvoyClusterMaxConnections, &out.EnvoyClusterMaxConnections + *out = new(uint32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalServiceSpec. diff --git a/config/crd/bases/egress.monzo.com_externalservices.yaml b/config/crd/bases/egress.monzo.com_externalservices.yaml index 1572503a..bd084788 100644 --- a/config/crd/bases/egress.monzo.com_externalservices.yaml +++ b/config/crd/bases/egress.monzo.com_externalservices.yaml @@ -39,6 +39,13 @@ spec: dnsName: description: DnsName is a DNS name target for the external service type: string + envoyClusterMaxConnections: + description: The maximum number of connections that Envoy will establish + to all hosts in an upstream cluster (defaults to 1024). If this + circuit breaker overflows the upstream_cx_overflow counter for the + cluster will increment. + format: int32 + type: integer hijackDns: description: 'If true, add a `egress.monzo.com/hijack-dns: true` label to produced Service objects CoreDNS can watch this label and decide diff --git a/controllers/configmap.go b/controllers/configmap.go index 06157691..4a41493a 100644 --- a/controllers/configmap.go +++ b/controllers/configmap.go @@ -16,6 +16,7 @@ import ( aggregatev3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" tcpproxyv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3" udpproxyv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/udp/udp_proxy/v3" + "github.com/golang/protobuf/ptypes/wrappers" "google.golang.org/protobuf/types/known/wrapperspb" "github.com/golang/protobuf/jsonpb" @@ -171,6 +172,20 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) { clusterNameForListener = aggregateCluster.Name } + if es.Spec.EnvoyClusterMaxConnections != nil { + cbs := &envoyv3.CircuitBreakers{ + Thresholds: []*envoyv3.CircuitBreakers_Thresholds{ + { + MaxConnections: &wrappers.UInt32Value{Value: *es.Spec.EnvoyClusterMaxConnections}, + }, + }, + PerHostThresholds: nil, + } + for _, cluster := range clusters { + cluster.CircuitBreakers = cbs + } + } + var listener *envoylistener.Listener switch protocol { case envoycorev3.SocketAddress_TCP: diff --git a/controllers/configmap_test.go b/controllers/configmap_test.go index c44e7f46..b3f8c69f 100644 --- a/controllers/configmap_test.go +++ b/controllers/configmap_test.go @@ -13,8 +13,11 @@ import ( func Test_envoyConfig(t *testing.T) { udp := corev1.ProtocolUDP tcp := corev1.ProtocolTCP + var maxConnections uint32 + maxConnections = 4096 type args struct { - es *egressv1.ExternalService + es *egressv1.ExternalService + maxConns *uint32 } tests := []struct { name string @@ -105,7 +108,7 @@ staticResources: name: foo_TCP_101 `, args: args{ - &egressv1.ExternalService{ + es: &egressv1.ExternalService{ ObjectMeta: metav1.ObjectMeta{ Name: "foo", Namespace: "foo", @@ -124,11 +127,115 @@ staticResources: }, }, }, + maxConns: nil, + }, + }, + { + name: "udp and tcp with max connections", + want: `admin: + accessLogPath: /dev/stdout + address: + socketAddress: + address: 0.0.0.0 + portValue: 11000 +node: + cluster: foo +staticResources: + clusters: + - circuitBreakers: + thresholds: + - maxConnections: 4096 + connectTimeout: 1s + dnsLookupFamily: V4_ONLY + loadAssignment: + clusterName: foo_UDP_100 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: google.com + portValue: 100 + protocol: UDP + name: foo_UDP_100 + type: LOGICAL_DNS + - circuitBreakers: + thresholds: + - maxConnections: 4096 + connectTimeout: 1s + dnsLookupFamily: V4_ONLY + loadAssignment: + clusterName: foo_TCP_101 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: google.com + portValue: 101 + name: foo_TCP_101 + type: LOGICAL_DNS + listeners: + - address: + socketAddress: + address: 0.0.0.0 + portValue: 100 + protocol: UDP + filterChains: + - filters: + - name: envoy.filters.udp_listener.udp_proxy + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig + cluster: foo_UDP_100 + statPrefix: udp_proxy + name: foo_UDP_100 + - address: + socketAddress: + address: 0.0.0.0 + portValue: 101 + filterChains: + - filters: + - name: envoy.tcp_proxy + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy + accessLog: + - name: envoy.file_access_log + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + format: | + [%START_TIME%] %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "%DOWNSTREAM_REMOTE_ADDRESS%" "%UPSTREAM_HOST%" "%UPSTREAM_CLUSTER%" + path: /dev/stdout + cluster: foo_TCP_101 + statPrefix: tcp_proxy + name: foo_TCP_101 +`, + args: args{ + es: &egressv1.ExternalService{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: "foo", + }, + Spec: egressv1.ExternalServiceSpec{ + DnsName: "google.com", + Ports: []egressv1.ExternalServicePort{ + { + Port: 100, + Protocol: &udp, + }, + { + Port: 101, + Protocol: &tcp, + }, + }, + }, + }, + maxConns: &maxConnections, }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + tt.args.es.Spec.EnvoyClusterMaxConnections = tt.args.maxConns if got, _ := envoyConfig(tt.args.es); got != tt.want { t.Errorf("envoyConfig() = %v, want %v", got, tt.want) t.Error(cmp.Diff(got, tt.want))