Adding envoy cluster max connections property to external service

Dockerfile

@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.20 as builder
+FROM golang@sha256:d6df8b2e22c9c87fde828b18e0d0d5707bfe03034a49c0bde72ff5a1f5ebb5fe as builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests

Makefile

@@ -70,7 +70,7 @@ ifeq (, $(shell which controller-gen))
 	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
 	cd $$CONTROLLER_GEN_TMP_DIR ;\
 	go mod init tmp ;\
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.4 ;\
+	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.4.0 ;\
 	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
 	}
 CONTROLLER_GEN=$(GOBIN)/controller-gen

api/v1/externalservice_types.go

@@ -57,6 +57,11 @@ type ExternalServiceSpec struct {
 	// When set allows overwriting the A records of the DNS being overridden.
 	// +optional
 	IpOverride []string `json:"ipOverride,omitempty"`
+
+	// The maximum number of connections that Envoy will establish to all hosts in an upstream cluster (defaults to 1024).
+	// If this circuit breaker overflows the upstream_cx_overflow counter for the cluster will increment.
+	// +optional
+	EnvoyClusterMaxConnections *uint32 `json:"envoyClusterMaxConnections,omitempty"`
 }
 
 type ExternalServicePort struct {

config/crd/bases/egress.monzo.com_externalservices.yaml

@@ -39,6 +39,13 @@ spec:
               dnsName:
                 description: DnsName is a DNS name target for the external service
                 type: string
+              envoyClusterMaxConnections:
+                description: The maximum number of connections that Envoy will establish
+                  to all hosts in an upstream cluster (defaults to 1024). If this
+                  circuit breaker overflows the upstream_cx_overflow counter for the
+                  cluster will increment.
+                format: int32
+                type: integer
               hijackDns:
                 description: 'If true, add a `egress.monzo.com/hijack-dns: true` label
                   to produced Service objects CoreDNS can watch this label and decide

controllers/configmap.go

@@ -16,6 +16,7 @@ import (
 	aggregatev3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3"
 	tcpproxyv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3"
 	udpproxyv3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/udp/udp_proxy/v3"
+	"github.com/golang/protobuf/ptypes/wrappers"
 	"google.golang.org/protobuf/types/known/wrapperspb"
 
 	"github.com/golang/protobuf/jsonpb"
@@ -171,6 +172,20 @@ func envoyConfig(es *egressv1.ExternalService) (string, error) {
 			clusterNameForListener = aggregateCluster.Name
 		}
 
+		if es.Spec.EnvoyClusterMaxConnections != nil {
+			cbs := &envoyv3.CircuitBreakers{
+				Thresholds: []*envoyv3.CircuitBreakers_Thresholds{
+					{
+						MaxConnections: &wrappers.UInt32Value{Value: *es.Spec.EnvoyClusterMaxConnections},
+					},
+				},
+				PerHostThresholds: nil,
+			}
+			for _, cluster := range clusters {
+				cluster.CircuitBreakers = cbs
+			}
+		}
+
 		var listener *envoylistener.Listener
 		switch protocol {
 		case envoycorev3.SocketAddress_TCP:

controllers/configmap_test.go

@@ -13,8 +13,11 @@ import (
 func Test_envoyConfig(t *testing.T) {
 	udp := corev1.ProtocolUDP
 	tcp := corev1.ProtocolTCP
+	var maxConnections uint32
+	maxConnections = 4096
 	type args struct {
-		es *egressv1.ExternalService
+		es       *egressv1.ExternalService
+		maxConns *uint32
 	}
 	tests := []struct {
 		name string
@@ -105,7 +108,7 @@ staticResources:
     name: foo_TCP_101
 `,
 			args: args{
-				&egressv1.ExternalService{
+				es: &egressv1.ExternalService{
 					ObjectMeta: metav1.ObjectMeta{
 						Name:      "foo",
 						Namespace: "foo",
@@ -124,11 +127,115 @@ staticResources:
 						},
 					},
 				},
+				maxConns: nil,
+			},
+		},
+		{
+			name: "udp and tcp with max connections",
+			want: `admin:
+  accessLogPath: /dev/stdout
+  address:
+    socketAddress:
+      address: 0.0.0.0
+      portValue: 11000
+node:
+  cluster: foo
+staticResources:
+  clusters:
+  - circuitBreakers:
+      thresholds:
+      - maxConnections: 4096
+    connectTimeout: 1s
+    dnsLookupFamily: V4_ONLY
+    loadAssignment:
+      clusterName: foo_UDP_100
+      endpoints:
+      - lbEndpoints:
+        - endpoint:
+            address:
+              socketAddress:
+                address: google.com
+                portValue: 100
+                protocol: UDP
+    name: foo_UDP_100
+    type: LOGICAL_DNS
+  - circuitBreakers:
+      thresholds:
+      - maxConnections: 4096
+    connectTimeout: 1s
+    dnsLookupFamily: V4_ONLY
+    loadAssignment:
+      clusterName: foo_TCP_101
+      endpoints:
+      - lbEndpoints:
+        - endpoint:
+            address:
+              socketAddress:
+                address: google.com
+                portValue: 101
+    name: foo_TCP_101
+    type: LOGICAL_DNS
+  listeners:
+  - address:
+      socketAddress:
+        address: 0.0.0.0
+        portValue: 100
+        protocol: UDP
+    filterChains:
+    - filters:
+      - name: envoy.filters.udp_listener.udp_proxy
+        typedConfig:
+          '@type': type.googleapis.com/envoy.extensions.filters.udp.udp_proxy.v3.UdpProxyConfig
+          cluster: foo_UDP_100
+          statPrefix: udp_proxy
+    name: foo_UDP_100
+  - address:
+      socketAddress:
+        address: 0.0.0.0
+        portValue: 101
+    filterChains:
+    - filters:
+      - name: envoy.tcp_proxy
+        typedConfig:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
+          accessLog:
+          - name: envoy.file_access_log
+            typedConfig:
+              '@type': type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog
+              format: |
+                [%START_TIME%] %BYTES_RECEIVED% %BYTES_SENT% %DURATION% "%DOWNSTREAM_REMOTE_ADDRESS%" "%UPSTREAM_HOST%" "%UPSTREAM_CLUSTER%"
+              path: /dev/stdout
+          cluster: foo_TCP_101
+          statPrefix: tcp_proxy
+    name: foo_TCP_101
+`,
+			args: args{
+				es: &egressv1.ExternalService{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "foo",
+						Namespace: "foo",
+					},
+					Spec: egressv1.ExternalServiceSpec{
+						DnsName: "google.com",
+						Ports: []egressv1.ExternalServicePort{
+							{
+								Port:     100,
+								Protocol: &udp,
+							},
+							{
+								Port:     101,
+								Protocol: &tcp,
+							},
+						},
+					},
+				},
+				maxConns: &maxConnections,
 			},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			tt.args.es.Spec.EnvoyClusterMaxConnections = tt.args.maxConns
 			if got, _ := envoyConfig(tt.args.es); got != tt.want {
 				t.Errorf("envoyConfig() = %v, want %v", got, tt.want)
 				t.Error(cmp.Diff(got, tt.want))