From 6f7173b60a7c5f09a7feba621bc52f86cb50f1bf Mon Sep 17 00:00:00 2001 From: Dom Goodwin Date: Fri, 19 Jan 2024 12:05:13 +0000 Subject: [PATCH] Allow setting service topology aware routing mode --- README.md | 31 ++++++++++++++--------- api/v1/externalservice_types.go | 4 +++ controllers/externalservice_controller.go | 16 ++++++++++-- 3 files changed, 37 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index db1c21a..d10ec38 100644 --- a/README.md +++ b/README.md @@ -202,15 +202,22 @@ spec: egress.monzo.com/gateway: egress-gateway-name ``` -| Variable name | Default | Description | -|------------------------------------|-------------------------------------------|----------------------------------------------------| -| ENVOY_IMAGE | `envoyproxy/envoy-alpine:v1.16.5` | Name of the Envoy Proxy image to use | -| TAINT_TOLERATION_KEY | Empty, no tolerations applied | Toleration key to apply to gateway pods | -| TAINT_TOLERATION_VALUE | Empty, no tolerations applied | Toleration value to apply to gateway pods | -| NODE_SELECTOR_KEY | Empty, no node selector added | Node selector label key to apply to gateway pods | -| NODE_SELECTOR_VALUE | Empty, no node selector added | Node selector label value to apply to gateway pods | -| POD_TOPOLOGY_ZONE_MAX_SKEW_KEY | `topology.kubernetes.io/zone` | Topology key for the zone constraint | -| POD_TOPOLOGY_ZONE_MAX_SKEW | Empty, won't inject a zone constraint | Value of maxSkew for the zone constraint | -| POD_TOPOLOGY_HOSTNAME_MAX_SKEW_KEY | `kubernetes.io/hostname` | Topology key for the hostname constraint | -| POD_TOPOLOGY_HOSTNAME_MAX_SKEW | Empty, won't inject a hostname constraint | Value of maxSkew for the hostname constraint | - +You can also setup services to have their endpoints aware of network topologies with [topology aware routing](https://kubernetes.io/docs/concepts/services-networking/topology-aware-routing/). + +If you set the environment variable `ENABLE_SERVICE_TOPOLOGY_MODE="true"` this will enable setting the annotations `service.kubernetes.io/topology-mode="auto"`. + +If you turn this on you can also set `spec.serviceTopologyMode` to any value if you want to disable this for particular egress'. For example +if you set `spec.serviceTopologyMode: "None"` this will disable topology aware routing for your ExternalService. + +| Variable name | Default | Description | +|------------------------------------|-------------------------------------------|-----------------------------------------------------------| +| ENVOY_IMAGE | `envoyproxy/envoy-alpine:v1.16.5` | Name of the Envoy Proxy image to use | +| TAINT_TOLERATION_KEY | Empty, no tolerations applied | Toleration key to apply to gateway pods | +| TAINT_TOLERATION_VALUE | Empty, no tolerations applied | Toleration value to apply to gateway pods | +| NODE_SELECTOR_KEY | Empty, no node selector added | Node selector label key to apply to gateway pods | +| NODE_SELECTOR_VALUE | Empty, no node selector added | Node selector label value to apply to gateway pods | +| POD_TOPOLOGY_ZONE_MAX_SKEW_KEY | `topology.kubernetes.io/zone` | Topology key for the zone constraint | +| POD_TOPOLOGY_ZONE_MAX_SKEW | Empty, won't inject a zone constraint | Value of maxSkew for the zone constraint | +| POD_TOPOLOGY_HOSTNAME_MAX_SKEW_KEY | `kubernetes.io/hostname` | Topology key for the hostname constraint | +| POD_TOPOLOGY_HOSTNAME_MAX_SKEW | Empty, won't inject a hostname constraint | Value of maxSkew for the hostname constraint | +| ENABLE_SERVICE_TOPOLOGY_MODE | Empty, won't add the annotation | Set to 'true' to add the topology mode service annotation | diff --git a/api/v1/externalservice_types.go b/api/v1/externalservice_types.go index 1a3fafc..3b106b4 100644 --- a/api/v1/externalservice_types.go +++ b/api/v1/externalservice_types.go @@ -62,6 +62,10 @@ type ExternalServiceSpec struct { // If this circuit breaker overflows the upstream_cx_overflow counter for the cluster will increment. // +optional EnvoyClusterMaxConnections *uint32 `json:"envoyClusterMaxConnections,omitempty"` + + // Provides a way to override the global default + // +optional + ServiceTopologyMode string `json:"serviceTopologyMode,omitempty"` } type ExternalServicePort struct { diff --git a/controllers/externalservice_controller.go b/controllers/externalservice_controller.go index 155889d..eff6b63 100644 --- a/controllers/externalservice_controller.go +++ b/controllers/externalservice_controller.go @@ -18,9 +18,9 @@ package controllers import ( "bytes" "context" + "os" "github.com/go-logr/logr" - egressv1 "github.com/monzo/egress-operator/api/v1" appsv1 "k8s.io/api/apps/v1" autoscalingv1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" @@ -29,6 +29,8 @@ import ( "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + + egressv1 "github.com/monzo/egress-operator/api/v1" ) const namespace = "egress-operator-system" @@ -106,9 +108,19 @@ func labels(es *egressv1.ExternalService) map[string]string { } func annotations(es *egressv1.ExternalService) map[string]string { - return map[string]string{ + annotations := map[string]string{ "egress.monzo.com/dns-name": es.Spec.DnsName, } + // Allow setting the topology aware routing annotation + value, ok := os.LookupEnv("ENABLE_SERVICE_TOPOLOGY_MODE") + if ok && value == "true" { + if es.Spec.ServiceTopologyMode != "" { + annotations["service.kubernetes.io/topology-mode"] = es.Spec.ServiceTopologyMode + } else { + annotations["service.kubernetes.io/topology-mode"] = "Auto" + } + } + return annotations } func labelsToSelect(es *egressv1.ExternalService) map[string]string {