From 1e57665b76b3eeec05d13e2b95b22fa4a1458d61 Mon Sep 17 00:00:00 2001 From: timricese Date: Fri, 12 Jan 2024 01:15:58 +0100 Subject: [PATCH] Feat: Support configuring externalTrafficPolicy on the envoy service (#2432) * Support configuring externalTrafficPolicy on the envoy service Signed-off-by: Tim Rice * Update some autogenerated things Signed-off-by: Tim Rice --------- Signed-off-by: Tim Rice Co-authored-by: Tim Rice --- api/v1alpha1/kubernetes_helpers.go | 11 ++++++++- api/v1alpha1/shared_types.go | 23 +++++++++++++++++++ api/v1alpha1/zz_generated.deepcopy.go | 5 ++++ .../gateway.envoyproxy.io_envoyproxies.yaml | 12 ++++++++++ .../kubernetes/resource/resource.go | 7 ++++-- .../kubernetes/resource/resource_test.go | 12 ++++++++++ site/content/en/latest/api/extension_types.md | 12 ++++++++++ 7 files changed, 79 insertions(+), 3 deletions(-) diff --git a/api/v1alpha1/kubernetes_helpers.go b/api/v1alpha1/kubernetes_helpers.go index 30068173642..3dc553dea33 100644 --- a/api/v1alpha1/kubernetes_helpers.go +++ b/api/v1alpha1/kubernetes_helpers.go @@ -66,7 +66,8 @@ func DefaultResourceRequirements() *corev1.ResourceRequirements { // DefaultKubernetesService returns a new KubernetesServiceSpec with default settings. func DefaultKubernetesService() *KubernetesServiceSpec { return &KubernetesServiceSpec{ - Type: DefaultKubernetesServiceType(), + Type: DefaultKubernetesServiceType(), + ExternalTrafficPolicy: DefaultKubernetesServiceExternalTrafficPolicy(), } } @@ -80,6 +81,14 @@ func GetKubernetesServiceType(serviceType ServiceType) *ServiceType { return &serviceType } +func DefaultKubernetesServiceExternalTrafficPolicy() *ServiceExternalTrafficPolicy { + return GetKubernetesServiceExternalTrafficPolicy(ServiceExternalTrafficPolicyLocal) +} + +func GetKubernetesServiceExternalTrafficPolicy(serviceExternalTrafficPolicy ServiceExternalTrafficPolicy) *ServiceExternalTrafficPolicy { + return &serviceExternalTrafficPolicy +} + // defaultKubernetesDeploymentSpec fill a default KubernetesDeploymentSpec if unspecified. func (deployment *KubernetesDeploymentSpec) defaultKubernetesDeploymentSpec(image string) { if deployment.Replicas == nil { diff --git a/api/v1alpha1/shared_types.go b/api/v1alpha1/shared_types.go index 32fec53e4ea..a75f7af43cd 100644 --- a/api/v1alpha1/shared_types.go +++ b/api/v1alpha1/shared_types.go @@ -192,6 +192,23 @@ const ( ServiceTypeNodePort ServiceType = "NodePort" ) +// ServiceExternalTrafficPolicy describes how nodes distribute service traffic they +// receive on one of the Service's "externally-facing" addresses (NodePorts, ExternalIPs, +// and LoadBalancer IPs. +// +enum +// +kubebuilder:validation:Enum=Local;Cluster +type ServiceExternalTrafficPolicy string + +const ( + // ServiceExternalTrafficPolicyCluster routes traffic to all endpoints. + ServiceExternalTrafficPolicyCluster ServiceExternalTrafficPolicy = "Cluster" + + // ServiceExternalTrafficPolicyLocal preserves the source IP of the traffic by + // routing only to endpoints on the same node as the traffic was received on + // (dropping the traffic if there are no local endpoints). + ServiceExternalTrafficPolicyLocal ServiceExternalTrafficPolicy = "Local" +) + // KubernetesServiceSpec defines the desired state of the Kubernetes service resource. type KubernetesServiceSpec struct { // Annotations that should be appended to the service. @@ -229,6 +246,12 @@ type KubernetesServiceSpec struct { // +optional LoadBalancerIP *string `json:"loadBalancerIP,omitempty"` + // ExternalTrafficPolicy determines the externalTrafficPolicy for the Envoy Service. Valid options + // are Local and Cluster. Default is "Local". "Local" means traffic will only go to pods on the node + // receiving the traffic. "Cluster" means connections are loadbalanced to all pods in the cluster. + // +kubebuilder:default:="Local" + // +optional + ExternalTrafficPolicy *ServiceExternalTrafficPolicy `json:"externalTrafficPolicy,omitempty"` // TODO: Expose config as use cases are better understood, e.g. labels. } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 42aa61b0dec..ec169ae60f6 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1836,6 +1836,11 @@ func (in *KubernetesServiceSpec) DeepCopyInto(out *KubernetesServiceSpec) { *out = new(string) **out = **in } + if in.ExternalTrafficPolicy != nil { + in, out := &in.ExternalTrafficPolicy, &out.ExternalTrafficPolicy + *out = new(ServiceExternalTrafficPolicy) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubernetesServiceSpec. diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 0b1cff64860..e22f535aecf 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -6540,6 +6540,18 @@ spec: description: Annotations that should be appended to the service. By default, no annotations are appended. type: object + externalTrafficPolicy: + default: Local + description: ExternalTrafficPolicy determines the externalTrafficPolicy + for the Envoy Service. Valid options are Local and Cluster. + Default is "Local". "Local" means traffic will only + go to pods on the node receiving the traffic. "Cluster" + means connections are loadbalanced to all pods in the + cluster. + enum: + - Local + - Cluster + type: string loadBalancerClass: description: LoadBalancerClass, when specified, allows for choosing the LoadBalancer provider implementation diff --git a/internal/infrastructure/kubernetes/resource/resource.go b/internal/infrastructure/kubernetes/resource/resource.go index b001334898e..0a98e155e06 100644 --- a/internal/infrastructure/kubernetes/resource/resource.go +++ b/internal/infrastructure/kubernetes/resource/resource.go @@ -27,6 +27,9 @@ func ExpectedServiceSpec(service *egv1a1.KubernetesServiceSpec) corev1.ServiceSp serviceSpec := corev1.ServiceSpec{} serviceSpec.Type = corev1.ServiceType(*service.Type) serviceSpec.SessionAffinity = corev1.ServiceAffinityNone + if service.ExternalTrafficPolicy == nil { + service.ExternalTrafficPolicy = egv1a1.DefaultKubernetesServiceExternalTrafficPolicy() + } if *service.Type == egv1a1.ServiceTypeLoadBalancer { if service.LoadBalancerClass != nil { serviceSpec.LoadBalancerClass = service.LoadBalancerClass @@ -37,9 +40,9 @@ func ExpectedServiceSpec(service *egv1a1.KubernetesServiceSpec) corev1.ServiceSp if service.LoadBalancerIP != nil { serviceSpec.LoadBalancerIP = *service.LoadBalancerIP } - // Preserve the client source IP and avoid a second hop for LoadBalancer. - serviceSpec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeLocal + serviceSpec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicy(*service.ExternalTrafficPolicy) } + return serviceSpec } diff --git a/internal/infrastructure/kubernetes/resource/resource_test.go b/internal/infrastructure/kubernetes/resource/resource_test.go index 1cb7c214548..61bd0fb2444 100644 --- a/internal/infrastructure/kubernetes/resource/resource_test.go +++ b/internal/infrastructure/kubernetes/resource/resource_test.go @@ -39,6 +39,18 @@ func TestExpectedServiceSpec(t *testing.T) { ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal, }, }, + { + name: "LoadBalancerWithExternalTrafficPolicyCluster", + args: args{service: &egv1a1.KubernetesServiceSpec{ + Type: egv1a1.GetKubernetesServiceType(egv1a1.ServiceTypeLoadBalancer), + ExternalTrafficPolicy: egv1a1.GetKubernetesServiceExternalTrafficPolicy(egv1a1.ServiceExternalTrafficPolicyCluster), + }}, + want: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + SessionAffinity: corev1.ServiceAffinityNone, + ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeCluster, + }, + }, { name: "LoadBalancerWithClass", args: args{service: &egv1a1.KubernetesServiceSpec{ diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index fbe5a6806cf..e6a36b130dd 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -1212,6 +1212,7 @@ _Appears in:_ | `loadBalancerClass` _string_ | LoadBalancerClass, when specified, allows for choosing the LoadBalancer provider implementation if more than one are available or is otherwise expected to be specified | | `allocateLoadBalancerNodePorts` _boolean_ | AllocateLoadBalancerNodePorts defines if NodePorts will be automatically allocated for services with type LoadBalancer. Default is "true". It may be set to "false" if the cluster load-balancer does not rely on NodePorts. If the caller requests specific NodePorts (by specifying a value), those requests will be respected, regardless of this field. This field may only be set for services with type LoadBalancer and will be cleared if the type is changed to any other type. | | `loadBalancerIP` _string_ | LoadBalancerIP defines the IP Address of the underlying load balancer service. This field may be ignored if the load balancer provider does not support this feature. This field has been deprecated in Kubernetes, but it is still used for setting the IP Address in some cloud providers such as GCP. | +| `externalTrafficPolicy` _[ServiceExternalTrafficPolicy](#serviceexternaltrafficpolicy)_ | ExternalTrafficPolicy determines the externalTrafficPolicy for the Envoy Service. Valid options are Local and Cluster. Default is "Local". "Local" means traffic will only go to pods on the node receiving the traffic. "Cluster" means connections are loadbalanced to all pods in the cluster. | #### KubernetesWatchMode @@ -1905,6 +1906,17 @@ _Appears in:_ +#### ServiceExternalTrafficPolicy + +_Underlying type:_ `string` + +ServiceExternalTrafficPolicy describes how nodes distribute service traffic they receive on one of the Service's "externally-facing" addresses (NodePorts, ExternalIPs, and LoadBalancer IPs. + +_Appears in:_ +- [KubernetesServiceSpec](#kubernetesservicespec) + + + #### ServiceType _Underlying type:_ `string`