kubernetes: support custom cluster dns domain

charts/gateway-helm/README.md

@@ -8,10 +8,10 @@ The Helm chart for Envoy Gateway
 
 ## Maintainers
 
-| Name | Email | Url |
-| ---- | ------ | --- |
-| envoy-gateway-steering-committee |  | <https://github.com/envoyproxy/gateway/blob/main/GOVERNANCE.md> |
-| envoy-gateway-maintainers |  | <https://github.com/envoyproxy/gateway/blob/main/CODEOWNERS> |
+| Name                             | Email | Url                                                             |
+| -------------------------------- | ----- | --------------------------------------------------------------- |
+| envoy-gateway-steering-committee |       | <https://github.com/envoyproxy/gateway/blob/main/GOVERNANCE.md> |
+| envoy-gateway-maintainers        |       | <https://github.com/envoyproxy/gateway/blob/main/CODEOWNERS>    |
 
 ## Source Code
 
@@ -56,28 +56,29 @@ To uninstall the chart:
 
 ## Values
 
-| Key | Type | Default | Description |
-|-----|------|---------|-------------|
-| config.envoyGateway.gateway.controllerName | string | `"gateway.envoyproxy.io/gatewayclass-controller"` |  |
-| config.envoyGateway.provider.type | string | `"Kubernetes"` |  |
-| deployment.envoyGateway.image.repository | string | `"docker.io/envoyproxy/gateway-dev"` |  |
-| deployment.envoyGateway.image.tag | string | `"latest"` |  |
-| deployment.envoyGateway.imagePullPolicy | string | `"Always"` |  |
-| deployment.envoyGateway.resources.limits.cpu | string | `"500m"` |  |
-| deployment.envoyGateway.resources.limits.memory | string | `"128Mi"` |  |
-| deployment.envoyGateway.resources.requests.cpu | string | `"10m"` |  |
-| deployment.envoyGateway.resources.requests.memory | string | `"64Mi"` |  |
-| deployment.kubeRbacProxy.image.repository | string | `"gcr.io/kubebuilder/kube-rbac-proxy"` |  |
-| deployment.kubeRbacProxy.image.tag | string | `"v0.11.0"` |  |
-| deployment.kubeRbacProxy.resources.limits.cpu | string | `"500m"` |  |
-| deployment.kubeRbacProxy.resources.limits.memory | string | `"128Mi"` |  |
-| deployment.kubeRbacProxy.resources.requests.cpu | string | `"5m"` |  |
-| deployment.kubeRbacProxy.resources.requests.memory | string | `"64Mi"` |  |
-| deployment.ports[0].name | string | `"grpc"` |  |
-| deployment.ports[0].port | int | `18000` |  |
-| deployment.ports[0].targetPort | int | `18000` |  |
-| deployment.replicas | int | `1` |  |
-| envoyGatewayMetricsService.ports[0].name | string | `"https"` |  |
-| envoyGatewayMetricsService.ports[0].port | int | `8443` |  |
-| envoyGatewayMetricsService.ports[0].protocol | string | `"TCP"` |  |
-| envoyGatewayMetricsService.ports[0].targetPort | string | `"https"` |  |
+| Key                                                | Type   | Default                                           | Description |
+| -------------------------------------------------- | ------ | ------------------------------------------------- | ----------- |
+| config.envoyGateway.gateway.controllerName         | string | `"gateway.envoyproxy.io/gatewayclass-controller"` |             |
+| config.envoyGateway.provider.type                  | string | `"Kubernetes"`                                    |             |
+| deployment.envoyGateway.image.repository           | string | `"docker.io/envoyproxy/gateway-dev"`              |             |
+| deployment.envoyGateway.image.tag                  | string | `"latest"`                                        |             |
+| deployment.envoyGateway.imagePullPolicy            | string | `"Always"`                                        |             |
+| deployment.envoyGateway.resources.limits.cpu       | string | `"500m"`                                          |             |
+| deployment.envoyGateway.resources.limits.memory    | string | `"128Mi"`                                         |             |
+| deployment.envoyGateway.resources.requests.cpu     | string | `"10m"`                                           |             |
+| deployment.envoyGateway.resources.requests.memory  | string | `"64Mi"`                                          |             |
+| deployment.kubeRbacProxy.image.repository          | string | `"gcr.io/kubebuilder/kube-rbac-proxy"`            |             |
+| deployment.kubeRbacProxy.image.tag                 | string | `"v0.11.0"`                                       |             |
+| deployment.kubeRbacProxy.resources.limits.cpu      | string | `"500m"`                                          |             |
+| deployment.kubeRbacProxy.resources.limits.memory   | string | `"128Mi"`                                         |             |
+| deployment.kubeRbacProxy.resources.requests.cpu    | string | `"5m"`                                            |             |
+| deployment.kubeRbacProxy.resources.requests.memory | string | `"64Mi"`                                          |             |
+| deployment.ports[0].name                           | string | `"grpc"`                                          |             |
+| deployment.ports[0].port                           | int    | `18000`                                           |             |
+| deployment.ports[0].targetPort                     | int    | `18000`                                           |             |
+| deployment.replicas                                | int    | `1`                                               |             |
+| envoyGatewayMetricsService.ports[0].name           | string | `"https"`                                         |             |
+| envoyGatewayMetricsService.ports[0].port           | int    | `8443`                                            |             |
+| envoyGatewayMetricsService.ports[0].protocol       | string | `"TCP"`                                           |             |
+| envoyGatewayMetricsService.ports[0].targetPort     | string | `"https"`                                         |             |
+| kubernetesClusterDomain                            | string | `"cluster.local"`                                 |             |

charts/gateway-helm/templates/certgen.yaml

@@ -26,6 +26,8 @@ spec:
             fieldRef:
               apiVersion: v1
               fieldPath: metadata.namespace
+        - name: KUBERNETES_CLUSTER_DOMAIN
+          value: {{ .Values.kubernetesClusterDomain }}
         image: {{ .Values.deployment.envoyGateway.image.repository }}:{{ .Values.deployment.envoyGateway.image.tag | default .Chart.AppVersion }}
         imagePullPolicy:  {{ .Values.deployment.envoyGateway.imagePullPolicy }}
         name: envoy-gateway-certgen

charts/gateway-helm/templates/envoy-gateway-deployment.yaml

@@ -71,7 +71,7 @@ spec:
         - --v=0
         env:
         - name: KUBERNETES_CLUSTER_DOMAIN
-          value: cluster.local
+          value: {{ .Values.kubernetesClusterDomain }}
         image: {{ .Values.deployment.kubeRbacProxy.image.repository }}:{{ .Values.deployment.kubeRbacProxy.image.tag | default .Chart.AppVersion }}
         name: kube-rbac-proxy
         ports:

charts/gateway-helm/values.tmpl.yaml

@@ -23,9 +23,9 @@ deployment:
         cpu: 5m
         memory: 64Mi
   ports:
-  - name: grpc
-    port: 18000
-    targetPort: 18000
+    - name: grpc
+      port: 18000
+      targetPort: 18000
   replicas: 1
 
 config:
@@ -37,9 +37,11 @@ config:
 
 envoyGatewayMetricsService:
   ports:
-  - name: https
-    port: 8443
-    protocol: TCP
-    targetPort: https
+    - name: https
+      port: 8443
+      protocol: TCP
+      targetPort: https
 
 createNamespace: false
+
+kubernetesClusterDomain: cluster.local

internal/cmd/egctl/translate.go

@@ -56,6 +56,7 @@ func NewTranslateCommand() *cobra.Command {
 		inFile, inType, output, resourceType string
 		addMissingResources                  bool
 		outTypes                             []string
+		dnsDomain                            string
 	)
 
 	translateCommand := &cobra.Command{
@@ -93,7 +94,7 @@ func NewTranslateCommand() *cobra.Command {
   egctl experimental translate --from gateway-api --to gateway-api,xds --type all --output yaml --file <input file>
 	`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			return translate(cmd.OutOrStdout(), inFile, inType, outTypes, output, resourceType, addMissingResources)
+			return translate(cmd.OutOrStdout(), inFile, inType, outTypes, output, resourceType, addMissingResources, dnsDomain)
 		},
 	}
 
@@ -106,6 +107,7 @@ func NewTranslateCommand() *cobra.Command {
 	translateCommand.PersistentFlags().StringVarP(&output, "output", "o", yamlOutput, "One of 'yaml' or 'json'")
 	translateCommand.PersistentFlags().StringVarP(&resourceType, "type", "t", string(AllEnvoyConfigType), getValidResourceTypesStr())
 	translateCommand.PersistentFlags().BoolVarP(&addMissingResources, "add-missing-resources", "", false, "Provides dummy resources if missed")
+	translateCommand.PersistentFlags().StringVarP(&dnsDomain, "dns-domain", "", "cluster.local", "DNS domain used by k8s services, default is cluster.local")
 	return translateCommand
 }
 
@@ -207,7 +209,7 @@ func validate(inFile, inType string, outTypes []string, resourceType string) err
 	return nil
 }
 
-func translate(w io.Writer, inFile, inType string, outTypes []string, output, resourceType string, addMissingResources bool) error {
+func translate(w io.Writer, inFile, inType string, outTypes []string, output, resourceType string, addMissingResources bool, dnsDomain string) error {
 	if err := validate(inFile, inType, outTypes, resourceType); err != nil {
 		return err
 	}
@@ -231,7 +233,7 @@ func translate(w io.Writer, inFile, inType string, outTypes []string, output, re
 				result.Resources = translateGatewayAPIToGatewayAPI(resources)
 			}
 			if outType == xdsType {
-				res, err := translateGatewayAPIToXds(resourceType, resources)
+				res, err := translateGatewayAPIToXds(dnsDomain, resourceType, resources)
 				if err != nil {
 					return err
 				}
@@ -274,7 +276,7 @@ func translateGatewayAPIToGatewayAPI(resources *gatewayapi.Resources) gatewayapi
 	return gRes.Resources
 }
 
-func translateGatewayAPIToXds(resourceType string, resources *gatewayapi.Resources) (map[string]any, error) {
+func translateGatewayAPIToXds(dnsDomain string, resourceType string, resources *gatewayapi.Resources) (map[string]any, error) {
 	// Translate from Gateway API to Xds IR
 	gTranslator := &gatewayapi.Translator{
 		GatewayControllerName:  egv1alpha1.GatewayControllerName,
@@ -297,7 +299,7 @@ func translateGatewayAPIToXds(resourceType string, resources *gatewayapi.Resourc
 		xTranslator := &translator.Translator{
 			// Set some default settings for translation
 			GlobalRateLimit: &translator.GlobalRateLimitSettings{
-				ServiceURL: ratelimit.GetServiceURL("envoy-gateway"),
+				ServiceURL: ratelimit.GetServiceURL("envoy-gateway", dnsDomain),
 			},
 		}
 		xRes, err := xTranslator.Translate(val)

internal/crypto/certgen.go

@@ -31,9 +31,6 @@ const (
 	// DefaultCertificateLifetime holds the default certificate lifetime (in days).
 	DefaultCertificateLifetime = 365
 
-	// DefaultDNSSuffix is the default DNS suffix name.
-	DefaultDNSSuffix = "cluster.local"
-
 	// keySize sets the RSA key size to 2048 bits. This is minimum recommended size
 	// for RSA keys.
 	keySize = 2048
@@ -108,7 +105,7 @@ func GenerateCerts(cfg *config.Server) (*Certificates, error) {
 		egProvider := cfg.EnvoyGateway.GetEnvoyGatewayProvider().Type
 		switch egProvider {
 		case v1alpha1.ProviderTypeKubernetes:
-			egDNSNames = kubeServiceNames(DefaultEnvoyGatewayDNSPrefix, cfg.Namespace, DefaultDNSSuffix)
+			egDNSNames = kubeServiceNames(DefaultEnvoyGatewayDNSPrefix, cfg.Namespace, cfg.DNSDomain)
 			envoyDNSNames = append(envoyDNSNames, fmt.Sprintf("*.%s", cfg.Namespace))
 		default:
 			// Kubernetes is the only supported Envoy Gateway provider.

internal/envoygateway/config/config.go

@@ -21,6 +21,8 @@ import (
 const (
 	// DefaultNamespace is the default namespace of Envoy Gateway.
 	DefaultNamespace = "envoy-gateway-system"
+	// DefaultDNSDomain is the default DNS domain used by k8s services.
+	DefaultDNSDomain = "cluster.local"
 	// EnvoyGatewayServiceName is the name of the Envoy Gateway service.
 	EnvoyGatewayServiceName = "envoy-gateway"
 	// EnvoyPrefix is the prefix applied to the Envoy ConfigMap, Service, Deployment, and ServiceAccount.
@@ -34,6 +36,8 @@ type Server struct {
 	EnvoyGateway *v1alpha1.EnvoyGateway
 	// Namespace is the namespace that Envoy Gateway runs in.
 	Namespace string
+	// DNSDomain is the dns domain used by k8s services. Defaults to "cluster.local".
+	DNSDomain string
 	// Logger is the logr implementation used by Envoy Gateway.
 	Logger logr.Logger
 }
@@ -47,6 +51,7 @@ func New() (*Server, error) {
 	return &Server{
 		EnvoyGateway: v1alpha1.DefaultEnvoyGateway(),
 		Namespace:    env.Lookup("ENVOY_GATEWAY_NAMESPACE", DefaultNamespace),
+		DNSDomain:    env.Lookup("KUBERNETES_CLUSTER_DOMAIN", DefaultDNSDomain),
 		Logger:       logger,
 	}, nil
 }

internal/infrastructure/kubernetes/ratelimit/resource.go

@@ -37,9 +37,8 @@ const (
 )
 
 // GetServiceURL returns the URL for the rate limit service.
-// TODO: support custom trust domain
-func GetServiceURL(namespace string) string {
-	return fmt.Sprintf("grpc://%s.%s.svc.cluster.local:%d", InfraName, namespace, InfraGRPCPort)
+func GetServiceURL(namespace string, dnsDomain string) string {
+	return fmt.Sprintf("grpc://%s.%s.svc.%s:%d", InfraName, namespace, dnsDomain, InfraGRPCPort)
 }
 
 // rateLimitLabels returns the labels used for all envoy rate limit resources.

internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go

@@ -375,6 +375,6 @@ func loadDeployment(caseName string) (*appsv1.Deployment, error) {
 }
 
 func TestGetServiceURL(t *testing.T) {
-	got := GetServiceURL("envoy-gateway-system")
-	assert.Equal(t, "grpc://envoy-ratelimit.envoy-gateway-system.svc.cluster.local:8081", got)
+	got := GetServiceURL("envoy-gateway-system", "example-cluster.local")
+	assert.Equal(t, "grpc://envoy-ratelimit.envoy-gateway-system.svc.example-cluster.local:8081", got)
 }

internal/xds/translator/runner/runner.go

@@ -65,7 +65,7 @@ func (r *Runner) subscribeAndTranslate(ctx context.Context) {
 				// Set the rate limit service URL if global rate limiting is enabled.
 				if r.EnvoyGateway.RateLimit != nil {
 					t.GlobalRateLimit = &translator.GlobalRateLimitSettings{
-						ServiceURL: ratelimit.GetServiceURL(r.Namespace),
+						ServiceURL: ratelimit.GetServiceURL(r.Namespace, r.DNSDomain),
 					}
 				}
 

internal/xds/translator/testdata/in/xds-ir/ratelimit-custom-domain.yaml

@@ -0,0 +1,49 @@
+http:
+- name: "first-listener"
+  address: "0.0.0.0"
+  port: 10080
+  hostnames:
+  - "*"
+  routes:
+  - name: "first-route"
+    rateLimit:
+      global:
+        rules:
+        - headerMatches:
+          - name: "x-user-id"
+            exact: "one"
+          limit:
+            requests: 5
+            unit: second
+    pathMatch:
+      exact: "foo/bar"
+    destinations:
+    - host: "1.2.3.4"
+      port: 50000    
+  - name: "second-route"
+    rateLimit:
+      global:
+        rules:
+        - headerMatches:
+          - name: "x-user-id"
+            distinct: true
+          limit:
+            requests: 5
+            unit: second
+    pathMatch:
+      exact: "example"
+    destinations:
+    - host: "1.2.3.4"
+      port: 50000     
+  - name: "third-route"
+    rateLimit:
+      global:
+        rules:
+        - limit:
+            requests: 5
+            unit: second
+    pathMatch:
+      exact: "test"
+    destinations:
+    - host: "1.2.3.4"
+      port: 50000

internal/xds/translator/testdata/out/xds-ir/ratelimit-custom-domain.clusters.yaml

@@ -0,0 +1,62 @@
+- commonLbConfig:
+    localityWeightedLbConfig: {}
+  connectTimeout: 10s
+  dnsLookupFamily: V4_ONLY
+  edsClusterConfig:
+    edsConfig:
+      ads: {}
+      resourceApiVersion: V3
+  name: first-route
+  outlierDetection: {}
+  perConnectionBufferLimitBytes: 32768
+  type: EDS
+- commonLbConfig:
+    localityWeightedLbConfig: {}
+  connectTimeout: 10s
+  dnsLookupFamily: V4_ONLY
+  edsClusterConfig:
+    edsConfig:
+      ads: {}
+      resourceApiVersion: V3
+  name: second-route
+  outlierDetection: {}
+  perConnectionBufferLimitBytes: 32768
+  type: EDS
+- commonLbConfig:
+    localityWeightedLbConfig: {}
+  connectTimeout: 10s
+  dnsLookupFamily: V4_ONLY
+  edsClusterConfig:
+    edsConfig:
+      ads: {}
+      resourceApiVersion: V3
+  name: third-route
+  outlierDetection: {}
+  perConnectionBufferLimitBytes: 32768
+  type: EDS
+- commonLbConfig:
+    localityWeightedLbConfig: {}
+  connectTimeout: 10s
+  dnsLookupFamily: V4_ONLY
+  dnsRefreshRate: 30s
+  loadAssignment:
+    clusterName: ratelimit_cluster
+    endpoints:
+    - lbEndpoints:
+      - endpoint:
+          address:
+            socketAddress:
+              address: envoy-ratelimit.envoy-gateway-system.svc.example-cluster.local
+              portValue: 8081
+      loadBalancingWeight: 1
+      locality: {}
+  name: ratelimit_cluster
+  outlierDetection: {}
+  perConnectionBufferLimitBytes: 32768
+  respectDnsTtl: true
+  type: STRICT_DNS
+  typedExtensionProtocolOptions:
+    envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+      '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+      explicitHttpConfig:
+        http2ProtocolOptions: {}

internal/xds/translator/testdata/out/xds-ir/ratelimit-custom-domain.endpoints.yaml

@@ -0,0 +1,30 @@
+- clusterName: first-route
+  endpoints:
+  - lbEndpoints:
+    - endpoint:
+        address:
+          socketAddress:
+            address: 1.2.3.4
+            portValue: 50000
+    loadBalancingWeight: 1
+    locality: {}
+- clusterName: second-route
+  endpoints:
+  - lbEndpoints:
+    - endpoint:
+        address:
+          socketAddress:
+            address: 1.2.3.4
+            portValue: 50000
+    loadBalancingWeight: 1
+    locality: {}
+- clusterName: third-route
+  endpoints:
+  - lbEndpoints:
+    - endpoint:
+        address:
+          socketAddress:
+            address: 1.2.3.4
+            portValue: 50000
+    loadBalancingWeight: 1
+    locality: {}