feat: support envoy & ratelimit pod affinity and tolerations.

api/config/v1alpha1/shared_types.go

@@ -75,6 +75,14 @@ type KubernetesPodSpec struct {
 	//
 	// +optional
 	SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"`
+
+	// If specified, the pod's scheduling constraints.
+	// +optional
+	Affinity *corev1.Affinity `json:"affinity,omitempty"`
+
+	// If specified, the pod's tolerations.
+	// +optional
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
 }
 
 // KubernetesContainerSpec defines the desired state of the Kubernetes container resource.

docs/latest/api/config_types.md

@@ -297,6 +297,8 @@ _Appears in:_
 | --- | --- |
 | `annotations` _object (keys:string, values:string)_ | Annotations are the annotations that should be appended to the pods. By default, no pod annotations are appended. |
 | `securityContext` _[PodSecurityContext](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#podsecuritycontext-v1-core)_ | SecurityContext holds pod-level security attributes and common container settings. Optional: Defaults to empty.  See type description for default values of each field. |
+| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#affinity-v1-core)_ | If specified, the pod's scheduling constraints. |
+| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.26/#toleration-v1-core) array_ | If specified, the pod's tolerations. |
 
 
 ## KubernetesServiceSpec

internal/gatewayapi/testdata/envoyproxy-valid.in.yaml

@@ -18,7 +18,7 @@ envoyproxy:
               value: env_a_value
             - name: env_b
               value: env_b_name
-            image: "envoyproxy/gateway:v0.4.0"
+            image: "envoyproxy/envoy-dev:latest"
             resources:
               requests:
                 cpu: 100m
@@ -30,6 +30,20 @@ envoyproxy:
             annotations:
               key1: val1
               key2: val2
+            affinity:
+              nodeAffinity:
+                requiredDuringSchedulingIgnoredDuringExecution:
+                  nodeSelectorTerms:
+                  - matchExpressions:
+                    - key: cloud.google.com/gke-nodepool
+                      operator: In
+                      values:
+                      - router-node
+            tolerations:
+            - effect: NoSchedule
+              key: node-type
+              operator: Exists
+              value: "router"
             securityContext:
               runAsUser: 1000
               runAsGroup: 3000

internal/gatewayapi/testdata/envoyproxy-valid.out.yaml

@@ -67,7 +67,7 @@ infraIR:
                     value: env_a_value
                   - name: env_b
                     value: env_b_name
-                  image: "envoyproxy/gateway:v0.4.0"
+                  image: "envoyproxy/envoy-dev:latest"
                   resources:
                     requests:
                       cpu: 100m
@@ -79,6 +79,20 @@ infraIR:
                   annotations:
                     key1: val1
                     key2: val2
+                  affinity:
+                    nodeAffinity:
+                      requiredDuringSchedulingIgnoredDuringExecution:
+                        nodeSelectorTerms:
+                        - matchExpressions:
+                          - key: cloud.google.com/gke-nodepool
+                            operator: In
+                            values:
+                            - router-node
+                  tolerations:
+                  - effect: NoSchedule
+                    key: node-type
+                    operator: Exists
+                    value: "router"
                   securityContext:
                     runAsUser: 1000
                     runAsGroup: 3000

internal/infrastructure/kubernetes/proxy/resource_provider.go

@@ -197,6 +197,8 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) {
 					RestartPolicy:                 corev1.RestartPolicyAlways,
 					SchedulerName:                 "default-scheduler",
 					SecurityContext:               deploymentConfig.Pod.SecurityContext,
+					Affinity:                      deploymentConfig.Pod.Affinity,
+					Tolerations:                   deploymentConfig.Pod.Tolerations,
 					Volumes: []corev1.Volume{
 						{
 							Name: "certs",

internal/infrastructure/kubernetes/ratelimit/resource_provider.go

@@ -148,6 +148,8 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) {
 					RestartPolicy:                 corev1.RestartPolicyAlways,
 					SchedulerName:                 "default-scheduler",
 					SecurityContext:               r.rateLimitDeployment.Pod.SecurityContext,
+					Affinity:                      r.rateLimitDeployment.Pod.Affinity,
+					Tolerations:                   r.rateLimitDeployment.Pod.Tolerations,
 					Volumes: []corev1.Volume{
 						{
 							Name: InfraName,

internal/infrastructure/kubernetes/ratelimit/testdata/deployments/affinity.yaml

@@ -0,0 +1,90 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app.kubernetes.io/name: envoy-ratelimit
+    app.kubernetes.io/component: ratelimit
+    app.kubernetes.io/managed-by: envoy-gateway
+  name: envoy-ratelimit
+  namespace: envoy-gateway-system
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: envoy-ratelimit
+      app.kubernetes.io/component: ratelimit
+      app.kubernetes.io/managed-by: envoy-gateway
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: envoy-ratelimit
+        app.kubernetes.io/component: ratelimit
+        app.kubernetes.io/managed-by: envoy-gateway
+      annotations:
+        prometheus.io/scrape: "true"
+    spec:
+      automountServiceAccountToken: false
+      containers:
+      - command:
+        - /bin/ratelimit
+        env:
+        - name: REDIS_SOCKET_TYPE
+          value: tcp
+        - name: REDIS_URL
+          value: redis.redis.svc:6379
+        - name: RUNTIME_ROOT
+          value: /data
+        - name: RUNTIME_SUBDIRECTORY
+          value: ratelimit
+        - name: RUNTIME_IGNOREDOTFILES
+          value: "true"
+        - name: RUNTIME_WATCH_ROOT
+          value: "false"
+        - name: LOG_LEVEL
+          value: info
+        - name: USE_STATSD
+          value: "true"
+        image: custom-image
+        imagePullPolicy: IfNotPresent
+        name: envoy-ratelimit
+        ports:
+        - containerPort: 8081
+          name: http
+          protocol: TCP
+        securityContext:
+          privileged: true
+        resources:
+          limits:
+            cpu: 400m
+            memory: 2Gi
+          requests:
+            cpu: 200m
+            memory: 1Gi
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /data/ratelimit/config
+          name: envoy-ratelimit
+          readOnly: true
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      serviceAccountName: envoy-ratelimit
+      securityContext:
+        runAsUser: 1000
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: cloud.google.com/gke-nodepool
+                operator: In
+                values:
+                - router-node
+      terminationGracePeriodSeconds: 300
+      volumes:
+      - configMap:
+          defaultMode: 420
+          name: envoy-ratelimit
+          optional: false
+        name: envoy-ratelimit

internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml

@@ -25,47 +25,47 @@ spec:
     spec:
       automountServiceAccountToken: false
       containers:
-        - command:
-            - /bin/ratelimit
-          env:
-            - name: REDIS_SOCKET_TYPE
-              value: tcp
-            - name: REDIS_URL
-              value: redis.redis.svc:6379
-            - name: RUNTIME_ROOT
-              value: /data
-            - name: RUNTIME_SUBDIRECTORY
-              value: ratelimit
-            - name: RUNTIME_IGNOREDOTFILES
-              value: "true"
-            - name: RUNTIME_WATCH_ROOT
-              value: "false"
-            - name: LOG_LEVEL
-              value: info
-            - name: USE_STATSD
-              value: "false"
-          image: custom-image
-          imagePullPolicy: IfNotPresent
+      - command:
+        - /bin/ratelimit
+        env:
+        - name: REDIS_SOCKET_TYPE
+          value: tcp
+        - name: REDIS_URL
+          value: redis.redis.svc:6379
+        - name: RUNTIME_ROOT
+          value: /data
+        - name: RUNTIME_SUBDIRECTORY
+          value: ratelimit
+        - name: RUNTIME_IGNOREDOTFILES
+          value: "true"
+        - name: RUNTIME_WATCH_ROOT
+          value: "false"
+        - name: LOG_LEVEL
+          value: info
+        - name: USE_STATSD
+          value: "false"
+        image: custom-image
+        imagePullPolicy: IfNotPresent
+        name: envoy-ratelimit
+        ports:
+        - containerPort: 8081
+          name: http
+          protocol: TCP
+        securityContext:
+          privileged: true
+        resources:
+          limits:
+            cpu: 400m
+            memory: 2Gi
+          requests:
+            cpu: 200m
+            memory: 1Gi
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /data/ratelimit/config
           name: envoy-ratelimit
-          ports:
-            - containerPort: 8081
-              name: http
-              protocol: TCP
-          securityContext:
-            privileged: true
-          resources:
-            limits:
-              cpu: 400m
-              memory: 2Gi
-            requests:
-              cpu: 200m
-              memory: 1Gi
-          terminationMessagePath: /dev/termination-log
-          terminationMessagePolicy: File
-          volumeMounts:
-            - mountPath: /data/ratelimit/config
-              name: envoy-ratelimit
-              readOnly: true
+          readOnly: true
       dnsPolicy: ClusterFirst
       restartPolicy: Always
       schedulerName: default-scheduler
@@ -74,8 +74,8 @@ spec:
         runAsUser: 1000
       terminationGracePeriodSeconds: 300
       volumes:
-        - configMap:
-            defaultMode: 420
-            name: envoy-ratelimit
-            optional: false
+      - configMap:
+          defaultMode: 420
           name: envoy-ratelimit
+          optional: false
+        name: envoy-ratelimit