From 84f6958c1a409513a001c4c673a144073da5574a Mon Sep 17 00:00:00 2001 From: Oscar Boher Date: Thu, 12 Sep 2024 22:24:28 +0200 Subject: [PATCH 1/9] fix labels and annotation merges for rate limit deployment Signed-off-by: Oscar Boher --- .../kubernetes/ratelimit/resource_provider.go | 11 +- .../ratelimit/resource_provider_test.go | 38 ++++ .../deployments/merge-annotations.yaml | 190 ++++++++++++++++++ .../testdata/deployments/merge-labels.yaml | 188 +++++++++++++++++ 4 files changed, 424 insertions(+), 3 deletions(-) create mode 100644 internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-annotations.yaml create mode 100644 internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-labels.yaml diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index e7519bb2569..79858722344 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -7,6 +7,7 @@ package ratelimit import ( _ "embed" + "golang.org/x/exp/maps" "strconv" appsv1 "k8s.io/api/apps/v1" @@ -186,8 +187,12 @@ func (r *ResourceRender) ServiceAccount() (*corev1.ServiceAccount, error) { // Deployment returns the expected rate limit Deployment based on the provided infra. func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { containers := expectedRateLimitContainers(r.rateLimit, r.rateLimitDeployment, r.Namespace) - labels := rateLimitLabels() - selector := resource.GetSelector(labels) + selector := resource.GetSelector(rateLimitLabels()) + + var labels map[string]string + if r.rateLimitDeployment.Pod.Labels != nil { + maps.Copy(labels, rateLimitLabels()) + } var annotations map[string]string if enablePrometheus(r.rateLimit) { @@ -198,7 +203,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { } } if r.rateLimitDeployment.Pod.Annotations != nil { - annotations = r.rateLimitDeployment.Pod.Annotations + maps.Copy(annotations, r.rateLimitDeployment.Pod.Annotations) } deployment := &appsv1.Deployment{ diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go index 47c4901e198..84e2aaf478e 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go @@ -678,6 +678,44 @@ func TestDeployment(t *testing.T) { }, }, }, + { + caseName: "merge-labels", + rateLimit: &egv1a1.RateLimit{ + Backend: egv1a1.RateLimitDatabaseBackend{ + Type: egv1a1.RedisBackendType, + Redis: &egv1a1.RateLimitRedisSettings{ + URL: "redis.redis.svc:6379", + }, + }, + }, + deploy: &egv1a1.KubernetesDeploymentSpec{ + Pod: &egv1a1.KubernetesPodSpec{ + Labels: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + }, + }, + { + caseName: "merge-annotations", + rateLimit: &egv1a1.RateLimit{ + Backend: egv1a1.RateLimitDatabaseBackend{ + Type: egv1a1.RedisBackendType, + Redis: &egv1a1.RateLimitRedisSettings{ + URL: "redis.redis.svc:6379", + }, + }, + }, + deploy: &egv1a1.KubernetesDeploymentSpec{ + Pod: &egv1a1.KubernetesPodSpec{ + Annotations: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + }, + }, } for _, tc := range cases { t.Run(tc.caseName, func(t *testing.T) { diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-annotations.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-annotations.yaml new file mode 100644 index 00000000000..1247654dfdb --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-annotations.yaml @@ -0,0 +1,190 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + key1: value1 + key2: value2 + name: envoy-ratelimit + namespace: envoy-gateway-system + ownerReferences: + - apiVersion: apps/v1 + kind: Deployment + name: envoy-gateway + uid: test-owner-reference-uid-for-deployment +spec: + progressDeadlineSeconds: 600 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + strategy: + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + key1: value1 + key2: value2 + spec: + automountServiceAccountToken: false + containers: + - command: + - /bin/ratelimit + env: + - name: RUNTIME_ROOT + value: /data + - name: RUNTIME_SUBDIRECTORY + value: ratelimit + - name: RUNTIME_IGNOREDOTFILES + value: "true" + - name: RUNTIME_WATCH_ROOT + value: "false" + - name: LOG_LEVEL + value: info + - name: USE_STATSD + value: "true" + - name: STATSD_PORT + value: "9125" + - name: CONFIG_TYPE + value: GRPC_XDS_SOTW + - name: CONFIG_GRPC_XDS_SERVER_URL + value: envoy-gateway:18001 + - name: CONFIG_GRPC_XDS_NODE_ID + value: envoy-ratelimit + - name: GRPC_SERVER_USE_TLS + value: "true" + - name: GRPC_SERVER_TLS_CERT + value: /certs/tls.crt + - name: GRPC_SERVER_TLS_KEY + value: /certs/tls.key + - name: GRPC_SERVER_TLS_CA_CERT + value: /certs/ca.crt + - name: CONFIG_GRPC_XDS_SERVER_USE_TLS + value: "true" + - name: CONFIG_GRPC_XDS_CLIENT_TLS_CERT + value: /certs/tls.crt + - name: CONFIG_GRPC_XDS_CLIENT_TLS_KEY + value: /certs/tls.key + - name: CONFIG_GRPC_XDS_SERVER_TLS_CACERT + value: /certs/ca.crt + - name: FORCE_START_WITHOUT_INITIAL_CONFIG + value: "true" + - name: REDIS_SOCKET_TYPE + value: tcp + - name: REDIS_URL + value: redis.redis.svc:6379 + image: envoyproxy/ratelimit:master + imagePullPolicy: IfNotPresent + name: envoy-ratelimit + ports: + - containerPort: 8081 + name: grpc + protocol: TCP + readinessProbe: + failureThreshold: 1 + httpGet: + path: /healthcheck + port: 8080 + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + startupProbe: + failureThreshold: 30 + httpGet: + path: /healthcheck + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - command: + - /bin/statsd_exporter + - --web.listen-address=:19001 + - --statsd.mapping-config=/etc/statsd-exporter/conf.yaml + image: prom/statsd-exporter:v0.18.0 + imagePullPolicy: IfNotPresent + name: prom-statsd-exporter + ports: + - containerPort: 9125 + name: statsd + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/statsd-exporter + name: statsd-exporter-config + readOnly: true + dnsPolicy: ClusterFirst + nodeSelector: + key1: value1 + key2: value2 + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-ratelimit + terminationGracePeriodSeconds: 300 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy-rate-limit + - configMap: + defaultMode: 420 + name: statsd-exporter-config + optional: true + name: statsd-exporter-config +status: {} diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-labels.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-labels.yaml new file mode 100644 index 00000000000..300eab56cce --- /dev/null +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-labels.yaml @@ -0,0 +1,188 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + name: envoy-ratelimit + namespace: envoy-gateway-system + ownerReferences: + - apiVersion: apps/v1 + kind: Deployment + name: envoy-gateway + uid: test-owner-reference-uid-for-deployment +spec: + progressDeadlineSeconds: 600 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + strategy: + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "19001" + prometheus.io/scrape: "true" + creationTimestamp: null + labels: + app.kubernetes.io/component: ratelimit + app.kubernetes.io/managed-by: envoy-gateway + app.kubernetes.io/name: envoy-ratelimit + key1: value1 + key2: value2 + spec: + automountServiceAccountToken: false + containers: + - command: + - /bin/ratelimit + env: + - name: RUNTIME_ROOT + value: /data + - name: RUNTIME_SUBDIRECTORY + value: ratelimit + - name: RUNTIME_IGNOREDOTFILES + value: "true" + - name: RUNTIME_WATCH_ROOT + value: "false" + - name: LOG_LEVEL + value: info + - name: USE_STATSD + value: "true" + - name: STATSD_PORT + value: "9125" + - name: CONFIG_TYPE + value: GRPC_XDS_SOTW + - name: CONFIG_GRPC_XDS_SERVER_URL + value: envoy-gateway:18001 + - name: CONFIG_GRPC_XDS_NODE_ID + value: envoy-ratelimit + - name: GRPC_SERVER_USE_TLS + value: "true" + - name: GRPC_SERVER_TLS_CERT + value: /certs/tls.crt + - name: GRPC_SERVER_TLS_KEY + value: /certs/tls.key + - name: GRPC_SERVER_TLS_CA_CERT + value: /certs/ca.crt + - name: CONFIG_GRPC_XDS_SERVER_USE_TLS + value: "true" + - name: CONFIG_GRPC_XDS_CLIENT_TLS_CERT + value: /certs/tls.crt + - name: CONFIG_GRPC_XDS_CLIENT_TLS_KEY + value: /certs/tls.key + - name: CONFIG_GRPC_XDS_SERVER_TLS_CACERT + value: /certs/ca.crt + - name: FORCE_START_WITHOUT_INITIAL_CONFIG + value: "true" + - name: REDIS_SOCKET_TYPE + value: tcp + - name: REDIS_URL + value: redis.redis.svc:6379 + image: envoyproxy/ratelimit:master + imagePullPolicy: IfNotPresent + name: envoy-ratelimit + ports: + - containerPort: 8081 + name: grpc + protocol: TCP + readinessProbe: + failureThreshold: 1 + httpGet: + path: /healthcheck + port: 8080 + scheme: HTTP + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + startupProbe: + failureThreshold: 30 + httpGet: + path: /healthcheck + port: 8080 + scheme: HTTP + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /certs + name: certs + readOnly: true + - command: + - /bin/statsd_exporter + - --web.listen-address=:19001 + - --statsd.mapping-config=/etc/statsd-exporter/conf.yaml + image: prom/statsd-exporter:v0.18.0 + imagePullPolicy: IfNotPresent + name: prom-statsd-exporter + ports: + - containerPort: 9125 + name: statsd + protocol: TCP + - containerPort: 19001 + name: metrics + protocol: TCP + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/statsd-exporter + name: statsd-exporter-config + readOnly: true + dnsPolicy: ClusterFirst + nodeSelector: + key1: value1 + key2: value2 + restartPolicy: Always + schedulerName: default-scheduler + serviceAccountName: envoy-ratelimit + terminationGracePeriodSeconds: 300 + volumes: + - name: certs + secret: + defaultMode: 420 + secretName: envoy-rate-limit + - configMap: + defaultMode: 420 + name: statsd-exporter-config + optional: true + name: statsd-exporter-config +status: {} From 6e5f6a0f69cf8a377dfa2a042de9c47ae96e6485 Mon Sep 17 00:00:00 2001 From: Oscar Boher Date: Thu, 12 Sep 2024 23:11:48 +0200 Subject: [PATCH 2/9] fix tests and label merge Signed-off-by: Oscar Boher --- .../kubernetes/ratelimit/resource_provider.go | 7 ++++--- .../ratelimit/resource_provider_test.go | 15 +++++++++++---- .../testdata/deployments/merge-annotations.yaml | 9 ++------- .../testdata/deployments/merge-labels.yaml | 3 --- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index 79858722344..bb2c8e449ce 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -189,8 +189,9 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { containers := expectedRateLimitContainers(r.rateLimit, r.rateLimitDeployment, r.Namespace) selector := resource.GetSelector(rateLimitLabels()) - var labels map[string]string + labels := rateLimitLabels() if r.rateLimitDeployment.Pod.Labels != nil { + maps.Copy(labels, r.rateLimitDeployment.Pod.Labels) maps.Copy(labels, rateLimitLabels()) } @@ -213,7 +214,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { }, ObjectMeta: metav1.ObjectMeta{ Namespace: r.Namespace, - Labels: labels, + Labels: rateLimitLabels(), }, Spec: appsv1.DeploymentSpec{ Replicas: r.rateLimitDeployment.Replicas, @@ -221,7 +222,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { Selector: selector, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: selector.MatchLabels, + Labels: labels, Annotations: annotations, }, Spec: corev1.PodSpec{ diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go index 84e2aaf478e..c7aa23f7943 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go @@ -9,6 +9,7 @@ import ( "flag" "fmt" "os" + "strconv" "testing" "github.com/stretchr/testify/assert" @@ -691,8 +692,11 @@ func TestDeployment(t *testing.T) { deploy: &egv1a1.KubernetesDeploymentSpec{ Pod: &egv1a1.KubernetesPodSpec{ Labels: map[string]string{ - "key1": "value1", - "key2": "value2", + "app.kubernetes.io/name": InfraName, + "app.kubernetes.io/component": "ratelimit", + "app.kubernetes.io/managed-by": "envoy-gateway", + "key1": "value1", + "key2": "value2", }, }, }, @@ -710,8 +714,11 @@ func TestDeployment(t *testing.T) { deploy: &egv1a1.KubernetesDeploymentSpec{ Pod: &egv1a1.KubernetesPodSpec{ Annotations: map[string]string{ - "key1": "value1", - "key2": "value2", + "prometheus.io/path": "/metrics", + "prometheus.io/port": strconv.Itoa(PrometheusPort), + "prometheus.io/scrape": "true", + "key1": "value1", + "key2": "value2", }, }, }, diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-annotations.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-annotations.yaml index 1247654dfdb..1061fe5337b 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-annotations.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-annotations.yaml @@ -6,8 +6,6 @@ metadata: app.kubernetes.io/component: ratelimit app.kubernetes.io/managed-by: envoy-gateway app.kubernetes.io/name: envoy-ratelimit - key1: value1 - key2: value2 name: envoy-ratelimit namespace: envoy-gateway-system ownerReferences: @@ -31,13 +29,13 @@ spec: prometheus.io/path: /metrics prometheus.io/port: "19001" prometheus.io/scrape: "true" + key1: value1 + key2: value2 creationTimestamp: null labels: app.kubernetes.io/component: ratelimit app.kubernetes.io/managed-by: envoy-gateway app.kubernetes.io/name: envoy-ratelimit - key1: value1 - key2: value2 spec: automountServiceAccountToken: false containers: @@ -170,9 +168,6 @@ spec: name: statsd-exporter-config readOnly: true dnsPolicy: ClusterFirst - nodeSelector: - key1: value1 - key2: value2 restartPolicy: Always schedulerName: default-scheduler serviceAccountName: envoy-ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-labels.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-labels.yaml index 300eab56cce..7507f1e590b 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-labels.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-labels.yaml @@ -168,9 +168,6 @@ spec: name: statsd-exporter-config readOnly: true dnsPolicy: ClusterFirst - nodeSelector: - key1: value1 - key2: value2 restartPolicy: Always schedulerName: default-scheduler serviceAccountName: envoy-ratelimit From c0f680db4097e1bf135293523fbb92980b30ebcd Mon Sep 17 00:00:00 2001 From: Oscar Boher Date: Thu, 12 Sep 2024 23:33:22 +0200 Subject: [PATCH 3/9] fix annotation merge if prometheus was disabled and annotations were defined Signed-off-by: Oscar Boher --- .../kubernetes/ratelimit/resource_provider.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index bb2c8e449ce..5f7c4ac81e7 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -204,7 +204,11 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { } } if r.rateLimitDeployment.Pod.Annotations != nil { - maps.Copy(annotations, r.rateLimitDeployment.Pod.Annotations) + if annotations != nil { + maps.Copy(annotations, r.rateLimitDeployment.Pod.Annotations) + } else { + annotations = r.rateLimitDeployment.Pod.Annotations + } } deployment := &appsv1.Deployment{ From 6a57aeb2db712357925b032ded8469d1672823b9 Mon Sep 17 00:00:00 2001 From: Oscar Boher Date: Fri, 13 Sep 2024 08:02:16 +0200 Subject: [PATCH 4/9] renamed labels and annotations to specify they apply to pods only Signed-off-by: Oscar Boher --- .../kubernetes/ratelimit/resource_provider.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index 5f7c4ac81e7..24bad46b4fe 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -189,25 +189,25 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { containers := expectedRateLimitContainers(r.rateLimit, r.rateLimitDeployment, r.Namespace) selector := resource.GetSelector(rateLimitLabels()) - labels := rateLimitLabels() + podLabels := rateLimitLabels() if r.rateLimitDeployment.Pod.Labels != nil { - maps.Copy(labels, r.rateLimitDeployment.Pod.Labels) - maps.Copy(labels, rateLimitLabels()) + maps.Copy(podLabels, r.rateLimitDeployment.Pod.Labels) + maps.Copy(podLabels, rateLimitLabels()) } - var annotations map[string]string + var podAnnotations map[string]string if enablePrometheus(r.rateLimit) { - annotations = map[string]string{ + podAnnotations = map[string]string{ "prometheus.io/path": "/metrics", "prometheus.io/port": strconv.Itoa(PrometheusPort), "prometheus.io/scrape": "true", } } if r.rateLimitDeployment.Pod.Annotations != nil { - if annotations != nil { - maps.Copy(annotations, r.rateLimitDeployment.Pod.Annotations) + if podAnnotations != nil { + maps.Copy(podAnnotations, r.rateLimitDeployment.Pod.Annotations) } else { - annotations = r.rateLimitDeployment.Pod.Annotations + podAnnotations = r.rateLimitDeployment.Pod.Annotations } } @@ -226,8 +226,8 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { Selector: selector, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: labels, - Annotations: annotations, + Labels: podLabels, + Annotations: podAnnotations, }, Spec: corev1.PodSpec{ Containers: containers, From 45e0263fd12890e21832f276079b8728a9c7ab7e Mon Sep 17 00:00:00 2001 From: Oscar Boher Date: Fri, 13 Sep 2024 12:49:45 +0200 Subject: [PATCH 5/9] linter Signed-off-by: Oscar Boher --- .../kubernetes/ratelimit/resource_provider.go | 7 +++---- .../kubernetes/ratelimit/resource_provider_test.go | 5 ++--- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index 24bad46b4fe..3d704b1c297 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -7,9 +7,11 @@ package ratelimit import ( _ "embed" - "golang.org/x/exp/maps" "strconv" + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource" + "golang.org/x/exp/maps" appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" @@ -18,9 +20,6 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" - - egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" - "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource" ) // ResourceKind indicates the main resources of envoy-ratelimit, diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go index c7aa23f7943..9ba9db1fc72 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go @@ -12,6 +12,8 @@ import ( "strconv" "testing" + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" @@ -23,9 +25,6 @@ import ( "k8s.io/utils/ptr" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/yaml" - - egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" - "github.com/envoyproxy/gateway/internal/envoygateway/config" ) var overrideTestData = flag.Bool("override-testdata", false, "if override the test output data.") From 2db9a57a3bb6409fe4000224e2b32a58b1268803 Mon Sep 17 00:00:00 2001 From: Oscar Boher Date: Wed, 18 Sep 2024 11:40:52 +0200 Subject: [PATCH 6/9] fix resource provider tests to new annotation behavior Signed-off-by: Oscar Boher --- .../kubernetes/ratelimit/testdata/deployments/custom.yaml | 2 ++ .../kubernetes/ratelimit/testdata/deployments/default-env.yaml | 2 ++ .../ratelimit/testdata/deployments/extension-env.yaml | 2 ++ .../kubernetes/ratelimit/testdata/deployments/override-env.yaml | 2 ++ .../ratelimit/testdata/deployments/redis-tls-settings.yaml | 2 ++ .../kubernetes/ratelimit/testdata/deployments/tolerations.yaml | 2 ++ .../kubernetes/ratelimit/testdata/deployments/volumes.yaml | 2 ++ 7 files changed, 14 insertions(+) diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml index c6c0bb1a696..a12bfedbfff 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml @@ -28,6 +28,8 @@ spec: metadata: annotations: prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "19001" creationTimestamp: null labels: app.kubernetes.io/component: ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml index c6c0bb1a696..a12bfedbfff 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml @@ -28,6 +28,8 @@ spec: metadata: annotations: prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "19001" creationTimestamp: null labels: app.kubernetes.io/component: ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml index 9bf03106f2d..e0e00675ab2 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml @@ -28,6 +28,8 @@ spec: metadata: annotations: prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "19001" creationTimestamp: null labels: app.kubernetes.io/component: ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml index c6c0bb1a696..a12bfedbfff 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml @@ -28,6 +28,8 @@ spec: metadata: annotations: prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "19001" creationTimestamp: null labels: app.kubernetes.io/component: ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml index 2e223af79c4..6d7be297e32 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml @@ -28,6 +28,8 @@ spec: metadata: annotations: prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "19001" creationTimestamp: null labels: app.kubernetes.io/component: ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml index 525c2b1f75d..27875b8789d 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml @@ -28,6 +28,8 @@ spec: metadata: annotations: prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "19001" creationTimestamp: null labels: app.kubernetes.io/component: ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml index 165a6819288..bb6f1b9e514 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml @@ -28,6 +28,8 @@ spec: metadata: annotations: prometheus.io/scrape: "true" + prometheus.io/path: "/metrics" + prometheus.io/port: "19001" creationTimestamp: null labels: app.kubernetes.io/component: ratelimit From 92a2b13412f640da0a09252f713ab9ef0880da04 Mon Sep 17 00:00:00 2001 From: Oscar Boher Date: Tue, 24 Sep 2024 18:41:25 +0200 Subject: [PATCH 7/9] go linter Signed-off-by: Oscar Boher --- .../infrastructure/kubernetes/ratelimit/resource_provider.go | 5 +++-- .../kubernetes/ratelimit/resource_provider_test.go | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index 3d704b1c297..36a19be561d 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -9,8 +9,6 @@ import ( _ "embed" "strconv" - egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" - "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource" "golang.org/x/exp/maps" appsv1 "k8s.io/api/apps/v1" autoscalingv2 "k8s.io/api/autoscaling/v2" @@ -20,6 +18,9 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/ptr" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/infrastructure/kubernetes/resource" ) // ResourceKind indicates the main resources of envoy-ratelimit, diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go index 9ba9db1fc72..c7aa23f7943 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go @@ -12,8 +12,6 @@ import ( "strconv" "testing" - egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" - "github.com/envoyproxy/gateway/internal/envoygateway/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" @@ -25,6 +23,9 @@ import ( "k8s.io/utils/ptr" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/yaml" + + egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" + "github.com/envoyproxy/gateway/internal/envoygateway/config" ) var overrideTestData = flag.Bool("override-testdata", false, "if override the test output data.") From d831f492ccf9cffb38c2f6402a15289cecafcb1e Mon Sep 17 00:00:00 2001 From: Oscar Boher Date: Wed, 25 Sep 2024 09:47:55 +0200 Subject: [PATCH 8/9] fix gen-check Signed-off-by: Oscar Boher --- .../testdata/deployments/custom.yaml | 4 +- .../testdata/deployments/default-env.yaml | 4 +- .../testdata/deployments/extension-env.yaml | 4 +- .../deployments/merge-annotations.yaml | 47 ++++--------------- .../testdata/deployments/merge-labels.yaml | 43 +++-------------- .../testdata/deployments/override-env.yaml | 4 +- .../deployments/redis-tls-settings.yaml | 4 +- .../testdata/deployments/tolerations.yaml | 4 +- .../testdata/deployments/volumes.yaml | 4 +- 9 files changed, 30 insertions(+), 88 deletions(-) diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml index 54732ea8492..0c1be549e83 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/custom.yaml @@ -27,9 +27,9 @@ spec: template: metadata: annotations: - prometheus.io/scrape: "true" - prometheus.io/path: "/metrics" + prometheus.io/path: /metrics prometheus.io/port: "19001" + prometheus.io/scrape: "true" creationTimestamp: null labels: app.kubernetes.io/component: ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml index 54732ea8492..0c1be549e83 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/default-env.yaml @@ -27,9 +27,9 @@ spec: template: metadata: annotations: - prometheus.io/scrape: "true" - prometheus.io/path: "/metrics" + prometheus.io/path: /metrics prometheus.io/port: "19001" + prometheus.io/scrape: "true" creationTimestamp: null labels: app.kubernetes.io/component: ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml index ae516fcbc4f..65c68972f9d 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/extension-env.yaml @@ -27,9 +27,9 @@ spec: template: metadata: annotations: - prometheus.io/scrape: "true" - prometheus.io/path: "/metrics" + prometheus.io/path: /metrics prometheus.io/port: "19001" + prometheus.io/scrape: "true" creationTimestamp: null labels: app.kubernetes.io/component: ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-annotations.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-annotations.yaml index 1061fe5337b..4bc241198c6 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-annotations.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-annotations.yaml @@ -26,11 +26,11 @@ spec: template: metadata: annotations: + key1: value1 + key2: value2 prometheus.io/path: /metrics prometheus.io/port: "19001" prometheus.io/scrape: "true" - key1: value1 - key2: value2 creationTimestamp: null labels: app.kubernetes.io/component: ratelimit @@ -53,9 +53,7 @@ spec: - name: LOG_LEVEL value: info - name: USE_STATSD - value: "true" - - name: STATSD_PORT - value: "9125" + value: "false" - name: CONFIG_TYPE value: GRPC_XDS_SOTW - name: CONFIG_GRPC_XDS_SERVER_URL @@ -84,6 +82,12 @@ spec: value: tcp - name: REDIS_URL value: redis.redis.svc:6379 + - name: USE_PROMETHEUS + value: "true" + - name: PROMETHEUS_ADDR + value: :19001 + - name: PROMETHEUS_MAPPER_YAML + value: /etc/statsd-exporter/conf.yaml image: envoyproxy/ratelimit:master imagePullPolicy: IfNotPresent name: envoy-ratelimit @@ -131,39 +135,6 @@ spec: - mountPath: /certs name: certs readOnly: true - - command: - - /bin/statsd_exporter - - --web.listen-address=:19001 - - --statsd.mapping-config=/etc/statsd-exporter/conf.yaml - image: prom/statsd-exporter:v0.18.0 - imagePullPolicy: IfNotPresent - name: prom-statsd-exporter - ports: - - containerPort: 9125 - name: statsd - protocol: TCP - - containerPort: 19001 - name: metrics - protocol: TCP - resources: - requests: - cpu: 100m - memory: 512Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - privileged: false - readOnlyRootFilesystem: true - runAsGroup: 65534 - runAsNonRoot: true - runAsUser: 65534 - seccompProfile: - type: RuntimeDefault - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - mountPath: /etc/statsd-exporter name: statsd-exporter-config readOnly: true diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-labels.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-labels.yaml index 7507f1e590b..6681232eeb8 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-labels.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/merge-labels.yaml @@ -53,9 +53,7 @@ spec: - name: LOG_LEVEL value: info - name: USE_STATSD - value: "true" - - name: STATSD_PORT - value: "9125" + value: "false" - name: CONFIG_TYPE value: GRPC_XDS_SOTW - name: CONFIG_GRPC_XDS_SERVER_URL @@ -84,6 +82,12 @@ spec: value: tcp - name: REDIS_URL value: redis.redis.svc:6379 + - name: USE_PROMETHEUS + value: "true" + - name: PROMETHEUS_ADDR + value: :19001 + - name: PROMETHEUS_MAPPER_YAML + value: /etc/statsd-exporter/conf.yaml image: envoyproxy/ratelimit:master imagePullPolicy: IfNotPresent name: envoy-ratelimit @@ -131,39 +135,6 @@ spec: - mountPath: /certs name: certs readOnly: true - - command: - - /bin/statsd_exporter - - --web.listen-address=:19001 - - --statsd.mapping-config=/etc/statsd-exporter/conf.yaml - image: prom/statsd-exporter:v0.18.0 - imagePullPolicy: IfNotPresent - name: prom-statsd-exporter - ports: - - containerPort: 9125 - name: statsd - protocol: TCP - - containerPort: 19001 - name: metrics - protocol: TCP - resources: - requests: - cpu: 100m - memory: 512Mi - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - privileged: false - readOnlyRootFilesystem: true - runAsGroup: 65534 - runAsNonRoot: true - runAsUser: 65534 - seccompProfile: - type: RuntimeDefault - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - volumeMounts: - mountPath: /etc/statsd-exporter name: statsd-exporter-config readOnly: true diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml index 63e007552a5..0c0f73f3c83 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/override-env.yaml @@ -27,9 +27,9 @@ spec: template: metadata: annotations: - prometheus.io/scrape: "true" - prometheus.io/path: "/metrics" + prometheus.io/path: /metrics prometheus.io/port: "19001" + prometheus.io/scrape: "true" creationTimestamp: null labels: app.kubernetes.io/component: ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml index d5b5b4d6f2e..29428fc447b 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/redis-tls-settings.yaml @@ -27,9 +27,9 @@ spec: template: metadata: annotations: - prometheus.io/scrape: "true" - prometheus.io/path: "/metrics" + prometheus.io/path: /metrics prometheus.io/port: "19001" + prometheus.io/scrape: "true" creationTimestamp: null labels: app.kubernetes.io/component: ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml index 6132dcde1f4..a2478222625 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/tolerations.yaml @@ -27,9 +27,9 @@ spec: template: metadata: annotations: - prometheus.io/scrape: "true" - prometheus.io/path: "/metrics" + prometheus.io/path: /metrics prometheus.io/port: "19001" + prometheus.io/scrape: "true" creationTimestamp: null labels: app.kubernetes.io/component: ratelimit diff --git a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml index aea23d96c99..30d8852d642 100644 --- a/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml +++ b/internal/infrastructure/kubernetes/ratelimit/testdata/deployments/volumes.yaml @@ -27,9 +27,9 @@ spec: template: metadata: annotations: - prometheus.io/scrape: "true" - prometheus.io/path: "/metrics" + prometheus.io/path: /metrics prometheus.io/port: "19001" + prometheus.io/scrape: "true" creationTimestamp: null labels: app.kubernetes.io/component: ratelimit From c3220c673b775388aae8a35f2f0ed78a80d9dd53 Mon Sep 17 00:00:00 2001 From: Oscar Boher Date: Thu, 26 Sep 2024 08:08:52 +0200 Subject: [PATCH 9/9] pod labels selector comment Signed-off-by: Oscar Boher --- .../infrastructure/kubernetes/ratelimit/resource_provider.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go index 36a19be561d..cdf1ad8747b 100644 --- a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go +++ b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go @@ -192,6 +192,8 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) { podLabels := rateLimitLabels() if r.rateLimitDeployment.Pod.Labels != nil { maps.Copy(podLabels, r.rateLimitDeployment.Pod.Labels) + // Copy overwrites values in the dest map if they exist in the src map https://pkg.go.dev/maps#Copy + // It's applied again with the rateLimitLabels that are used as deployment selector to ensure those are not overwritten by user input maps.Copy(podLabels, rateLimitLabels()) }