ratelimit support prometheus (#2729)

* ratelimit support prometheus Signed-off-by: zirain <[email protected]> * lint Signed-off-by: zirain <[email protected]> * use 19001 Signed-off-by: zirain <[email protected]> --------- Signed-off-by: zirain <[email protected]>
envoyproxy · Mar 2, 2024 · 4816db4 · 4816db4
1 parent 74f86f9
commit 4816db4
Show file tree

Hide file tree

Showing 21 changed files with 1,159 additions and 353 deletions.
diff --git a/api/v1alpha1/envoygateway_types.go b/api/v1alpha1/envoygateway_types.go
@@ -326,6 +326,25 @@ type RateLimit struct {
 	// otherwise, don't let the traffic pass and return 500.
 	// If not set, FailClosed is False.
 	FailClosed bool `json:"failClosed"`
+
+	// Telemetry defines telemetry configuration for RateLimit.
+	// +optional
+	Telemetry *RateLimitTelemetry `json:"telemetry,omitempty"`
+}
+
+type RateLimitTelemetry struct {
+	// Metrics defines metrics configuration for RateLimit.
+	Metrics *RateLimitMetrics `json:"metrics,omitempty"`
+}
+
+type RateLimitMetrics struct {
+	// Prometheus defines the configuration for prometheus endpoint.
+	Prometheus *RateLimitMetricsPrometheusProvider `json:"prometheus,omitempty"`
+}
+
+type RateLimitMetricsPrometheusProvider struct {
+	// Disable the Prometheus endpoint.
+	Disable bool `json:"disable,omitempty"`
 }
 
 // RateLimitDatabaseBackend defines the configuration associated with

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/internal/infrastructure/kubernetes/ratelimit/resource.go b/internal/infrastructure/kubernetes/ratelimit/resource.go
@@ -91,7 +91,9 @@ const (
 	// ReadinessPath is readiness path for readiness probe.
 	ReadinessPath = "/healthcheck"
 	// ReadinessPort is readiness port for readiness probe.
-	ReadinessPort = 8080
+	ReadinessPort  = 8080
+	StatsdPort     = 9125
+	PrometheusPort = 19001
 )
 
 // GetServiceURL returns the URL for the rate limit service.
@@ -163,9 +165,47 @@ func expectedRateLimitContainers(rateLimit *egv1a1.RateLimit, rateLimitDeploymen
 		},
 	}
 
+	if enablePrometheus(rateLimit) {
+		containers = append(containers, promStatsdExporterContainer())
+	}
+
 	return containers
 }
 
+func promStatsdExporterContainer() corev1.Container {
+	return corev1.Container{
+		Name:            "prom-statsd-exporter",
+		Image:           "prom/statsd-exporter:v0.18.0",
+		ImagePullPolicy: corev1.PullIfNotPresent,
+		Command: []string{
+			"/bin/statsd_exporter",
+			fmt.Sprintf("--web.listen-address=:%d", PrometheusPort),
+			"--statsd.mapping-config=/etc/statsd-exporter/conf.yaml",
+		},
+		Ports: []corev1.ContainerPort{
+			{
+				Name:          "statsd",
+				ContainerPort: StatsdPort,
+				Protocol:      corev1.ProtocolTCP,
+			},
+			{
+				Name:          "metrics",
+				ContainerPort: PrometheusPort,
+				Protocol:      corev1.ProtocolTCP,
+			},
+		},
+		VolumeMounts: []corev1.VolumeMount{
+			{
+				Name:      "statsd-exporter-config",
+				ReadOnly:  true,
+				MountPath: "/etc/statsd-exporter",
+			},
+		},
+		TerminationMessagePolicy: corev1.TerminationMessageReadFile,
+		TerminationMessagePath:   "/dev/termination-log",
+	}
+}
+
 // expectedContainerVolumeMounts returns expected rateLimit container volume mounts.
 func expectedContainerVolumeMounts(rateLimit *egv1a1.RateLimit, rateLimitDeployment *egv1a1.KubernetesDeploymentSpec) []corev1.VolumeMount {
 	var volumeMounts []corev1.VolumeMount
@@ -214,6 +254,21 @@ func expectedDeploymentVolumes(rateLimit *egv1a1.RateLimit, rateLimitDeployment
 		},
 	})
 
+	if enablePrometheus(rateLimit) {
+		volumes = append(volumes, corev1.Volume{
+			Name: "statsd-exporter-config",
+			VolumeSource: corev1.VolumeSource{
+				ConfigMap: &corev1.ConfigMapVolumeSource{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: "statsd-exporter-config",
+					},
+					Optional:    ptr.To(true),
+					DefaultMode: ptr.To[int32](420),
+				},
+			},
+		})
+	}
+
 	return resource.ExpectedDeploymentVolumes(rateLimitDeployment.Pod, volumes)
 }
 

diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider.go
@@ -6,6 +6,9 @@
 package ratelimit
 
 import (
+	_ "embed"
+	"strconv"
+
 	appsv1 "k8s.io/api/apps/v1"
 	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	corev1 "k8s.io/api/core/v1"
@@ -24,8 +27,12 @@ const (
 	ResourceKindService        = "Service"
 	ResourceKindDeployment     = "Deployment"
 	ResourceKindServiceAccount = "ServiceAccount"
+	appsAPIVersion             = "apps/v1"
 )
 
+//go:embed statsd_conf.yaml
+var statsConf string
+
 type ResourceRender struct {
 	// Namespace is the Namespace used for managed infra.
 	Namespace string
@@ -51,9 +58,36 @@ func (r *ResourceRender) Name() string {
 	return InfraName
 }
 
-// ConfigMap is deprecated since ratelimit supports xds grpc config server.
+func enablePrometheus(rl *egv1a1.RateLimit) bool {
+	if rl != nil &&
+		rl.Telemetry != nil &&
+		rl.Telemetry.Metrics.Prometheus != nil {
+		return !rl.Telemetry.Metrics.Prometheus.Disable
+	}
+
+	return true
+}
+
+// ConfigMap returns the expected rate limit ConfigMap based on the provided infra.
 func (r *ResourceRender) ConfigMap() (*corev1.ConfigMap, error) {
-	return nil, nil
+	if !enablePrometheus(r.rateLimit) {
+		return nil, nil
+	}
+
+	return &corev1.ConfigMap{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ConfigMap",
+			APIVersion: "v1",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: r.Namespace,
+			Name:      "statsd-exporter-config",
+			Labels:    rateLimitLabels(),
+		},
+		Data: map[string]string{
+			"conf.yaml": statsConf,
+		},
+	}, nil
 }
 
 // Service returns the expected rate limit Service based on the provided infra.
@@ -139,21 +173,26 @@ func (r *ResourceRender) ServiceAccount() (*corev1.ServiceAccount, error) {
 
 // Deployment returns the expected rate limit Deployment based on the provided infra.
 func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) {
-	const apiVersion = "apps/v1"
-
 	containers := expectedRateLimitContainers(r.rateLimit, r.rateLimitDeployment)
 	labels := rateLimitLabels()
 	selector := resource.GetSelector(labels)
 
 	var annotations map[string]string
+	if enablePrometheus(r.rateLimit) {
+		annotations = map[string]string{
+			"prometheus.io/path":   "/metrics",
+			"prometheus.io/port":   strconv.Itoa(PrometheusPort),
+			"prometheus.io/scrape": "true",
+		}
+	}
 	if r.rateLimitDeployment.Pod.Annotations != nil {
 		annotations = r.rateLimitDeployment.Pod.Annotations
 	}
 
 	deployment := &appsv1.Deployment{
 		TypeMeta: metav1.TypeMeta{
 			Kind:       ResourceKindDeployment,
-			APIVersion: apiVersion,
+			APIVersion: appsAPIVersion,
 		},
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: r.Namespace,
@@ -197,7 +236,7 @@ func (r *ResourceRender) Deployment() (*appsv1.Deployment, error) {
 			deployment.OwnerReferences = []metav1.OwnerReference{
 				{
 					Kind:       ResourceKindDeployment,
-					APIVersion: apiVersion,
+					APIVersion: appsAPIVersion,
 					Name:       "envoy-gateway",
 					UID:        uid,
 				},

diff --git a/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go b/internal/infrastructure/kubernetes/ratelimit/resource_provider_test.go
@@ -6,6 +6,7 @@
 package ratelimit
 
 import (
+	"flag"
 	"fmt"
 	"os"
 	"testing"
@@ -26,6 +27,10 @@ import (
 	"github.com/envoyproxy/gateway/internal/envoygateway/config"
 )
 
+var (
+	overrideTestData = flag.Bool("override-testdata", false, "if override the test output data.")
+)
+
 const (
 	// RedisAuthEnvVar is the redis auth.
 	RedisAuthEnvVar = "REDIS_AUTH"
@@ -152,6 +157,47 @@ func loadService() (*corev1.Service, error) {
 	return svc, nil
 }
 
+func TestConfigmap(t *testing.T) {
+	cfg, err := config.New()
+	require.NoError(t, err)
+
+	cfg.EnvoyGateway.RateLimit = &egv1a1.RateLimit{
+		Backend: egv1a1.RateLimitDatabaseBackend{
+			Type: egv1a1.RedisBackendType,
+			Redis: &egv1a1.RateLimitRedisSettings{
+				URL: "redis.redis.svc:6379",
+			},
+		},
+	}
+	r := NewResourceRender(cfg.Namespace, cfg.EnvoyGateway, ownerReferenceUID)
+	cm, err := r.ConfigMap()
+	require.NoError(t, err)
+
+	if *overrideTestData {
+		cmYAML, err := yaml.Marshal(cm)
+		require.NoError(t, err)
+		// nolint:gosec
+		err = os.WriteFile("testdata/envoy-ratelimit-configmap.yaml", cmYAML, 0644)
+		require.NoError(t, err)
+		return
+	}
+
+	expected, err := loadConfigmap()
+	require.NoError(t, err)
+
+	assert.Equal(t, expected, cm)
+}
+
+func loadConfigmap() (*corev1.ConfigMap, error) {
+	configmapYAML, err := os.ReadFile("testdata/envoy-ratelimit-configmap.yaml")
+	if err != nil {
+		return nil, err
+	}
+	cm := &corev1.ConfigMap{}
+	_ = yaml.Unmarshal(configmapYAML, cm)
+	return cm, nil
+}
+
 func TestDeployment(t *testing.T) {
 	cfg, err := config.New()
 	require.NoError(t, err)
@@ -173,6 +219,25 @@ func TestDeployment(t *testing.T) {
 			rateLimit: rateLimit,
 			deploy:    cfg.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment,
 		},
+		{
+			caseName: "disable-prometheus",
+			rateLimit: &egv1a1.RateLimit{
+				Backend: egv1a1.RateLimitDatabaseBackend{
+					Type: egv1a1.RedisBackendType,
+					Redis: &egv1a1.RateLimitRedisSettings{
+						URL: "redis.redis.svc:6379",
+					},
+				},
+				Telemetry: &egv1a1.RateLimitTelemetry{
+					Metrics: &egv1a1.RateLimitMetrics{
+						Prometheus: &egv1a1.RateLimitMetricsPrometheusProvider{
+							Disable: true,
+						},
+					},
+				},
+			},
+			deploy: cfg.EnvoyGateway.GetEnvoyGatewayProvider().GetEnvoyGatewayKubeProvider().RateLimitDeployment,
+		},
 		{
 			caseName:  "patch-deployment",
 			rateLimit: rateLimit,
@@ -566,6 +631,15 @@ func TestDeployment(t *testing.T) {
 			dp, err := r.Deployment()
 			require.NoError(t, err)
 
+			if *overrideTestData {
+				deploymentYAML, err := yaml.Marshal(dp)
+				require.NoError(t, err)
+				// nolint:gosec
+				err = os.WriteFile(fmt.Sprintf("testdata/deployments/%s.yaml", tc.caseName), deploymentYAML, 0644)
+				require.NoError(t, err)
+				return
+			}
+
 			expected, err := loadDeployment(tc.caseName)
 			require.NoError(t, err)