Remove name label from k8s client metrics

The `name` label in the `controller_clientset_k8s_request_total` metric produce an excessive amount of cardinality as `events` metrics generate a new string for each event. Similarly, each new `replicasets` also generates a new label set. This can lead to hundreds of thousands of unique metrics over a couple weeks in a large deployment. Signed-off-by: SuperQ <[email protected]>
argoproj · Jun 19, 2023 · c87bbd8 · c87bbd8
1 parent 51c867d
commit c87bbd8
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 10 deletions.
diff --git a/controller/metrics/client.go b/controller/metrics/client.go
@@ -23,20 +23,15 @@ func (f *K8sRequestsCountProvider) MustRegister(registerer prometheus.Registerer
 
 // IncKubernetesRequest increments the kubernetes client counter
 func (m *K8sRequestsCountProvider) IncKubernetesRequest(resourceInfo kubeclientmetrics.ResourceInfo) error {
-	name := resourceInfo.Name
 	namespace := resourceInfo.Namespace
 	kind := resourceInfo.Kind
 	statusCode := strconv.Itoa(resourceInfo.StatusCode)
-	if resourceInfo.Verb == kubeclientmetrics.List {
-		name = "N/A"
-	}
 	if resourceInfo.Verb == kubeclientmetrics.Unknown {
 		namespace = "Unknown"
-		name = "Unknown"
 		kind = "Unknown"
 	}
 	if m.k8sRequestsCount != nil {
-		m.k8sRequestsCount.WithLabelValues(kind, namespace, name, string(resourceInfo.Verb), statusCode).Inc()
+		m.k8sRequestsCount.WithLabelValues(kind, namespace, string(resourceInfo.Verb), statusCode).Inc()
 	}
 	return nil
 }
diff --git a/controller/metrics/client_test.go b/controller/metrics/client_test.go
@@ -9,16 +9,15 @@ import (
 )
 
 const expectedKubernetesRequest = `# TYPE controller_clientset_k8s_request_total counter
-controller_clientset_k8s_request_total{kind="Unknown",name="Unknown",namespace="Unknown",status_code="200",verb="Unknown"} 1
-controller_clientset_k8s_request_total{kind="replicasets",name="N/A",namespace="default",status_code="200",verb="List"} 1`
+controller_clientset_k8s_request_total{kind="Unknown",namespace="Unknown",status_code="200",verb="Unknown"} 1
+controller_clientset_k8s_request_total{kind="replicasets",namespace="default",status_code="200",verb="List"} 1`
 
 func TestIncKubernetesRequest(t *testing.T) {
 	config := newFakeServerConfig()
 	metricsServ := NewMetricsServer(config)
 	config.K8SRequestProvider.IncKubernetesRequest(kubeclientmetrics.ResourceInfo{
 		Kind:       "replicasets",
 		Namespace:  "default",
-		Name:       "test",
 		Verb:       kubeclientmetrics.List,
 		StatusCode: 200,
 	})

diff --git a/controller/metrics/prommetrics.go b/controller/metrics/prommetrics.go
@@ -219,7 +219,7 @@ var (
 			Name:      "k8s_request_total",
 			Help:      "Number of kubernetes requests executed during application reconciliation.",
 		},
-		[]string{"kind", "namespace", "name", "verb", "status_code"},
+		[]string{"kind", "namespace", "verb", "status_code"},
 	)
 )