From fdd778799fb363d9fe42f065c91a8606df73b56f Mon Sep 17 00:00:00 2001 From: Ethan Wood-Thomas Date: Fri, 20 Dec 2024 11:48:22 -0500 Subject: [PATCH] [CONTINT-4416] Optimize KSM configmap collection (#32368) --- pkg/kubestatemetrics/builder/builder.go | 111 ++++++++++++++++++ ...configmap-collection-dcd3bfaa71fb2865.yaml | 13 ++ 2 files changed, 124 insertions(+) create mode 100644 releasenotes-dca/notes/optimize-ksm-configmap-collection-dcd3bfaa71fb2865.yaml diff --git a/pkg/kubestatemetrics/builder/builder.go b/pkg/kubestatemetrics/builder/builder.go index d0c5ebd1ea998..ea26602567f56 100644 --- a/pkg/kubestatemetrics/builder/builder.go +++ b/pkg/kubestatemetrics/builder/builder.go @@ -9,6 +9,7 @@ package builder import ( "context" + "fmt" "reflect" "time" @@ -18,7 +19,11 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + apiwatch "k8s.io/apimachinery/pkg/watch" clientset "k8s.io/client-go/kubernetes" + "k8s.io/client-go/metadata" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" ksmbuild "k8s.io/kube-state-metrics/v2/pkg/builder" ksmtypes "k8s.io/kube-state-metrics/v2/pkg/builder/types" @@ -196,6 +201,14 @@ func GenerateStores[T any]( isPod = true } else if u, ok := expectedType.(*unstructured.Unstructured); ok { isPod = u.GetAPIVersion() == "v1" && u.GetKind() == "Pod" + } else if _, ok := expectedType.(*corev1.ConfigMap); ok { + configMapStore, err := generateConfigMapStores(b, metricFamilies, useAPIServerCache) + if err != nil { + log.Debugf("Defaulting to kube-state-metrics for configmap collection: %v", err) + } else { + log.Debug("Using meta.k8s.io API for configmap collection") + return configMapStore + } } if b.namespaces.IsAllNamespaces() { @@ -340,3 +353,101 @@ func handlePodCollection[T any](b *Builder, store cache.Store, client T, listWat listWatcher := listWatchFunc(client, namespace, fieldSelector) b.startReflector(&corev1.Pod{}, store, listWatcher, useAPIServerCache) } + +func generateConfigMapStores( + b *Builder, + metricFamilies []generator.FamilyGenerator, + useAPIServerCache bool, +) ([]cache.Store, error) { + restConfig, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("failed to create in-cluster config for metadata client: %w", err) + } + + metadataClient, err := metadata.NewForConfig(restConfig) + if err != nil { + return nil, fmt.Errorf("failed to create metadata client: %w", err) + } + + gvr := schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "configmaps", + } + + filteredMetricFamilies := generator.FilterFamilyGenerators(b.allowDenyList, metricFamilies) + composedMetricGenFuncs := generator.ComposeMetricGenFuncs(filteredMetricFamilies) + + stores := make([]cache.Store, 0) + + if b.namespaces.IsAllNamespaces() { + log.Infof("Using NamespaceAll for ConfigMap collection.") + store := store.NewMetricsStore(composedMetricGenFuncs, "configmap") + listWatcher := createConfigMapListWatch(metadataClient, gvr, v1.NamespaceAll) + b.startReflector(&corev1.ConfigMap{}, store, listWatcher, useAPIServerCache) + return []cache.Store{store}, nil + } + + for _, ns := range b.namespaces { + store := store.NewMetricsStore(composedMetricGenFuncs, "configmap") + listWatcher := createConfigMapListWatch(metadataClient, gvr, ns) + b.startReflector(&corev1.ConfigMap{}, store, listWatcher, useAPIServerCache) + stores = append(stores, store) + } + + return stores, nil +} + +func createConfigMapListWatch(metadataClient metadata.Interface, gvr schema.GroupVersionResource, namespace string) *cache.ListWatch { + return &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + result, err := metadataClient.Resource(gvr).Namespace(namespace).List(context.TODO(), options) + if err != nil { + return nil, err + } + + configMapList := &corev1.ConfigMapList{} + for _, item := range result.Items { + configMapList.Items = append(configMapList.Items, corev1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{ + Name: item.GetName(), + Namespace: item.GetNamespace(), + UID: item.GetUID(), + ResourceVersion: item.GetResourceVersion(), + }, + }) + } + + return configMapList, nil + }, + WatchFunc: func(options v1.ListOptions) (apiwatch.Interface, error) { + watcher, err := metadataClient.Resource(gvr).Namespace(namespace).Watch(context.TODO(), options) + if err != nil { + return nil, err + } + + return apiwatch.Filter(watcher, func(event apiwatch.Event) (apiwatch.Event, bool) { + if event.Object == nil { + return event, false + } + + partialObject, ok := event.Object.(*v1.PartialObjectMetadata) + if !ok { + return event, false + } + + configMap := &corev1.ConfigMap{ + ObjectMeta: v1.ObjectMeta{ + Name: partialObject.GetName(), + Namespace: partialObject.GetNamespace(), + UID: partialObject.GetUID(), + ResourceVersion: partialObject.GetResourceVersion(), + }, + } + + event.Object = configMap + return event, true + }), nil + }, + } +} diff --git a/releasenotes-dca/notes/optimize-ksm-configmap-collection-dcd3bfaa71fb2865.yaml b/releasenotes-dca/notes/optimize-ksm-configmap-collection-dcd3bfaa71fb2865.yaml new file mode 100644 index 0000000000000..0e5a6cd7f5c92 --- /dev/null +++ b/releasenotes-dca/notes/optimize-ksm-configmap-collection-dcd3bfaa71fb2865.yaml @@ -0,0 +1,13 @@ +# Each section from every release note are combined when the +# CHANGELOG-DCA.rst is rendered. So the text needs to be worded so that +# it does not depend on any information only available in another +# section. This may mean repeating some details, but each section +# must be readable independently of the other. +# +# Each section note must be formatted as reStructuredText. +--- +enhancements: + - | + The `kubernetes_state_core` check now collects only metadata for configmaps, + reducing memory, CPU, and network usage in the Cluster Agent while preserving + full metric functionality.