From bfd59a0d3b4e2cda4806818bb5087abbc51f25aa Mon Sep 17 00:00:00 2001 From: Garrybest Date: Mon, 29 Nov 2021 19:38:25 +0800 Subject: [PATCH] Reform Builder to adapt custom resource registry Signed-off-by: Garrybest --- internal/store/builder.go | 115 +++++++++++++++++++++++++++----- pkg/builder/builder.go | 33 ++++++++- pkg/builder/types/interfaces.go | 17 ++++- 3 files changed, 146 insertions(+), 19 deletions(-) diff --git a/internal/store/builder.go b/internal/store/builder.go index aace8bad75..7d7da21eff 100644 --- a/internal/store/builder.go +++ b/internal/store/builder.go @@ -43,6 +43,7 @@ import ( "k8s.io/klog/v2" ksmtypes "k8s.io/kube-state-metrics/v2/pkg/builder/types" + "k8s.io/kube-state-metrics/v2/pkg/customresource" generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" metricsstore "k8s.io/kube-state-metrics/v2/pkg/metrics_store" "k8s.io/kube-state-metrics/v2/pkg/options" @@ -57,21 +58,23 @@ var _ ksmtypes.BuilderInterface = &Builder{} // Builder helps to build store. It follows the builder pattern // (https://en.wikipedia.org/wiki/Builder_pattern). type Builder struct { - kubeClient clientset.Interface - vpaClient vpaclientset.Interface - namespaces options.NamespaceList - namespaceFilter string - ctx context.Context - enabledResources []string - familyGeneratorFilter generator.FamilyGeneratorFilter - listWatchMetrics *watch.ListWatchMetrics - shardingMetrics *sharding.Metrics - shard int32 - totalShards int - buildStoresFunc ksmtypes.BuildStoresFunc - allowAnnotationsList map[string][]string - allowLabelsList map[string][]string - useAPIServerCache bool + kubeClient clientset.Interface + customResourceClients map[string]interface{} + vpaClient vpaclientset.Interface + namespaces options.NamespaceList + namespaceFilter string + ctx context.Context + enabledResources []string + familyGeneratorFilter generator.FamilyGeneratorFilter + listWatchMetrics *watch.ListWatchMetrics + shardingMetrics *sharding.Metrics + shard int32 + totalShards int + buildStoresFunc ksmtypes.BuildStoresFunc + buildCustomResourceStoresFunc ksmtypes.BuildCustomResourceStoresFunc + allowAnnotationsList map[string][]string + allowLabelsList map[string][]string + useAPIServerCache bool } // NewBuilder returns a new builder. @@ -134,6 +137,16 @@ func (b *Builder) WithVPAClient(c vpaclientset.Interface) { b.vpaClient = c } +// WithCustomResourceClients sets the customResourceClients property of a Builder. +func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) { + b.customResourceClients = cs +} + +// WithUsingAPIServerCache configures whether using APIServer cache or not. +func (b *Builder) WithUsingAPIServerCache(u bool) { + b.useAPIServerCache = u +} + // WithFamilyGeneratorFilter configures the family generator filter which decides which // metrics are to be exposed by the store build by the Builder. func (b *Builder) WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter) { @@ -141,9 +154,13 @@ func (b *Builder) WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter) { } // WithGenerateStoresFunc configures a custom generate store function -func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc, u bool) { +func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc) { b.buildStoresFunc = f - b.useAPIServerCache = u +} + +// WithGenerateCustomResourceStoresFunc configures a custom generate custom resource store function +func (b *Builder) WithGenerateCustomResourceStoresFunc(f ksmtypes.BuildCustomResourceStoresFunc) { + b.buildCustomResourceStoresFunc = f } // DefaultGenerateStoresFunc returns default buildStores function @@ -151,6 +168,29 @@ func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc { return b.buildStores } +// DefaultGenerateCustomResourceStoresFunc returns default buildCustomResourceStores function +func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc { + return b.buildCustomResourceStores +} + +// WithCustomResourceStoreFactories returns configures a custom resource stores factory +func (b *Builder) WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory) { + for _, f := range fs { + if _, ok := availableStores[f.Name()]; ok { + klog.Warningf("The internal resource store named %s already exists and is overridden by a custom resource store with the same name, please make sure it meets your expectation", f.Name()) + } + availableStores[f.Name()] = func(b *Builder) []cache.Store { + return b.buildCustomResourceStoresFunc( + f.Name(), + f.MetricFamilyGenerators(b.allowAnnotationsList[f.Name()], b.allowLabelsList[f.Name()]), + f.ExpectedType(), + f.ListWatch, + b.useAPIServerCache, + ) + } + } +} + // WithAllowAnnotations configures which annotations can be returned for metrics func (b *Builder) WithAllowAnnotations(annotations map[string][]string) { if len(annotations) > 0 { @@ -414,6 +454,47 @@ func (b *Builder) buildStores( return stores } +// TODO(Garrybest): Merge `buildStores` and `buildCustomResourceStores` +func (b *Builder) buildCustomResourceStores(resourceName string, + metricFamilies []generator.FamilyGenerator, + expectedType interface{}, + listWatchFunc func(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher, + useAPIServerCache bool, +) []cache.Store { + metricFamilies = generator.FilterFamilyGenerators(b.familyGeneratorFilter, metricFamilies) + composedMetricGenFuncs := generator.ComposeMetricGenFuncs(metricFamilies) + familyHeaders := generator.ExtractMetricFamilyHeaders(metricFamilies) + + customResourceClient, ok := b.customResourceClients[resourceName] + if !ok { + klog.Warningf("Custom resource client %s does not exist", resourceName) + return []cache.Store{} + } + + if b.namespaces.IsAllNamespaces() { + store := metricsstore.NewMetricsStore( + familyHeaders, + composedMetricGenFuncs, + ) + listWatcher := listWatchFunc(customResourceClient, v1.NamespaceAll, b.namespaceFilter) + b.startReflector(expectedType, store, listWatcher, useAPIServerCache) + return []cache.Store{store} + } + + stores := make([]cache.Store, 0, len(b.namespaces)) + for _, ns := range b.namespaces { + store := metricsstore.NewMetricsStore( + familyHeaders, + composedMetricGenFuncs, + ) + listWatcher := listWatchFunc(customResourceClient, ns, b.namespaceFilter) + b.startReflector(expectedType, store, listWatcher, useAPIServerCache) + stores = append(stores, store) + } + + return stores +} + // startReflector starts a Kubernetes client-go reflector with the given // listWatcher and registers it with the given store. func (b *Builder) startReflector( diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index cd28bf1a0a..1e2763c455 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -28,6 +28,7 @@ import ( internalstore "k8s.io/kube-state-metrics/v2/internal/store" ksmtypes "k8s.io/kube-state-metrics/v2/pkg/builder/types" + "k8s.io/kube-state-metrics/v2/pkg/customresource" metricsstore "k8s.io/kube-state-metrics/v2/pkg/metrics_store" "k8s.io/kube-state-metrics/v2/pkg/options" ) @@ -81,12 +82,27 @@ func (b *Builder) WithVPAClient(c vpaclientset.Interface) { b.internal.WithVPAClient(c) } +// WithCustomResourceClients sets the customResourceClients property of a Builder. +func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) { + b.internal.WithCustomResourceClients(cs) +} + +// WithUsingAPIServerCache configures whether using APIServer cache or not. +func (b *Builder) WithUsingAPIServerCache(u bool) { + b.internal.WithUsingAPIServerCache(u) +} + // WithFamilyGeneratorFilter configures the family generator filter which decides which // metrics are to be exposed by the store build by the Builder. func (b *Builder) WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter) { b.internal.WithFamilyGeneratorFilter(l) } +// WithAllowAnnotations configures which annotations can be returned for metrics +func (b *Builder) WithAllowAnnotations(annotations map[string][]string) { + b.internal.WithAllowAnnotations(annotations) +} + // WithAllowLabels configures which labels can be returned for metrics func (b *Builder) WithAllowLabels(l map[string][]string) { b.internal.WithAllowLabels(l) @@ -94,7 +110,12 @@ func (b *Builder) WithAllowLabels(l map[string][]string) { // WithGenerateStoresFunc configures a custom generate store function func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc) { - b.internal.WithGenerateStoresFunc(f, false) + b.internal.WithGenerateStoresFunc(f) +} + +// WithGenerateCustomResourceStoresFunc configures a custom generate custom resource store function +func (b *Builder) WithGenerateCustomResourceStoresFunc(f ksmtypes.BuildCustomResourceStoresFunc) { + b.internal.WithGenerateCustomResourceStoresFunc(f) } // DefaultGenerateStoresFunc returns default buildStore function @@ -102,6 +123,16 @@ func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc { return b.internal.DefaultGenerateStoresFunc() } +// DefaultGenerateCustomResourceStoresFunc returns default buildStores function +func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc { + return b.internal.DefaultGenerateCustomResourceStoresFunc() +} + +// WithCustomResourceStoreFactories returns configures a custom resource stores factory +func (b *Builder) WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory) { + b.internal.WithCustomResourceStoreFactories(fs...) +} + // Build initializes and registers all enabled stores. // Returns metric writers. func (b *Builder) Build() []metricsstore.MetricsWriter { diff --git a/pkg/builder/types/interfaces.go b/pkg/builder/types/interfaces.go index 15995435ad..21b657a24f 100644 --- a/pkg/builder/types/interfaces.go +++ b/pkg/builder/types/interfaces.go @@ -26,6 +26,7 @@ import ( clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" + "k8s.io/kube-state-metrics/v2/pkg/customresource" generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator" "k8s.io/kube-state-metrics/v2/pkg/options" ) @@ -39,10 +40,16 @@ type BuilderInterface interface { WithContext(ctx context.Context) WithKubeClient(c clientset.Interface) WithVPAClient(c vpaclientset.Interface) + WithCustomResourceClients(cs map[string]interface{}) + WithUsingAPIServerCache(u bool) WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter) + WithAllowAnnotations(a map[string][]string) WithAllowLabels(l map[string][]string) - WithGenerateStoresFunc(f BuildStoresFunc, useAPIServerCache bool) + WithGenerateStoresFunc(f BuildStoresFunc) + WithGenerateCustomResourceStoresFunc(f BuildCustomResourceStoresFunc) DefaultGenerateStoresFunc() BuildStoresFunc + DefaultGenerateCustomResourceStoresFunc() BuildCustomResourceStoresFunc + WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory) Build() []metricsstore.MetricsWriter BuildStores() [][]cache.Store } @@ -54,6 +61,14 @@ type BuildStoresFunc func(metricFamilies []generator.FamilyGenerator, useAPIServerCache bool, ) []cache.Store +// BuildCustomResourceStoresFunc function signature that is used to return a list of custom resource cache.Store +type BuildCustomResourceStoresFunc func(resourceName string, + metricFamilies []generator.FamilyGenerator, + expectedType interface{}, + listWatchFunc func(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher, + useAPIServerCache bool, +) []cache.Store + // AllowDenyLister interface for AllowDeny lister that can allow or exclude metrics by there names type AllowDenyLister interface { IsIncluded(string) bool