From 577bf2e8b5aacf88962c1d9bdbfb939c55edec78 Mon Sep 17 00:00:00 2001
From: Garrybest <garrybest@foxmail.com>
Date: Mon, 29 Nov 2021 19:36:54 +0800
Subject: [PATCH 1/4] Design a custom resource RegistryFactory for out-of-tree
 implements

Signed-off-by: Garrybest <garrybest@foxmail.com>
---
 pkg/customresource/registry_factory.go | 116 +++++++++++++++++++++++++
 1 file changed, 116 insertions(+)
 create mode 100644 pkg/customresource/registry_factory.go

diff --git a/pkg/customresource/registry_factory.go b/pkg/customresource/registry_factory.go
new file mode 100644
index 0000000000..e4ccee9b1e
--- /dev/null
+++ b/pkg/customresource/registry_factory.go
@@ -0,0 +1,116 @@
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package customresource
+
+import (
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+
+	generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
+)
+
+// RegistryFactory is a registry interface for a CustomResourceStore.
+// Users who want to extend the kube-state-metrics to support Custom Resource metrics should
+// implement this interface.
+type RegistryFactory interface {
+	// Name returns the name of custom resource.
+	//
+	// Example:
+	//
+	// func (f *FooFactory) Name() string {
+	//	return "foos"
+	// }
+	Name() string
+
+	// CreateClient creates a new custom resource client for the given config.
+	//
+	// Example:
+	//
+	// func (f *FooFactory) CreateClient(cfg *rest.Config) (interface{}, error) {
+	// 	return clientset.NewForConfig(cfg)
+	// }
+	CreateClient(cfg *rest.Config) (interface{}, error)
+
+	// MetricFamilyGenerators returns the metric family generators to generate metric families with a
+	// Kubernetes custom resource object.
+	//
+	// Example:
+	//
+	// func (f *FooFactory) MetricFamilyGenerators(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator {
+	//	return []generator.FamilyGenerator{
+	//		*generator.NewFamilyGenerator(
+	//			"kube_foo_spec_replicas",
+	//			"Number of desired replicas for a foo.",
+	//			metric.Gauge,
+	//			"",
+	//			wrapFooFunc(func(f *samplev1alpha1.Foo) *metric.Family {
+	//				return &metric.Family{
+	//					Metrics: []*metric.Metric{
+	//						{
+	//							Value: float64(*f.Spec.Replicas),
+	//						},
+	//					},
+	//				}
+	//			}),
+	//		),
+	//		*generator.NewFamilyGenerator(
+	//			"kube_foo_status_replicas_available",
+	//			"The number of available replicas per foo.",
+	//			metric.Gauge,
+	//			"",
+	//			wrapFooFunc(func(f *samplev1alpha1.Foo) *metric.Family {
+	//				return &metric.Family{
+	//					Metrics: []*metric.Metric{
+	//						{
+	//							Value: float64(f.Status.AvailableReplicas),
+	//						},
+	//					},
+	//				}
+	//			}),
+	//		),
+	//	}
+	// }
+	MetricFamilyGenerators(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator
+
+	// ExpectedType returns a pointer to an empty custom resource object.
+	//
+	// Example:
+	//
+	// func (f *FooFactory) ExpectedType() interface{} {
+	//	return &samplev1alpha1.Foo{}
+	// }
+	ExpectedType() interface{}
+
+	// ListWatch constructs a cache.ListerWatcher of the custom resource object.
+	//
+	// Example:
+	//
+	// func (f *FooFactory) ListWatch(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher {
+	// 	client := customResourceClient.(*clientset.Clientset)
+	//	return &cache.ListWatch{
+	//		ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
+	//			opts.FieldSelector = fieldSelector
+	//			return client.SamplecontrollerV1alpha1().Foos(ns).List(context.Background(), opts)
+	//		},
+	//		WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
+	//			opts.FieldSelector = fieldSelector
+	//			return client.SamplecontrollerV1alpha1().Foos(ns).Watch(context.Background(), opts)
+	//		},
+	//	}
+	// }
+	ListWatch(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher
+}

From 3ca06c2692d5f29cfdbeecde7214f81ac99c2041 Mon Sep 17 00:00:00 2001
From: Garrybest <garrybest@foxmail.com>
Date: Mon, 29 Nov 2021 19:38:25 +0800
Subject: [PATCH 2/4] Reform Builder to adapt custom resource registry

Signed-off-by: Garrybest <garrybest@foxmail.com>
---
 internal/store/builder.go       | 116 +++++++++++++++++++++++++++-----
 pkg/builder/builder.go          |  33 ++++++++-
 pkg/builder/types/interfaces.go |  17 ++++-
 3 files changed, 147 insertions(+), 19 deletions(-)

diff --git a/internal/store/builder.go b/internal/store/builder.go
index aace8bad75..1564a0feab 100644
--- a/internal/store/builder.go
+++ b/internal/store/builder.go
@@ -43,6 +43,7 @@ import (
 	"k8s.io/klog/v2"
 
 	ksmtypes "k8s.io/kube-state-metrics/v2/pkg/builder/types"
+	"k8s.io/kube-state-metrics/v2/pkg/customresource"
 	generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
 	metricsstore "k8s.io/kube-state-metrics/v2/pkg/metrics_store"
 	"k8s.io/kube-state-metrics/v2/pkg/options"
@@ -57,21 +58,23 @@ var _ ksmtypes.BuilderInterface = &Builder{}
 // Builder helps to build store. It follows the builder pattern
 // (https://en.wikipedia.org/wiki/Builder_pattern).
 type Builder struct {
-	kubeClient            clientset.Interface
-	vpaClient             vpaclientset.Interface
-	namespaces            options.NamespaceList
-	namespaceFilter       string
-	ctx                   context.Context
-	enabledResources      []string
-	familyGeneratorFilter generator.FamilyGeneratorFilter
-	listWatchMetrics      *watch.ListWatchMetrics
-	shardingMetrics       *sharding.Metrics
-	shard                 int32
-	totalShards           int
-	buildStoresFunc       ksmtypes.BuildStoresFunc
-	allowAnnotationsList  map[string][]string
-	allowLabelsList       map[string][]string
-	useAPIServerCache     bool
+	kubeClient                    clientset.Interface
+	customResourceClients         map[string]interface{}
+	vpaClient                     vpaclientset.Interface
+	namespaces                    options.NamespaceList
+	namespaceFilter               string
+	ctx                           context.Context
+	enabledResources              []string
+	familyGeneratorFilter         generator.FamilyGeneratorFilter
+	listWatchMetrics              *watch.ListWatchMetrics
+	shardingMetrics               *sharding.Metrics
+	shard                         int32
+	totalShards                   int
+	buildStoresFunc               ksmtypes.BuildStoresFunc
+	buildCustomResourceStoresFunc ksmtypes.BuildCustomResourceStoresFunc
+	allowAnnotationsList          map[string][]string
+	allowLabelsList               map[string][]string
+	useAPIServerCache             bool
 }
 
 // NewBuilder returns a new builder.
@@ -134,6 +137,16 @@ func (b *Builder) WithVPAClient(c vpaclientset.Interface) {
 	b.vpaClient = c
 }
 
+// WithCustomResourceClients sets the customResourceClients property of a Builder.
+func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) {
+	b.customResourceClients = cs
+}
+
+// WithUsingAPIServerCache configures whether using APIServer cache or not.
+func (b *Builder) WithUsingAPIServerCache(u bool) {
+	b.useAPIServerCache = u
+}
+
 // WithFamilyGeneratorFilter configures the family generator filter which decides which
 // metrics are to be exposed by the store build by the Builder.
 func (b *Builder) WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter) {
@@ -141,9 +154,13 @@ func (b *Builder) WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter) {
 }
 
 // WithGenerateStoresFunc configures a custom generate store function
-func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc, u bool) {
+func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc) {
 	b.buildStoresFunc = f
-	b.useAPIServerCache = u
+}
+
+// WithGenerateCustomResourceStoresFunc configures a custom generate custom resource store function
+func (b *Builder) WithGenerateCustomResourceStoresFunc(f ksmtypes.BuildCustomResourceStoresFunc) {
+	b.buildCustomResourceStoresFunc = f
 }
 
 // DefaultGenerateStoresFunc returns default buildStores function
@@ -151,6 +168,30 @@ func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc {
 	return b.buildStores
 }
 
+// DefaultGenerateCustomResourceStoresFunc returns default buildCustomResourceStores function
+func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc {
+	return b.buildCustomResourceStores
+}
+
+// WithCustomResourceStoreFactories returns configures a custom resource stores factory
+func (b *Builder) WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory) {
+	for i := range fs {
+		f := fs[i]
+		if _, ok := availableStores[f.Name()]; ok {
+			klog.Warningf("The internal resource store named %s already exists and is overridden by a custom resource store with the same name, please make sure it meets your expectation", f.Name())
+		}
+		availableStores[f.Name()] = func(b *Builder) []cache.Store {
+			return b.buildCustomResourceStoresFunc(
+				f.Name(),
+				f.MetricFamilyGenerators(b.allowAnnotationsList[f.Name()], b.allowLabelsList[f.Name()]),
+				f.ExpectedType(),
+				f.ListWatch,
+				b.useAPIServerCache,
+			)
+		}
+	}
+}
+
 // WithAllowAnnotations configures which annotations can be returned for metrics
 func (b *Builder) WithAllowAnnotations(annotations map[string][]string) {
 	if len(annotations) > 0 {
@@ -414,6 +455,47 @@ func (b *Builder) buildStores(
 	return stores
 }
 
+// TODO(Garrybest): Merge `buildStores` and `buildCustomResourceStores`
+func (b *Builder) buildCustomResourceStores(resourceName string,
+	metricFamilies []generator.FamilyGenerator,
+	expectedType interface{},
+	listWatchFunc func(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher,
+	useAPIServerCache bool,
+) []cache.Store {
+	metricFamilies = generator.FilterFamilyGenerators(b.familyGeneratorFilter, metricFamilies)
+	composedMetricGenFuncs := generator.ComposeMetricGenFuncs(metricFamilies)
+	familyHeaders := generator.ExtractMetricFamilyHeaders(metricFamilies)
+
+	customResourceClient, ok := b.customResourceClients[resourceName]
+	if !ok {
+		klog.Warningf("Custom resource client %s does not exist", resourceName)
+		return []cache.Store{}
+	}
+
+	if b.namespaces.IsAllNamespaces() {
+		store := metricsstore.NewMetricsStore(
+			familyHeaders,
+			composedMetricGenFuncs,
+		)
+		listWatcher := listWatchFunc(customResourceClient, v1.NamespaceAll, b.namespaceFilter)
+		b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
+		return []cache.Store{store}
+	}
+
+	stores := make([]cache.Store, 0, len(b.namespaces))
+	for _, ns := range b.namespaces {
+		store := metricsstore.NewMetricsStore(
+			familyHeaders,
+			composedMetricGenFuncs,
+		)
+		listWatcher := listWatchFunc(customResourceClient, ns, b.namespaceFilter)
+		b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
+		stores = append(stores, store)
+	}
+
+	return stores
+}
+
 // startReflector starts a Kubernetes client-go reflector with the given
 // listWatcher and registers it with the given store.
 func (b *Builder) startReflector(
diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go
index cd28bf1a0a..1e2763c455 100644
--- a/pkg/builder/builder.go
+++ b/pkg/builder/builder.go
@@ -28,6 +28,7 @@ import (
 
 	internalstore "k8s.io/kube-state-metrics/v2/internal/store"
 	ksmtypes "k8s.io/kube-state-metrics/v2/pkg/builder/types"
+	"k8s.io/kube-state-metrics/v2/pkg/customresource"
 	metricsstore "k8s.io/kube-state-metrics/v2/pkg/metrics_store"
 	"k8s.io/kube-state-metrics/v2/pkg/options"
 )
@@ -81,12 +82,27 @@ func (b *Builder) WithVPAClient(c vpaclientset.Interface) {
 	b.internal.WithVPAClient(c)
 }
 
+// WithCustomResourceClients sets the customResourceClients property of a Builder.
+func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) {
+	b.internal.WithCustomResourceClients(cs)
+}
+
+// WithUsingAPIServerCache configures whether using APIServer cache or not.
+func (b *Builder) WithUsingAPIServerCache(u bool) {
+	b.internal.WithUsingAPIServerCache(u)
+}
+
 // WithFamilyGeneratorFilter configures the family generator filter which decides which
 // metrics are to be exposed by the store build by the Builder.
 func (b *Builder) WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter) {
 	b.internal.WithFamilyGeneratorFilter(l)
 }
 
+// WithAllowAnnotations configures which annotations can be returned for metrics
+func (b *Builder) WithAllowAnnotations(annotations map[string][]string) {
+	b.internal.WithAllowAnnotations(annotations)
+}
+
 // WithAllowLabels configures which labels can be returned for metrics
 func (b *Builder) WithAllowLabels(l map[string][]string) {
 	b.internal.WithAllowLabels(l)
@@ -94,7 +110,12 @@ func (b *Builder) WithAllowLabels(l map[string][]string) {
 
 // WithGenerateStoresFunc configures a custom generate store function
 func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc) {
-	b.internal.WithGenerateStoresFunc(f, false)
+	b.internal.WithGenerateStoresFunc(f)
+}
+
+// WithGenerateCustomResourceStoresFunc configures a custom generate custom resource store function
+func (b *Builder) WithGenerateCustomResourceStoresFunc(f ksmtypes.BuildCustomResourceStoresFunc) {
+	b.internal.WithGenerateCustomResourceStoresFunc(f)
 }
 
 // DefaultGenerateStoresFunc returns default buildStore function
@@ -102,6 +123,16 @@ func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc {
 	return b.internal.DefaultGenerateStoresFunc()
 }
 
+// DefaultGenerateCustomResourceStoresFunc returns default buildStores function
+func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc {
+	return b.internal.DefaultGenerateCustomResourceStoresFunc()
+}
+
+// WithCustomResourceStoreFactories returns configures a custom resource stores factory
+func (b *Builder) WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory) {
+	b.internal.WithCustomResourceStoreFactories(fs...)
+}
+
 // Build initializes and registers all enabled stores.
 // Returns metric writers.
 func (b *Builder) Build() []metricsstore.MetricsWriter {
diff --git a/pkg/builder/types/interfaces.go b/pkg/builder/types/interfaces.go
index 15995435ad..21b657a24f 100644
--- a/pkg/builder/types/interfaces.go
+++ b/pkg/builder/types/interfaces.go
@@ -26,6 +26,7 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
 
+	"k8s.io/kube-state-metrics/v2/pkg/customresource"
 	generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
 	"k8s.io/kube-state-metrics/v2/pkg/options"
 )
@@ -39,10 +40,16 @@ type BuilderInterface interface {
 	WithContext(ctx context.Context)
 	WithKubeClient(c clientset.Interface)
 	WithVPAClient(c vpaclientset.Interface)
+	WithCustomResourceClients(cs map[string]interface{})
+	WithUsingAPIServerCache(u bool)
 	WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter)
+	WithAllowAnnotations(a map[string][]string)
 	WithAllowLabels(l map[string][]string)
-	WithGenerateStoresFunc(f BuildStoresFunc, useAPIServerCache bool)
+	WithGenerateStoresFunc(f BuildStoresFunc)
+	WithGenerateCustomResourceStoresFunc(f BuildCustomResourceStoresFunc)
 	DefaultGenerateStoresFunc() BuildStoresFunc
+	DefaultGenerateCustomResourceStoresFunc() BuildCustomResourceStoresFunc
+	WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory)
 	Build() []metricsstore.MetricsWriter
 	BuildStores() [][]cache.Store
 }
@@ -54,6 +61,14 @@ type BuildStoresFunc func(metricFamilies []generator.FamilyGenerator,
 	useAPIServerCache bool,
 ) []cache.Store
 
+// BuildCustomResourceStoresFunc function signature that is used to return a list of custom resource cache.Store
+type BuildCustomResourceStoresFunc func(resourceName string,
+	metricFamilies []generator.FamilyGenerator,
+	expectedType interface{},
+	listWatchFunc func(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher,
+	useAPIServerCache bool,
+) []cache.Store
+
 // AllowDenyLister interface for AllowDeny lister that can allow or exclude metrics by there names
 type AllowDenyLister interface {
 	IsIncluded(string) bool

From 854bc7c36bf8db17da6e43e7c01b3e35123253bf Mon Sep 17 00:00:00 2001
From: Garrybest <garrybest@foxmail.com>
Date: Mon, 29 Nov 2021 23:01:22 +0800
Subject: [PATCH 3/4] Reform main function to adapt custom resource registry

Signed-off-by: Garrybest <garrybest@foxmail.com>
---
 main.go           | 269 +--------------------------------------
 pkg/app/server.go | 313 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 319 insertions(+), 263 deletions(-)
 create mode 100644 pkg/app/server.go

diff --git a/main.go b/main.go
index a1a15b3ef5..ff8e5ff0cd 100644
--- a/main.go
+++ b/main.go
@@ -19,65 +19,21 @@ package main
 import (
 	"context"
 	"fmt"
-	"net"
-	"net/http"
-	"net/http/pprof"
 	"os"
-	"strconv"
-	"time"
 
-	"github.com/oklog/run"
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/collectors"
-	"github.com/prometheus/client_golang/prometheus/promauto"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/common/version"
-	"github.com/prometheus/exporter-toolkit/web"
-	vpaclientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
-	clientset "k8s.io/client-go/kubernetes"
-	_ "k8s.io/client-go/plugin/pkg/client/auth"
-	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 
-	"k8s.io/kube-state-metrics/v2/internal/store"
-	"k8s.io/kube-state-metrics/v2/pkg/allowdenylist"
-	generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
-	"k8s.io/kube-state-metrics/v2/pkg/metricshandler"
-	"k8s.io/kube-state-metrics/v2/pkg/optin"
+	"k8s.io/kube-state-metrics/v2/pkg/app"
 	"k8s.io/kube-state-metrics/v2/pkg/options"
-	"k8s.io/kube-state-metrics/v2/pkg/util/proc"
 )
 
-const (
-	metricsPath = "/metrics"
-	healthzPath = "/healthz"
-)
-
-// promLogger implements promhttp.Logger
-type promLogger struct{}
-
-func (pl promLogger) Println(v ...interface{}) {
-	klog.Error(v...)
-}
-
-// promLogger implements the Logger interface
-func (pl promLogger) Log(v ...interface{}) error {
-	klog.Info(v...)
-	return nil
-}
-
 func main() {
 	opts := options.NewOptions()
 	opts.AddFlags()
 
-	promLogger := promLogger{}
-
-	ctx := context.Background()
-
-	err := opts.Parse()
-	if err != nil {
-		klog.Fatalf("Error: %s", err)
+	if err := opts.Parse(); err != nil {
+		klog.Fatalf("Parsing flag definitions error: %v", err)
 	}
 
 	if opts.Version {
@@ -89,222 +45,9 @@ func main() {
 		opts.Usage()
 		os.Exit(0)
 	}
-	storeBuilder := store.NewBuilder()
-
-	ksmMetricsRegistry := prometheus.NewRegistry()
-	ksmMetricsRegistry.MustRegister(version.NewCollector("kube_state_metrics"))
-	durationVec := promauto.With(ksmMetricsRegistry).NewHistogramVec(
-		prometheus.HistogramOpts{
-			Name:        "http_request_duration_seconds",
-			Help:        "A histogram of requests for kube-state-metrics metrics handler.",
-			Buckets:     prometheus.DefBuckets,
-			ConstLabels: prometheus.Labels{"handler": "metrics"},
-		}, []string{"method"},
-	)
-	storeBuilder.WithMetrics(ksmMetricsRegistry)
-
-	var resources []string
-	if len(opts.Resources) == 0 {
-		klog.Info("Using default resources")
-		resources = options.DefaultResources.AsSlice()
-	} else {
-		klog.Infof("Using resources %s", opts.Resources.String())
-		resources = opts.Resources.AsSlice()
-	}
-
-	if err := storeBuilder.WithEnabledResources(resources); err != nil {
-		klog.Fatalf("Failed to set up resources: %v", err)
-	}
-
-	namespaces := opts.Namespaces.GetNamespaces()
-	nsFieldSelector := namespaces.GetExcludeNSFieldSelector(opts.NamespacesDenylist)
-	storeBuilder.WithNamespaces(namespaces, nsFieldSelector)
-
-	allowDenyList, err := allowdenylist.New(opts.MetricAllowlist, opts.MetricDenylist)
-	if err != nil {
-		klog.Fatal(err)
-	}
-
-	err = allowDenyList.Parse()
-	if err != nil {
-		klog.Fatalf("error initializing the allowdeny list : %v", err)
-	}
-
-	klog.Infof("metric allow-denylisting: %v", allowDenyList.Status())
-
-	optInMetricFamilyFilter, err := optin.NewMetricFamilyFilter(opts.MetricOptInList)
-	if err != nil {
-		klog.Fatalf("error initializing the opt-in metric list : %v", err)
-	}
-
-	if optInMetricFamilyFilter.Count() > 0 {
-		klog.Infof("metrics which were opted into: %v", optInMetricFamilyFilter.Status())
-	}
-
-	storeBuilder.WithFamilyGeneratorFilter(generator.NewCompositeFamilyGeneratorFilter(
-		allowDenyList,
-		optInMetricFamilyFilter,
-	))
-
-	storeBuilder.WithGenerateStoresFunc(storeBuilder.DefaultGenerateStoresFunc(), opts.UseAPIServerCache)
-
-	proc.StartReaper()
-
-	kubeClient, vpaClient, err := createKubeClient(opts.Apiserver, opts.Kubeconfig)
-	if err != nil {
-		klog.Fatalf("Failed to create client: %v", err)
-	}
-	storeBuilder.WithKubeClient(kubeClient)
-	storeBuilder.WithVPAClient(vpaClient)
-	storeBuilder.WithSharding(opts.Shard, opts.TotalShards)
-	storeBuilder.WithAllowAnnotations(opts.AnnotationsAllowList)
-	storeBuilder.WithAllowLabels(opts.LabelsAllowList)
 
-	ksmMetricsRegistry.MustRegister(
-		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
-		collectors.NewGoCollector(),
-	)
-
-	var g run.Group
-
-	m := metricshandler.New(
-		opts,
-		kubeClient,
-		storeBuilder,
-		opts.EnableGZIPEncoding,
-	)
-	// Run MetricsHandler
-	{
-		ctxMetricsHandler, cancel := context.WithCancel(ctx)
-		g.Add(func() error {
-			return m.Run(ctxMetricsHandler)
-		}, func(error) {
-			cancel()
-		})
-	}
-
-	tlsConfig := opts.TLSConfig
-
-	telemetryMux := buildTelemetryServer(ksmMetricsRegistry)
-	telemetryListenAddress := net.JoinHostPort(opts.TelemetryHost, strconv.Itoa(opts.TelemetryPort))
-	telemetryServer := http.Server{Handler: telemetryMux, Addr: telemetryListenAddress}
-
-	metricsMux := buildMetricsServer(m, durationVec)
-	metricsServerListenAddress := net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port))
-	metricsServer := http.Server{Handler: metricsMux, Addr: metricsServerListenAddress}
-
-	// Run Telemetry server
-	{
-		g.Add(func() error {
-			klog.Infof("Starting kube-state-metrics self metrics server: %s", telemetryListenAddress)
-			return web.ListenAndServe(&telemetryServer, tlsConfig, promLogger)
-		}, func(error) {
-			ctxShutDown, cancel := context.WithTimeout(ctx, 3*time.Second)
-			defer cancel()
-			telemetryServer.Shutdown(ctxShutDown)
-		})
-	}
-	// Run Metrics server
-	{
-		g.Add(func() error {
-			klog.Infof("Starting metrics server: %s", metricsServerListenAddress)
-			return web.ListenAndServe(&metricsServer, tlsConfig, promLogger)
-		}, func(error) {
-			ctxShutDown, cancel := context.WithTimeout(ctx, 3*time.Second)
-			defer cancel()
-			metricsServer.Shutdown(ctxShutDown)
-		})
-	}
-
-	if err := g.Run(); err != nil {
-		klog.Fatalf("RunGroup Error: %v", err)
-	}
-	klog.Info("Exiting")
-}
-
-func createKubeClient(apiserver string, kubeconfig string) (clientset.Interface, vpaclientset.Interface, error) {
-	config, err := clientcmd.BuildConfigFromFlags(apiserver, kubeconfig)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	config.UserAgent = version.Version
-	config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json"
-	config.ContentType = "application/vnd.kubernetes.protobuf"
-
-	kubeClient, err := clientset.NewForConfig(config)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	vpaClient, err := vpaclientset.NewForConfig(config)
-	if err != nil {
-		return nil, nil, err
-	}
-	// Informers don't seem to do a good job logging error messages when it
-	// can't reach the server, making debugging hard. This makes it easier to
-	// figure out if apiserver is configured incorrectly.
-	klog.Infof("Testing communication with server")
-	v, err := kubeClient.Discovery().ServerVersion()
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "error while trying to communicate with apiserver")
+	ctx := context.Background()
+	if err := app.RunKubeStateMetrics(ctx, opts); err != nil {
+		klog.Fatalf("Failed to run kube-state-metrics: %v", err)
 	}
-	klog.Infof("Running with Kubernetes cluster version: v%s.%s. git version: %s. git tree state: %s. commit: %s. platform: %s",
-		v.Major, v.Minor, v.GitVersion, v.GitTreeState, v.GitCommit, v.Platform)
-	klog.Infof("Communication with server successful")
-
-	return kubeClient, vpaClient, nil
-}
-
-func buildTelemetryServer(registry prometheus.Gatherer) *http.ServeMux {
-	mux := http.NewServeMux()
-
-	// Add metricsPath
-	mux.Handle(metricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{ErrorLog: promLogger{}}))
-	// Add index
-	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		w.Write([]byte(`<html>
-             <head><title>Kube-State-Metrics Metrics Server</title></head>
-             <body>
-             <h1>Kube-State-Metrics Metrics</h1>
-			 <ul>
-             <li><a href='` + metricsPath + `'>metrics</a></li>
-			 </ul>
-             </body>
-             </html>`))
-	})
-	return mux
-}
-
-func buildMetricsServer(m *metricshandler.MetricsHandler, durationObserver prometheus.ObserverVec) *http.ServeMux {
-	mux := http.NewServeMux()
-
-	// TODO: This doesn't belong into serveMetrics
-	mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
-	mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
-	mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
-	mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
-	mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
-
-	mux.Handle(metricsPath, promhttp.InstrumentHandlerDuration(durationObserver, m))
-
-	// Add healthzPath
-	mux.HandleFunc(healthzPath, func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(http.StatusOK)
-		w.Write([]byte(http.StatusText(http.StatusOK)))
-	})
-	// Add index
-	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
-		w.Write([]byte(`<html>
-             <head><title>Kube Metrics Server</title></head>
-             <body>
-             <h1>Kube Metrics</h1>
-			 <ul>
-             <li><a href='` + metricsPath + `'>metrics</a></li>
-             <li><a href='` + healthzPath + `'>healthz</a></li>
-			 </ul>
-             </body>
-             </html>`))
-	})
-	return mux
 }
diff --git a/pkg/app/server.go b/pkg/app/server.go
new file mode 100644
index 0000000000..d8205de28f
--- /dev/null
+++ b/pkg/app/server.go
@@ -0,0 +1,313 @@
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/pprof"
+	"strconv"
+	"time"
+
+	"github.com/oklog/run"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/prometheus/common/version"
+	"github.com/prometheus/exporter-toolkit/web"
+	vpaclientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
+	clientset "k8s.io/client-go/kubernetes"
+	_ "k8s.io/client-go/plugin/pkg/client/auth" // Initialize common client auth plugins.
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog/v2"
+
+	"k8s.io/kube-state-metrics/v2/internal/store"
+	"k8s.io/kube-state-metrics/v2/pkg/allowdenylist"
+	"k8s.io/kube-state-metrics/v2/pkg/customresource"
+	generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
+	"k8s.io/kube-state-metrics/v2/pkg/metricshandler"
+	"k8s.io/kube-state-metrics/v2/pkg/optin"
+	"k8s.io/kube-state-metrics/v2/pkg/options"
+	"k8s.io/kube-state-metrics/v2/pkg/util/proc"
+)
+
+const (
+	metricsPath = "/metrics"
+	healthzPath = "/healthz"
+)
+
+// promLogger implements promhttp.Logger
+type promLogger struct{}
+
+func (pl promLogger) Println(v ...interface{}) {
+	klog.Error(v...)
+}
+
+// promLogger implements the Logger interface
+func (pl promLogger) Log(v ...interface{}) error {
+	klog.Info(v...)
+	return nil
+}
+
+// RunKubeStateMetrics will build and run the kube-state-metrics.
+// Any out-of-tree custom resource metrics could be registered by newing a registry factory
+// which implements customresource.RegistryFactory and pass all factories into this function.
+func RunKubeStateMetrics(ctx context.Context, opts *options.Options, factories ...customresource.RegistryFactory) error {
+	promLogger := promLogger{}
+
+	storeBuilder := store.NewBuilder()
+	storeBuilder.WithCustomResourceStoreFactories(factories...)
+
+	ksmMetricsRegistry := prometheus.NewRegistry()
+	ksmMetricsRegistry.MustRegister(version.NewCollector("kube_state_metrics"))
+	durationVec := promauto.With(ksmMetricsRegistry).NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:        "http_request_duration_seconds",
+			Help:        "A histogram of requests for kube-state-metrics metrics handler.",
+			Buckets:     prometheus.DefBuckets,
+			ConstLabels: prometheus.Labels{"handler": "metrics"},
+		}, []string{"method"},
+	)
+	storeBuilder.WithMetrics(ksmMetricsRegistry)
+
+	var resources []string
+	if len(opts.Resources) == 0 {
+		klog.Info("Using default resources")
+		resources = options.DefaultResources.AsSlice()
+		// enable custom resource
+		for _, factory := range factories {
+			resources = append(resources, factory.Name())
+		}
+	} else {
+		klog.Infof("Using resources %s", opts.Resources.String())
+		resources = opts.Resources.AsSlice()
+	}
+
+	if err := storeBuilder.WithEnabledResources(resources); err != nil {
+		return fmt.Errorf("failed to set up resources: %v", err)
+	}
+
+	namespaces := opts.Namespaces.GetNamespaces()
+	nsFieldSelector := namespaces.GetExcludeNSFieldSelector(opts.NamespacesDenylist)
+	storeBuilder.WithNamespaces(namespaces, nsFieldSelector)
+
+	allowDenyList, err := allowdenylist.New(opts.MetricAllowlist, opts.MetricDenylist)
+	if err != nil {
+		return err
+	}
+
+	err = allowDenyList.Parse()
+	if err != nil {
+		return fmt.Errorf("error initializing the allowdeny list: %v", err)
+	}
+
+	klog.Infof("Metric allow-denylisting: %v", allowDenyList.Status())
+
+	optInMetricFamilyFilter, err := optin.NewMetricFamilyFilter(opts.MetricOptInList)
+	if err != nil {
+		return fmt.Errorf("error initializing the opt-in metric list: %v", err)
+	}
+
+	if optInMetricFamilyFilter.Count() > 0 {
+		klog.Infof("Metrics which were opted into: %v", optInMetricFamilyFilter.Status())
+	}
+
+	storeBuilder.WithFamilyGeneratorFilter(generator.NewCompositeFamilyGeneratorFilter(
+		allowDenyList,
+		optInMetricFamilyFilter,
+	))
+
+	storeBuilder.WithUsingAPIServerCache(opts.UseAPIServerCache)
+	storeBuilder.WithGenerateStoresFunc(storeBuilder.DefaultGenerateStoresFunc())
+	storeBuilder.WithGenerateCustomResourceStoresFunc(storeBuilder.DefaultGenerateCustomResourceStoresFunc())
+
+	proc.StartReaper()
+
+	kubeClient, vpaClient, customResourceClients, err := createKubeClient(opts.Apiserver, opts.Kubeconfig, factories...)
+	if err != nil {
+		return fmt.Errorf("failed to create client: %v", err)
+	}
+	storeBuilder.WithKubeClient(kubeClient)
+	storeBuilder.WithVPAClient(vpaClient)
+	storeBuilder.WithCustomResourceClients(customResourceClients)
+	storeBuilder.WithSharding(opts.Shard, opts.TotalShards)
+	storeBuilder.WithAllowAnnotations(opts.AnnotationsAllowList)
+	storeBuilder.WithAllowLabels(opts.LabelsAllowList)
+
+	ksmMetricsRegistry.MustRegister(
+		collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+		collectors.NewGoCollector(),
+	)
+
+	var g run.Group
+
+	m := metricshandler.New(
+		opts,
+		kubeClient,
+		storeBuilder,
+		opts.EnableGZIPEncoding,
+	)
+	// Run MetricsHandler
+	{
+		ctxMetricsHandler, cancel := context.WithCancel(ctx)
+		g.Add(func() error {
+			return m.Run(ctxMetricsHandler)
+		}, func(error) {
+			cancel()
+		})
+	}
+
+	tlsConfig := opts.TLSConfig
+
+	telemetryMux := buildTelemetryServer(ksmMetricsRegistry)
+	telemetryListenAddress := net.JoinHostPort(opts.TelemetryHost, strconv.Itoa(opts.TelemetryPort))
+	telemetryServer := http.Server{Handler: telemetryMux, Addr: telemetryListenAddress}
+
+	metricsMux := buildMetricsServer(m, durationVec)
+	metricsServerListenAddress := net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port))
+	metricsServer := http.Server{Handler: metricsMux, Addr: metricsServerListenAddress}
+
+	// Run Telemetry server
+	{
+		g.Add(func() error {
+			klog.Infof("Starting kube-state-metrics self metrics server: %s", telemetryListenAddress)
+			return web.ListenAndServe(&telemetryServer, tlsConfig, promLogger)
+		}, func(error) {
+			ctxShutDown, cancel := context.WithTimeout(ctx, 3*time.Second)
+			defer cancel()
+			telemetryServer.Shutdown(ctxShutDown)
+		})
+	}
+	// Run Metrics server
+	{
+		g.Add(func() error {
+			klog.Infof("Starting metrics server: %s", metricsServerListenAddress)
+			return web.ListenAndServe(&metricsServer, tlsConfig, promLogger)
+		}, func(error) {
+			ctxShutDown, cancel := context.WithTimeout(ctx, 3*time.Second)
+			defer cancel()
+			metricsServer.Shutdown(ctxShutDown)
+		})
+	}
+
+	if err := g.Run(); err != nil {
+		return fmt.Errorf("run server group error: %v", err)
+	}
+	klog.Info("Exiting")
+	return nil
+}
+
+func createKubeClient(apiserver string, kubeconfig string, factories ...customresource.RegistryFactory) (clientset.Interface, vpaclientset.Interface, map[string]interface{}, error) {
+	config, err := clientcmd.BuildConfigFromFlags(apiserver, kubeconfig)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	config.UserAgent = version.Version
+	config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json"
+	config.ContentType = "application/vnd.kubernetes.protobuf"
+
+	kubeClient, err := clientset.NewForConfig(config)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	vpaClient, err := vpaclientset.NewForConfig(config)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	customResourceClients := make(map[string]interface{}, len(factories))
+	for _, f := range factories {
+		customResourceClient, err := f.CreateClient(config)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+		customResourceClients[f.Name()] = customResourceClient
+	}
+
+	// Informers don't seem to do a good job logging error messages when it
+	// can't reach the server, making debugging hard. This makes it easier to
+	// figure out if apiserver is configured incorrectly.
+	klog.Infof("Testing communication with server")
+	v, err := kubeClient.Discovery().ServerVersion()
+	if err != nil {
+		return nil, nil, nil, errors.Wrap(err, "error while trying to communicate with apiserver")
+	}
+	klog.Infof("Running with Kubernetes cluster version: v%s.%s. git version: %s. git tree state: %s. commit: %s. platform: %s",
+		v.Major, v.Minor, v.GitVersion, v.GitTreeState, v.GitCommit, v.Platform)
+	klog.Infof("Communication with server successful")
+
+	return kubeClient, vpaClient, customResourceClients, nil
+}
+
+func buildTelemetryServer(registry prometheus.Gatherer) *http.ServeMux {
+	mux := http.NewServeMux()
+
+	// Add metricsPath
+	mux.Handle(metricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{ErrorLog: promLogger{}}))
+	// Add index
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`<html>
+             <head><title>Kube-State-Metrics Metrics Server</title></head>
+             <body>
+             <h1>Kube-State-Metrics Metrics</h1>
+			 <ul>
+             <li><a href='` + metricsPath + `'>metrics</a></li>
+			 </ul>
+             </body>
+             </html>`))
+	})
+	return mux
+}
+
+func buildMetricsServer(m *metricshandler.MetricsHandler, durationObserver prometheus.ObserverVec) *http.ServeMux {
+	mux := http.NewServeMux()
+
+	// TODO: This doesn't belong into serveMetrics
+	mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
+	mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
+	mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
+	mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
+	mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
+
+	mux.Handle(metricsPath, promhttp.InstrumentHandlerDuration(durationObserver, m))
+
+	// Add healthzPath
+	mux.HandleFunc(healthzPath, func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Write([]byte(http.StatusText(http.StatusOK)))
+	})
+	// Add index
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte(`<html>
+             <head><title>Kube Metrics Server</title></head>
+             <body>
+             <h1>Kube Metrics</h1>
+			 <ul>
+             <li><a href='` + metricsPath + `'>metrics</a></li>
+             <li><a href='` + healthzPath + `'>healthz</a></li>
+			 </ul>
+             </body>
+             </html>`))
+	})
+	return mux
+}

From acd5e74f50eb3c6771f7100c3d26d27799534011 Mon Sep 17 00:00:00 2001
From: Garrybest <garrybest@foxmail.com>
Date: Mon, 29 Nov 2021 23:01:42 +0800
Subject: [PATCH 4/4] Add a custom resource test based on foo object in
 samplecontroller

Signed-off-by: Garrybest <garrybest@foxmail.com>
---
 go.mod                                 |   1 +
 go.sum                                 |  11 ++
 main_test.go => pkg/app/server_test.go | 251 ++++++++++++++++++++++++-
 3 files changed, 255 insertions(+), 8 deletions(-)
 rename main_test.go => pkg/app/server_test.go (77%)

diff --git a/go.mod b/go.mod
index a915dc7af5..db764e5206 100644
--- a/go.mod
+++ b/go.mod
@@ -21,6 +21,7 @@ require (
 	k8s.io/autoscaler/vertical-pod-autoscaler v0.9.2
 	k8s.io/client-go v0.23.0
 	k8s.io/klog/v2 v2.30.0
+	k8s.io/sample-controller v0.23.0
 )
 
 require (
diff --git a/go.sum b/go.sum
index 95146b968c..7232a7f1e9 100644
--- a/go.sum
+++ b/go.sum
@@ -148,11 +148,13 @@ github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34
 github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
 github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
 github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -249,6 +251,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -280,6 +283,7 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
@@ -379,6 +383,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -478,6 +483,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY=
 golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -565,6 +571,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0=
 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -641,6 +648,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM=
 golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
 golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -793,6 +801,7 @@ k8s.io/client-go v0.18.3/go.mod h1:4a/dpQEvzAhT1BbuWW09qvIaGw6Gbu1gZYiQZIi1DMw=
 k8s.io/client-go v0.23.0 h1:vcsOqyPq7XV3QmQRCBH/t9BICJM9Q1M18qahjv+rebY=
 k8s.io/client-go v0.23.0/go.mod h1:hrDnpnK1mSr65lHHcUuIZIXDgEbzc7/683c6hyG4jTA=
 k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
+k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE=
 k8s.io/component-base v0.18.3/go.mod h1:bp5GzGR0aGkYEfTj+eTY0AN/vXTgkJdQXjNTTVUaa3k=
 k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
@@ -809,6 +818,8 @@ k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C
 k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4=
 k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=
 k8s.io/metrics v0.18.3/go.mod h1:TkuJE3ezDZ1ym8pYkZoEzJB7HDiFE7qxl+EmExEBoPA=
+k8s.io/sample-controller v0.23.0 h1:Zwxi+BQxEQDg63jGzxrLUbWBp+Qkqse/web1nkpSG9g=
+k8s.io/sample-controller v0.23.0/go.mod h1:8a1Cgok9A5JRa1rJgg9AQKrOF0hqwbaHt/wcndZ6fmY=
 k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
 k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs=
diff --git a/main_test.go b/pkg/app/server_test.go
similarity index 77%
rename from main_test.go
rename to pkg/app/server_test.go
index fc21c38c0f..53cd0356dd 100644
--- a/main_test.go
+++ b/pkg/app/server_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package main
+package app
 
 import (
 	"bytes"
@@ -28,17 +28,24 @@ import (
 	"testing"
 	"time"
 
-	generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
-
 	"github.com/prometheus/client_golang/prometheus"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/client-go/kubernetes/fake"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/cache"
+	samplev1alpha1 "k8s.io/sample-controller/pkg/apis/samplecontroller/v1alpha1"
+	samplefake "k8s.io/sample-controller/pkg/generated/clientset/versioned/fake"
 
 	"k8s.io/kube-state-metrics/v2/internal/store"
 	"k8s.io/kube-state-metrics/v2/pkg/allowdenylist"
+	"k8s.io/kube-state-metrics/v2/pkg/customresource"
+	"k8s.io/kube-state-metrics/v2/pkg/metric"
+	generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
 	"k8s.io/kube-state-metrics/v2/pkg/metricshandler"
 	"k8s.io/kube-state-metrics/v2/pkg/options"
 )
@@ -69,7 +76,7 @@ func BenchmarkKubeStateMetrics(b *testing.B) {
 	builder.WithSharding(0, 1)
 	builder.WithContext(ctx)
 	builder.WithNamespaces(options.DefaultNamespaces, "")
-	builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc(), false)
+	builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc())
 
 	allowDenyListFilter, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{})
 	if err != nil {
@@ -138,7 +145,7 @@ func TestFullScrapeCycle(t *testing.T) {
 	builder.WithEnabledResources(options.DefaultResources.AsSlice())
 	builder.WithKubeClient(kubeClient)
 	builder.WithNamespaces(options.DefaultNamespaces, "")
-	builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc(), false)
+	builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc())
 
 	l, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{})
 	if err != nil {
@@ -407,7 +414,7 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
 	unshardedBuilder.WithNamespaces(options.DefaultNamespaces, "")
 	unshardedBuilder.WithFamilyGeneratorFilter(l)
 	unshardedBuilder.WithAllowLabels(map[string][]string{})
-	unshardedBuilder.WithGenerateStoresFunc(unshardedBuilder.DefaultGenerateStoresFunc(), false)
+	unshardedBuilder.WithGenerateStoresFunc(unshardedBuilder.DefaultGenerateStoresFunc())
 
 	unshardedHandler := metricshandler.New(&options.Options{}, kubeClient, unshardedBuilder, false)
 	unshardedHandler.ConfigureSharding(ctx, 0, 1)
@@ -420,7 +427,7 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
 	shardedBuilder1.WithNamespaces(options.DefaultNamespaces, "")
 	shardedBuilder1.WithFamilyGeneratorFilter(l)
 	shardedBuilder1.WithAllowLabels(map[string][]string{})
-	shardedBuilder1.WithGenerateStoresFunc(shardedBuilder1.DefaultGenerateStoresFunc(), false)
+	shardedBuilder1.WithGenerateStoresFunc(shardedBuilder1.DefaultGenerateStoresFunc())
 
 	shardedHandler1 := metricshandler.New(&options.Options{}, kubeClient, shardedBuilder1, false)
 	shardedHandler1.ConfigureSharding(ctx, 0, 2)
@@ -433,7 +440,7 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
 	shardedBuilder2.WithNamespaces(options.DefaultNamespaces, "")
 	shardedBuilder2.WithFamilyGeneratorFilter(l)
 	shardedBuilder2.WithAllowLabels(map[string][]string{})
-	shardedBuilder2.WithGenerateStoresFunc(shardedBuilder2.DefaultGenerateStoresFunc(), false)
+	shardedBuilder2.WithGenerateStoresFunc(shardedBuilder2.DefaultGenerateStoresFunc())
 
 	shardedHandler2 := metricshandler.New(&options.Options{}, kubeClient, shardedBuilder2, false)
 	shardedHandler2.ConfigureSharding(ctx, 1, 2)
@@ -541,6 +548,121 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
 	}
 }
 
+// TestCustomResourceExtension is a simple smoke test covering the custom resource metrics collection.
+// We use custom resource object samplev1alpha1.Foo in kubernetes/sample-controller as an example.
+func TestCustomResourceExtension(t *testing.T) {
+	kubeClient := fake.NewSimpleClientset()
+	factories := []customresource.RegistryFactory{new(fooFactory)}
+	resources := options.DefaultResources.AsSlice()
+	customResourceClients := make(map[string]interface{}, len(factories))
+	// enable custom resource
+	for _, f := range factories {
+		resources = append(resources, f.Name())
+		customResourceClient, err := f.CreateClient(nil)
+		if err != nil {
+			t.Fatalf("Failed to create customResourceClient for foo: %v", err)
+		}
+		customResourceClients[f.Name()] = customResourceClient
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	reg := prometheus.NewRegistry()
+	builder := store.NewBuilder()
+	builder.WithCustomResourceStoreFactories(factories...)
+	builder.WithMetrics(reg)
+	builder.WithEnabledResources(resources)
+	builder.WithKubeClient(kubeClient)
+	builder.WithCustomResourceClients(customResourceClients)
+	builder.WithNamespaces(options.DefaultNamespaces, "")
+	builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc())
+	builder.WithGenerateCustomResourceStoresFunc(builder.DefaultGenerateCustomResourceStoresFunc())
+
+	l, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	builder.WithFamilyGeneratorFilter(l)
+	builder.WithAllowLabels(map[string][]string{
+		"kube_foo_labels": {
+			"namespace",
+			"foo",
+			"uid",
+		},
+	})
+
+	handler := metricshandler.New(&options.Options{}, kubeClient, builder, false)
+	handler.ConfigureSharding(ctx, 0, 1)
+
+	// Wait for caches to fill
+	time.Sleep(time.Second)
+
+	req := httptest.NewRequest("GET", "http://localhost:8080/metrics", nil)
+
+	w := httptest.NewRecorder()
+	handler.ServeHTTP(w, req)
+
+	resp := w.Result()
+	if resp.StatusCode != 200 {
+		t.Fatalf("expected 200 status code but got %v", resp.StatusCode)
+	}
+
+	body, _ := io.ReadAll(resp.Body)
+
+	expected := `# HELP kube_foo_spec_replicas Number of desired replicas for a foo.
+# HELP kube_foo_status_replicas_available The number of available replicas per foo.
+# TYPE kube_foo_spec_replicas gauge
+# TYPE kube_foo_status_replicas_available gauge
+kube_foo_spec_replicas{namespace="default",foo="foo0"} 0
+kube_foo_spec_replicas{namespace="default",foo="foo1"} 1
+kube_foo_spec_replicas{namespace="default",foo="foo2"} 2
+kube_foo_spec_replicas{namespace="default",foo="foo3"} 3
+kube_foo_spec_replicas{namespace="default",foo="foo4"} 4
+kube_foo_spec_replicas{namespace="default",foo="foo5"} 5
+kube_foo_spec_replicas{namespace="default",foo="foo6"} 6
+kube_foo_spec_replicas{namespace="default",foo="foo7"} 7
+kube_foo_spec_replicas{namespace="default",foo="foo8"} 8
+kube_foo_spec_replicas{namespace="default",foo="foo9"} 9
+kube_foo_status_replicas_available{namespace="default",foo="foo0"} 0
+kube_foo_status_replicas_available{namespace="default",foo="foo1"} 1
+kube_foo_status_replicas_available{namespace="default",foo="foo2"} 2
+kube_foo_status_replicas_available{namespace="default",foo="foo3"} 3
+kube_foo_status_replicas_available{namespace="default",foo="foo5"} 5
+kube_foo_status_replicas_available{namespace="default",foo="foo6"} 6
+kube_foo_status_replicas_available{namespace="default",foo="foo7"} 7
+kube_foo_status_replicas_available{namespace="default",foo="foo8"} 8
+kube_foo_status_replicas_available{namespace="default",foo="foo4"} 4
+kube_foo_status_replicas_available{namespace="default",foo="foo9"} 9
+`
+
+	expectedSplit := strings.Split(strings.TrimSpace(expected), "\n")
+	sort.Strings(expectedSplit)
+
+	gotSplit := strings.Split(strings.TrimSpace(string(body)), "\n")
+
+	gotFiltered := []string{}
+	for _, l := range gotSplit {
+		if strings.Contains(l, "kube_foo_") {
+			gotFiltered = append(gotFiltered, l)
+		}
+	}
+
+	sort.Strings(gotFiltered)
+
+	if len(expectedSplit) != len(gotFiltered) {
+		fmt.Println(len(expectedSplit))
+		fmt.Println(len(gotFiltered))
+		t.Fatalf("expected different output length, expected \n\n%s\n\ngot\n\n%s", expected, strings.Join(gotFiltered, "\n"))
+	}
+
+	for i := 0; i < len(expectedSplit); i++ {
+		if expectedSplit[i] != gotFiltered[i] {
+			t.Fatalf("expected:\n\n%v\n, but got:\n\n%v", expectedSplit[i], gotFiltered[i])
+		}
+	}
+}
+
 func injectFixtures(client *fake.Clientset, multiplier int) error {
 	creators := []func(*fake.Clientset, int) error{
 		configMap,
@@ -673,3 +795,116 @@ func pod(client *fake.Clientset, index int) error {
 	_, err := client.CoreV1().Pods(metav1.NamespaceDefault).Create(context.TODO(), &pod, metav1.CreateOptions{})
 	return err
 }
+
+func foo(client *samplefake.Clientset, index int) error {
+	i := strconv.Itoa(index)
+	desiredReplicas := int32(index)
+
+	foo := samplev1alpha1.Foo{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:              "foo" + i,
+			CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)},
+			UID:               types.UID("abc-" + i),
+		},
+		Spec: samplev1alpha1.FooSpec{
+			DeploymentName: "foo" + i,
+			Replicas:       &desiredReplicas,
+		},
+		Status: samplev1alpha1.FooStatus{
+			AvailableReplicas: desiredReplicas,
+		},
+	}
+
+	_, err := client.SamplecontrollerV1alpha1().Foos(metav1.NamespaceDefault).Create(context.TODO(), &foo, metav1.CreateOptions{})
+	return err
+}
+
+var (
+	descFooLabelsDefaultLabels = []string{"namespace", "foo"}
+)
+
+type fooFactory struct{}
+
+func (f *fooFactory) Name() string {
+	return "foos"
+}
+
+// CreateClient use fake client set to establish 10 foos.
+func (f *fooFactory) CreateClient(cfg *rest.Config) (interface{}, error) {
+	fooClient := samplefake.NewSimpleClientset()
+	for i := 0; i < 10; i++ {
+		err := foo(fooClient, i)
+		if err != nil {
+			return nil, fmt.Errorf("failed to insert sample pod %v", err)
+		}
+	}
+	return fooClient, nil
+}
+
+func (f *fooFactory) MetricFamilyGenerators(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator {
+	return []generator.FamilyGenerator{
+		*generator.NewFamilyGenerator(
+			"kube_foo_spec_replicas",
+			"Number of desired replicas for a foo.",
+			metric.Gauge,
+			"",
+			wrapFooFunc(func(f *samplev1alpha1.Foo) *metric.Family {
+				return &metric.Family{
+					Metrics: []*metric.Metric{
+						{
+							Value: float64(*f.Spec.Replicas),
+						},
+					},
+				}
+			}),
+		),
+		*generator.NewFamilyGenerator(
+			"kube_foo_status_replicas_available",
+			"The number of available replicas per foo.",
+			metric.Gauge,
+			"",
+			wrapFooFunc(func(f *samplev1alpha1.Foo) *metric.Family {
+				return &metric.Family{
+					Metrics: []*metric.Metric{
+						{
+							Value: float64(f.Status.AvailableReplicas),
+						},
+					},
+				}
+			}),
+		),
+	}
+}
+
+func wrapFooFunc(f func(*samplev1alpha1.Foo) *metric.Family) func(interface{}) *metric.Family {
+	return func(obj interface{}) *metric.Family {
+		foo := obj.(*samplev1alpha1.Foo)
+
+		metricFamily := f(foo)
+
+		for _, m := range metricFamily.Metrics {
+			m.LabelKeys = append(descFooLabelsDefaultLabels, m.LabelKeys...)
+			m.LabelValues = append([]string{foo.Namespace, foo.Name}, m.LabelValues...)
+		}
+
+		return metricFamily
+	}
+}
+
+func (f *fooFactory) ExpectedType() interface{} {
+	return &samplev1alpha1.Foo{}
+}
+
+func (f *fooFactory) ListWatch(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher {
+	client := customResourceClient.(*samplefake.Clientset)
+	return &cache.ListWatch{
+		ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
+			opts.FieldSelector = fieldSelector
+			return client.SamplecontrollerV1alpha1().Foos(ns).List(context.Background(), opts)
+		},
+		WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
+			opts.FieldSelector = fieldSelector
+			return client.SamplecontrollerV1alpha1().Foos(ns).Watch(context.Background(), opts)
+		},
+	}
+}