diff --git a/go.mod b/go.mod
index a915dc7af5..db764e5206 100644
--- a/go.mod
+++ b/go.mod
@@ -21,6 +21,7 @@ require (
k8s.io/autoscaler/vertical-pod-autoscaler v0.9.2
k8s.io/client-go v0.23.0
k8s.io/klog/v2 v2.30.0
+ k8s.io/sample-controller v0.23.0
)
require (
diff --git a/go.sum b/go.sum
index 95146b968c..7232a7f1e9 100644
--- a/go.sum
+++ b/go.sum
@@ -148,11 +148,13 @@ github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34
github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc=
github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
+github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg=
github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -249,6 +251,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q=
github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -280,6 +283,7 @@ github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
@@ -379,6 +383,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -478,6 +483,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f h1:OfiFi4JbukWwe3lzw+xunroH1mnC1e2Gy5cxNJApiSY=
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -565,6 +571,7 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0=
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -641,6 +648,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
+golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM=
golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w=
golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -793,6 +801,7 @@ k8s.io/client-go v0.18.3/go.mod h1:4a/dpQEvzAhT1BbuWW09qvIaGw6Gbu1gZYiQZIi1DMw=
k8s.io/client-go v0.23.0 h1:vcsOqyPq7XV3QmQRCBH/t9BICJM9Q1M18qahjv+rebY=
k8s.io/client-go v0.23.0/go.mod h1:hrDnpnK1mSr65lHHcUuIZIXDgEbzc7/683c6hyG4jTA=
k8s.io/code-generator v0.18.3/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c=
+k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE=
k8s.io/component-base v0.18.3/go.mod h1:bp5GzGR0aGkYEfTj+eTY0AN/vXTgkJdQXjNTTVUaa3k=
k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
@@ -809,6 +818,8 @@ k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 h1:E3J9oCLlaobFUqsjG9DfKbP2BmgwBL2p7pn0A3dG9W4=
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=
k8s.io/metrics v0.18.3/go.mod h1:TkuJE3ezDZ1ym8pYkZoEzJB7HDiFE7qxl+EmExEBoPA=
+k8s.io/sample-controller v0.23.0 h1:Zwxi+BQxEQDg63jGzxrLUbWBp+Qkqse/web1nkpSG9g=
+k8s.io/sample-controller v0.23.0/go.mod h1:8a1Cgok9A5JRa1rJgg9AQKrOF0hqwbaHt/wcndZ6fmY=
k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew=
k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b h1:wxEMGetGMur3J1xuGLQY7GEQYg9bZxKn3tKo5k/eYcs=
diff --git a/internal/store/builder.go b/internal/store/builder.go
index aace8bad75..1564a0feab 100644
--- a/internal/store/builder.go
+++ b/internal/store/builder.go
@@ -43,6 +43,7 @@ import (
"k8s.io/klog/v2"
ksmtypes "k8s.io/kube-state-metrics/v2/pkg/builder/types"
+ "k8s.io/kube-state-metrics/v2/pkg/customresource"
generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
metricsstore "k8s.io/kube-state-metrics/v2/pkg/metrics_store"
"k8s.io/kube-state-metrics/v2/pkg/options"
@@ -57,21 +58,23 @@ var _ ksmtypes.BuilderInterface = &Builder{}
// Builder helps to build store. It follows the builder pattern
// (https://en.wikipedia.org/wiki/Builder_pattern).
type Builder struct {
- kubeClient clientset.Interface
- vpaClient vpaclientset.Interface
- namespaces options.NamespaceList
- namespaceFilter string
- ctx context.Context
- enabledResources []string
- familyGeneratorFilter generator.FamilyGeneratorFilter
- listWatchMetrics *watch.ListWatchMetrics
- shardingMetrics *sharding.Metrics
- shard int32
- totalShards int
- buildStoresFunc ksmtypes.BuildStoresFunc
- allowAnnotationsList map[string][]string
- allowLabelsList map[string][]string
- useAPIServerCache bool
+ kubeClient clientset.Interface
+ customResourceClients map[string]interface{}
+ vpaClient vpaclientset.Interface
+ namespaces options.NamespaceList
+ namespaceFilter string
+ ctx context.Context
+ enabledResources []string
+ familyGeneratorFilter generator.FamilyGeneratorFilter
+ listWatchMetrics *watch.ListWatchMetrics
+ shardingMetrics *sharding.Metrics
+ shard int32
+ totalShards int
+ buildStoresFunc ksmtypes.BuildStoresFunc
+ buildCustomResourceStoresFunc ksmtypes.BuildCustomResourceStoresFunc
+ allowAnnotationsList map[string][]string
+ allowLabelsList map[string][]string
+ useAPIServerCache bool
}
// NewBuilder returns a new builder.
@@ -134,6 +137,16 @@ func (b *Builder) WithVPAClient(c vpaclientset.Interface) {
b.vpaClient = c
}
+// WithCustomResourceClients sets the customResourceClients property of a Builder.
+func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) {
+ b.customResourceClients = cs
+}
+
+// WithUsingAPIServerCache configures whether using APIServer cache or not.
+func (b *Builder) WithUsingAPIServerCache(u bool) {
+ b.useAPIServerCache = u
+}
+
// WithFamilyGeneratorFilter configures the family generator filter which decides which
// metrics are to be exposed by the store build by the Builder.
func (b *Builder) WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter) {
@@ -141,9 +154,13 @@ func (b *Builder) WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter) {
}
// WithGenerateStoresFunc configures a custom generate store function
-func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc, u bool) {
+func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc) {
b.buildStoresFunc = f
- b.useAPIServerCache = u
+}
+
+// WithGenerateCustomResourceStoresFunc configures a custom generate custom resource store function
+func (b *Builder) WithGenerateCustomResourceStoresFunc(f ksmtypes.BuildCustomResourceStoresFunc) {
+ b.buildCustomResourceStoresFunc = f
}
// DefaultGenerateStoresFunc returns default buildStores function
@@ -151,6 +168,30 @@ func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc {
return b.buildStores
}
+// DefaultGenerateCustomResourceStoresFunc returns default buildCustomResourceStores function
+func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc {
+ return b.buildCustomResourceStores
+}
+
+// WithCustomResourceStoreFactories returns configures a custom resource stores factory
+func (b *Builder) WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory) {
+ for i := range fs {
+ f := fs[i]
+ if _, ok := availableStores[f.Name()]; ok {
+ klog.Warningf("The internal resource store named %s already exists and is overridden by a custom resource store with the same name, please make sure it meets your expectation", f.Name())
+ }
+ availableStores[f.Name()] = func(b *Builder) []cache.Store {
+ return b.buildCustomResourceStoresFunc(
+ f.Name(),
+ f.MetricFamilyGenerators(b.allowAnnotationsList[f.Name()], b.allowLabelsList[f.Name()]),
+ f.ExpectedType(),
+ f.ListWatch,
+ b.useAPIServerCache,
+ )
+ }
+ }
+}
+
// WithAllowAnnotations configures which annotations can be returned for metrics
func (b *Builder) WithAllowAnnotations(annotations map[string][]string) {
if len(annotations) > 0 {
@@ -414,6 +455,47 @@ func (b *Builder) buildStores(
return stores
}
+// TODO(Garrybest): Merge `buildStores` and `buildCustomResourceStores`
+func (b *Builder) buildCustomResourceStores(resourceName string,
+ metricFamilies []generator.FamilyGenerator,
+ expectedType interface{},
+ listWatchFunc func(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher,
+ useAPIServerCache bool,
+) []cache.Store {
+ metricFamilies = generator.FilterFamilyGenerators(b.familyGeneratorFilter, metricFamilies)
+ composedMetricGenFuncs := generator.ComposeMetricGenFuncs(metricFamilies)
+ familyHeaders := generator.ExtractMetricFamilyHeaders(metricFamilies)
+
+ customResourceClient, ok := b.customResourceClients[resourceName]
+ if !ok {
+ klog.Warningf("Custom resource client %s does not exist", resourceName)
+ return []cache.Store{}
+ }
+
+ if b.namespaces.IsAllNamespaces() {
+ store := metricsstore.NewMetricsStore(
+ familyHeaders,
+ composedMetricGenFuncs,
+ )
+ listWatcher := listWatchFunc(customResourceClient, v1.NamespaceAll, b.namespaceFilter)
+ b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
+ return []cache.Store{store}
+ }
+
+ stores := make([]cache.Store, 0, len(b.namespaces))
+ for _, ns := range b.namespaces {
+ store := metricsstore.NewMetricsStore(
+ familyHeaders,
+ composedMetricGenFuncs,
+ )
+ listWatcher := listWatchFunc(customResourceClient, ns, b.namespaceFilter)
+ b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
+ stores = append(stores, store)
+ }
+
+ return stores
+}
+
// startReflector starts a Kubernetes client-go reflector with the given
// listWatcher and registers it with the given store.
func (b *Builder) startReflector(
diff --git a/main.go b/main.go
index a1a15b3ef5..ff8e5ff0cd 100644
--- a/main.go
+++ b/main.go
@@ -19,65 +19,21 @@ package main
import (
"context"
"fmt"
- "net"
- "net/http"
- "net/http/pprof"
"os"
- "strconv"
- "time"
- "github.com/oklog/run"
- "github.com/pkg/errors"
- "github.com/prometheus/client_golang/prometheus"
- "github.com/prometheus/client_golang/prometheus/collectors"
- "github.com/prometheus/client_golang/prometheus/promauto"
- "github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/prometheus/common/version"
- "github.com/prometheus/exporter-toolkit/web"
- vpaclientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
- clientset "k8s.io/client-go/kubernetes"
- _ "k8s.io/client-go/plugin/pkg/client/auth"
- "k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2"
- "k8s.io/kube-state-metrics/v2/internal/store"
- "k8s.io/kube-state-metrics/v2/pkg/allowdenylist"
- generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
- "k8s.io/kube-state-metrics/v2/pkg/metricshandler"
- "k8s.io/kube-state-metrics/v2/pkg/optin"
+ "k8s.io/kube-state-metrics/v2/pkg/app"
"k8s.io/kube-state-metrics/v2/pkg/options"
- "k8s.io/kube-state-metrics/v2/pkg/util/proc"
)
-const (
- metricsPath = "/metrics"
- healthzPath = "/healthz"
-)
-
-// promLogger implements promhttp.Logger
-type promLogger struct{}
-
-func (pl promLogger) Println(v ...interface{}) {
- klog.Error(v...)
-}
-
-// promLogger implements the Logger interface
-func (pl promLogger) Log(v ...interface{}) error {
- klog.Info(v...)
- return nil
-}
-
func main() {
opts := options.NewOptions()
opts.AddFlags()
- promLogger := promLogger{}
-
- ctx := context.Background()
-
- err := opts.Parse()
- if err != nil {
- klog.Fatalf("Error: %s", err)
+ if err := opts.Parse(); err != nil {
+ klog.Fatalf("Parsing flag definitions error: %v", err)
}
if opts.Version {
@@ -89,222 +45,9 @@ func main() {
opts.Usage()
os.Exit(0)
}
- storeBuilder := store.NewBuilder()
-
- ksmMetricsRegistry := prometheus.NewRegistry()
- ksmMetricsRegistry.MustRegister(version.NewCollector("kube_state_metrics"))
- durationVec := promauto.With(ksmMetricsRegistry).NewHistogramVec(
- prometheus.HistogramOpts{
- Name: "http_request_duration_seconds",
- Help: "A histogram of requests for kube-state-metrics metrics handler.",
- Buckets: prometheus.DefBuckets,
- ConstLabels: prometheus.Labels{"handler": "metrics"},
- }, []string{"method"},
- )
- storeBuilder.WithMetrics(ksmMetricsRegistry)
-
- var resources []string
- if len(opts.Resources) == 0 {
- klog.Info("Using default resources")
- resources = options.DefaultResources.AsSlice()
- } else {
- klog.Infof("Using resources %s", opts.Resources.String())
- resources = opts.Resources.AsSlice()
- }
-
- if err := storeBuilder.WithEnabledResources(resources); err != nil {
- klog.Fatalf("Failed to set up resources: %v", err)
- }
-
- namespaces := opts.Namespaces.GetNamespaces()
- nsFieldSelector := namespaces.GetExcludeNSFieldSelector(opts.NamespacesDenylist)
- storeBuilder.WithNamespaces(namespaces, nsFieldSelector)
-
- allowDenyList, err := allowdenylist.New(opts.MetricAllowlist, opts.MetricDenylist)
- if err != nil {
- klog.Fatal(err)
- }
-
- err = allowDenyList.Parse()
- if err != nil {
- klog.Fatalf("error initializing the allowdeny list : %v", err)
- }
-
- klog.Infof("metric allow-denylisting: %v", allowDenyList.Status())
-
- optInMetricFamilyFilter, err := optin.NewMetricFamilyFilter(opts.MetricOptInList)
- if err != nil {
- klog.Fatalf("error initializing the opt-in metric list : %v", err)
- }
-
- if optInMetricFamilyFilter.Count() > 0 {
- klog.Infof("metrics which were opted into: %v", optInMetricFamilyFilter.Status())
- }
-
- storeBuilder.WithFamilyGeneratorFilter(generator.NewCompositeFamilyGeneratorFilter(
- allowDenyList,
- optInMetricFamilyFilter,
- ))
-
- storeBuilder.WithGenerateStoresFunc(storeBuilder.DefaultGenerateStoresFunc(), opts.UseAPIServerCache)
-
- proc.StartReaper()
-
- kubeClient, vpaClient, err := createKubeClient(opts.Apiserver, opts.Kubeconfig)
- if err != nil {
- klog.Fatalf("Failed to create client: %v", err)
- }
- storeBuilder.WithKubeClient(kubeClient)
- storeBuilder.WithVPAClient(vpaClient)
- storeBuilder.WithSharding(opts.Shard, opts.TotalShards)
- storeBuilder.WithAllowAnnotations(opts.AnnotationsAllowList)
- storeBuilder.WithAllowLabels(opts.LabelsAllowList)
- ksmMetricsRegistry.MustRegister(
- collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
- collectors.NewGoCollector(),
- )
-
- var g run.Group
-
- m := metricshandler.New(
- opts,
- kubeClient,
- storeBuilder,
- opts.EnableGZIPEncoding,
- )
- // Run MetricsHandler
- {
- ctxMetricsHandler, cancel := context.WithCancel(ctx)
- g.Add(func() error {
- return m.Run(ctxMetricsHandler)
- }, func(error) {
- cancel()
- })
- }
-
- tlsConfig := opts.TLSConfig
-
- telemetryMux := buildTelemetryServer(ksmMetricsRegistry)
- telemetryListenAddress := net.JoinHostPort(opts.TelemetryHost, strconv.Itoa(opts.TelemetryPort))
- telemetryServer := http.Server{Handler: telemetryMux, Addr: telemetryListenAddress}
-
- metricsMux := buildMetricsServer(m, durationVec)
- metricsServerListenAddress := net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port))
- metricsServer := http.Server{Handler: metricsMux, Addr: metricsServerListenAddress}
-
- // Run Telemetry server
- {
- g.Add(func() error {
- klog.Infof("Starting kube-state-metrics self metrics server: %s", telemetryListenAddress)
- return web.ListenAndServe(&telemetryServer, tlsConfig, promLogger)
- }, func(error) {
- ctxShutDown, cancel := context.WithTimeout(ctx, 3*time.Second)
- defer cancel()
- telemetryServer.Shutdown(ctxShutDown)
- })
- }
- // Run Metrics server
- {
- g.Add(func() error {
- klog.Infof("Starting metrics server: %s", metricsServerListenAddress)
- return web.ListenAndServe(&metricsServer, tlsConfig, promLogger)
- }, func(error) {
- ctxShutDown, cancel := context.WithTimeout(ctx, 3*time.Second)
- defer cancel()
- metricsServer.Shutdown(ctxShutDown)
- })
- }
-
- if err := g.Run(); err != nil {
- klog.Fatalf("RunGroup Error: %v", err)
- }
- klog.Info("Exiting")
-}
-
-func createKubeClient(apiserver string, kubeconfig string) (clientset.Interface, vpaclientset.Interface, error) {
- config, err := clientcmd.BuildConfigFromFlags(apiserver, kubeconfig)
- if err != nil {
- return nil, nil, err
- }
-
- config.UserAgent = version.Version
- config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json"
- config.ContentType = "application/vnd.kubernetes.protobuf"
-
- kubeClient, err := clientset.NewForConfig(config)
- if err != nil {
- return nil, nil, err
- }
-
- vpaClient, err := vpaclientset.NewForConfig(config)
- if err != nil {
- return nil, nil, err
- }
- // Informers don't seem to do a good job logging error messages when it
- // can't reach the server, making debugging hard. This makes it easier to
- // figure out if apiserver is configured incorrectly.
- klog.Infof("Testing communication with server")
- v, err := kubeClient.Discovery().ServerVersion()
- if err != nil {
- return nil, nil, errors.Wrap(err, "error while trying to communicate with apiserver")
+ ctx := context.Background()
+ if err := app.RunKubeStateMetrics(ctx, opts); err != nil {
+ klog.Fatalf("Failed to run kube-state-metrics: %v", err)
}
- klog.Infof("Running with Kubernetes cluster version: v%s.%s. git version: %s. git tree state: %s. commit: %s. platform: %s",
- v.Major, v.Minor, v.GitVersion, v.GitTreeState, v.GitCommit, v.Platform)
- klog.Infof("Communication with server successful")
-
- return kubeClient, vpaClient, nil
-}
-
-func buildTelemetryServer(registry prometheus.Gatherer) *http.ServeMux {
- mux := http.NewServeMux()
-
- // Add metricsPath
- mux.Handle(metricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{ErrorLog: promLogger{}}))
- // Add index
- mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte(`
- Kube-State-Metrics Metrics Server
-
- Kube-State-Metrics Metrics
-
-
- `))
- })
- return mux
-}
-
-func buildMetricsServer(m *metricshandler.MetricsHandler, durationObserver prometheus.ObserverVec) *http.ServeMux {
- mux := http.NewServeMux()
-
- // TODO: This doesn't belong into serveMetrics
- mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
- mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
- mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
- mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
- mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
-
- mux.Handle(metricsPath, promhttp.InstrumentHandlerDuration(durationObserver, m))
-
- // Add healthzPath
- mux.HandleFunc(healthzPath, func(w http.ResponseWriter, r *http.Request) {
- w.WriteHeader(http.StatusOK)
- w.Write([]byte(http.StatusText(http.StatusOK)))
- })
- // Add index
- mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
- w.Write([]byte(`
- Kube Metrics Server
-
- Kube Metrics
-
-
- `))
- })
- return mux
}
diff --git a/pkg/app/server.go b/pkg/app/server.go
new file mode 100644
index 0000000000..d8205de28f
--- /dev/null
+++ b/pkg/app/server.go
@@ -0,0 +1,313 @@
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+ "context"
+ "fmt"
+ "net"
+ "net/http"
+ "net/http/pprof"
+ "strconv"
+ "time"
+
+ "github.com/oklog/run"
+ "github.com/pkg/errors"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/collectors"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+ "github.com/prometheus/common/version"
+ "github.com/prometheus/exporter-toolkit/web"
+ vpaclientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
+ clientset "k8s.io/client-go/kubernetes"
+ _ "k8s.io/client-go/plugin/pkg/client/auth" // Initialize common client auth plugins.
+ "k8s.io/client-go/tools/clientcmd"
+ "k8s.io/klog/v2"
+
+ "k8s.io/kube-state-metrics/v2/internal/store"
+ "k8s.io/kube-state-metrics/v2/pkg/allowdenylist"
+ "k8s.io/kube-state-metrics/v2/pkg/customresource"
+ generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
+ "k8s.io/kube-state-metrics/v2/pkg/metricshandler"
+ "k8s.io/kube-state-metrics/v2/pkg/optin"
+ "k8s.io/kube-state-metrics/v2/pkg/options"
+ "k8s.io/kube-state-metrics/v2/pkg/util/proc"
+)
+
+const (
+ metricsPath = "/metrics"
+ healthzPath = "/healthz"
+)
+
+// promLogger implements promhttp.Logger
+type promLogger struct{}
+
+func (pl promLogger) Println(v ...interface{}) {
+ klog.Error(v...)
+}
+
+// promLogger implements the Logger interface
+func (pl promLogger) Log(v ...interface{}) error {
+ klog.Info(v...)
+ return nil
+}
+
+// RunKubeStateMetrics will build and run the kube-state-metrics.
+// Any out-of-tree custom resource metrics could be registered by newing a registry factory
+// which implements customresource.RegistryFactory and pass all factories into this function.
+func RunKubeStateMetrics(ctx context.Context, opts *options.Options, factories ...customresource.RegistryFactory) error {
+ promLogger := promLogger{}
+
+ storeBuilder := store.NewBuilder()
+ storeBuilder.WithCustomResourceStoreFactories(factories...)
+
+ ksmMetricsRegistry := prometheus.NewRegistry()
+ ksmMetricsRegistry.MustRegister(version.NewCollector("kube_state_metrics"))
+ durationVec := promauto.With(ksmMetricsRegistry).NewHistogramVec(
+ prometheus.HistogramOpts{
+ Name: "http_request_duration_seconds",
+ Help: "A histogram of requests for kube-state-metrics metrics handler.",
+ Buckets: prometheus.DefBuckets,
+ ConstLabels: prometheus.Labels{"handler": "metrics"},
+ }, []string{"method"},
+ )
+ storeBuilder.WithMetrics(ksmMetricsRegistry)
+
+ var resources []string
+ if len(opts.Resources) == 0 {
+ klog.Info("Using default resources")
+ resources = options.DefaultResources.AsSlice()
+ // enable custom resource
+ for _, factory := range factories {
+ resources = append(resources, factory.Name())
+ }
+ } else {
+ klog.Infof("Using resources %s", opts.Resources.String())
+ resources = opts.Resources.AsSlice()
+ }
+
+ if err := storeBuilder.WithEnabledResources(resources); err != nil {
+ return fmt.Errorf("failed to set up resources: %v", err)
+ }
+
+ namespaces := opts.Namespaces.GetNamespaces()
+ nsFieldSelector := namespaces.GetExcludeNSFieldSelector(opts.NamespacesDenylist)
+ storeBuilder.WithNamespaces(namespaces, nsFieldSelector)
+
+ allowDenyList, err := allowdenylist.New(opts.MetricAllowlist, opts.MetricDenylist)
+ if err != nil {
+ return err
+ }
+
+ err = allowDenyList.Parse()
+ if err != nil {
+ return fmt.Errorf("error initializing the allowdeny list: %v", err)
+ }
+
+ klog.Infof("Metric allow-denylisting: %v", allowDenyList.Status())
+
+ optInMetricFamilyFilter, err := optin.NewMetricFamilyFilter(opts.MetricOptInList)
+ if err != nil {
+ return fmt.Errorf("error initializing the opt-in metric list: %v", err)
+ }
+
+ if optInMetricFamilyFilter.Count() > 0 {
+ klog.Infof("Metrics which were opted into: %v", optInMetricFamilyFilter.Status())
+ }
+
+ storeBuilder.WithFamilyGeneratorFilter(generator.NewCompositeFamilyGeneratorFilter(
+ allowDenyList,
+ optInMetricFamilyFilter,
+ ))
+
+ storeBuilder.WithUsingAPIServerCache(opts.UseAPIServerCache)
+ storeBuilder.WithGenerateStoresFunc(storeBuilder.DefaultGenerateStoresFunc())
+ storeBuilder.WithGenerateCustomResourceStoresFunc(storeBuilder.DefaultGenerateCustomResourceStoresFunc())
+
+ proc.StartReaper()
+
+ kubeClient, vpaClient, customResourceClients, err := createKubeClient(opts.Apiserver, opts.Kubeconfig, factories...)
+ if err != nil {
+ return fmt.Errorf("failed to create client: %v", err)
+ }
+ storeBuilder.WithKubeClient(kubeClient)
+ storeBuilder.WithVPAClient(vpaClient)
+ storeBuilder.WithCustomResourceClients(customResourceClients)
+ storeBuilder.WithSharding(opts.Shard, opts.TotalShards)
+ storeBuilder.WithAllowAnnotations(opts.AnnotationsAllowList)
+ storeBuilder.WithAllowLabels(opts.LabelsAllowList)
+
+ ksmMetricsRegistry.MustRegister(
+ collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
+ collectors.NewGoCollector(),
+ )
+
+ var g run.Group
+
+ m := metricshandler.New(
+ opts,
+ kubeClient,
+ storeBuilder,
+ opts.EnableGZIPEncoding,
+ )
+ // Run MetricsHandler
+ {
+ ctxMetricsHandler, cancel := context.WithCancel(ctx)
+ g.Add(func() error {
+ return m.Run(ctxMetricsHandler)
+ }, func(error) {
+ cancel()
+ })
+ }
+
+ tlsConfig := opts.TLSConfig
+
+ telemetryMux := buildTelemetryServer(ksmMetricsRegistry)
+ telemetryListenAddress := net.JoinHostPort(opts.TelemetryHost, strconv.Itoa(opts.TelemetryPort))
+ telemetryServer := http.Server{Handler: telemetryMux, Addr: telemetryListenAddress}
+
+ metricsMux := buildMetricsServer(m, durationVec)
+ metricsServerListenAddress := net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port))
+ metricsServer := http.Server{Handler: metricsMux, Addr: metricsServerListenAddress}
+
+ // Run Telemetry server
+ {
+ g.Add(func() error {
+ klog.Infof("Starting kube-state-metrics self metrics server: %s", telemetryListenAddress)
+ return web.ListenAndServe(&telemetryServer, tlsConfig, promLogger)
+ }, func(error) {
+ ctxShutDown, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
+ telemetryServer.Shutdown(ctxShutDown)
+ })
+ }
+ // Run Metrics server
+ {
+ g.Add(func() error {
+ klog.Infof("Starting metrics server: %s", metricsServerListenAddress)
+ return web.ListenAndServe(&metricsServer, tlsConfig, promLogger)
+ }, func(error) {
+ ctxShutDown, cancel := context.WithTimeout(ctx, 3*time.Second)
+ defer cancel()
+ metricsServer.Shutdown(ctxShutDown)
+ })
+ }
+
+ if err := g.Run(); err != nil {
+ return fmt.Errorf("run server group error: %v", err)
+ }
+ klog.Info("Exiting")
+ return nil
+}
+
+func createKubeClient(apiserver string, kubeconfig string, factories ...customresource.RegistryFactory) (clientset.Interface, vpaclientset.Interface, map[string]interface{}, error) {
+ config, err := clientcmd.BuildConfigFromFlags(apiserver, kubeconfig)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+
+ config.UserAgent = version.Version
+ config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json"
+ config.ContentType = "application/vnd.kubernetes.protobuf"
+
+ kubeClient, err := clientset.NewForConfig(config)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+
+ vpaClient, err := vpaclientset.NewForConfig(config)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+
+ customResourceClients := make(map[string]interface{}, len(factories))
+ for _, f := range factories {
+ customResourceClient, err := f.CreateClient(config)
+ if err != nil {
+ return nil, nil, nil, err
+ }
+ customResourceClients[f.Name()] = customResourceClient
+ }
+
+ // Informers don't seem to do a good job logging error messages when it
+ // can't reach the server, making debugging hard. This makes it easier to
+ // figure out if apiserver is configured incorrectly.
+ klog.Infof("Testing communication with server")
+ v, err := kubeClient.Discovery().ServerVersion()
+ if err != nil {
+ return nil, nil, nil, errors.Wrap(err, "error while trying to communicate with apiserver")
+ }
+ klog.Infof("Running with Kubernetes cluster version: v%s.%s. git version: %s. git tree state: %s. commit: %s. platform: %s",
+ v.Major, v.Minor, v.GitVersion, v.GitTreeState, v.GitCommit, v.Platform)
+ klog.Infof("Communication with server successful")
+
+ return kubeClient, vpaClient, customResourceClients, nil
+}
+
+func buildTelemetryServer(registry prometheus.Gatherer) *http.ServeMux {
+ mux := http.NewServeMux()
+
+ // Add metricsPath
+ mux.Handle(metricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{ErrorLog: promLogger{}}))
+ // Add index
+ mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`
+ Kube-State-Metrics Metrics Server
+
+ Kube-State-Metrics Metrics
+
+
+ `))
+ })
+ return mux
+}
+
+func buildMetricsServer(m *metricshandler.MetricsHandler, durationObserver prometheus.ObserverVec) *http.ServeMux {
+ mux := http.NewServeMux()
+
+ // TODO: This doesn't belong into serveMetrics
+ mux.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
+ mux.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
+ mux.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
+ mux.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
+ mux.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))
+
+ mux.Handle(metricsPath, promhttp.InstrumentHandlerDuration(durationObserver, m))
+
+ // Add healthzPath
+ mux.HandleFunc(healthzPath, func(w http.ResponseWriter, r *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ w.Write([]byte(http.StatusText(http.StatusOK)))
+ })
+ // Add index
+ mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+ w.Write([]byte(`
+ Kube Metrics Server
+
+ Kube Metrics
+
+
+ `))
+ })
+ return mux
+}
diff --git a/main_test.go b/pkg/app/server_test.go
similarity index 77%
rename from main_test.go
rename to pkg/app/server_test.go
index fc21c38c0f..53cd0356dd 100644
--- a/main_test.go
+++ b/pkg/app/server_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-package main
+package app
import (
"bytes"
@@ -28,17 +28,24 @@ import (
"testing"
"time"
- generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
-
"github.com/prometheus/client_golang/prometheus"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
+ "k8s.io/apimachinery/pkg/watch"
"k8s.io/client-go/kubernetes/fake"
+ "k8s.io/client-go/rest"
+ "k8s.io/client-go/tools/cache"
+ samplev1alpha1 "k8s.io/sample-controller/pkg/apis/samplecontroller/v1alpha1"
+ samplefake "k8s.io/sample-controller/pkg/generated/clientset/versioned/fake"
"k8s.io/kube-state-metrics/v2/internal/store"
"k8s.io/kube-state-metrics/v2/pkg/allowdenylist"
+ "k8s.io/kube-state-metrics/v2/pkg/customresource"
+ "k8s.io/kube-state-metrics/v2/pkg/metric"
+ generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
"k8s.io/kube-state-metrics/v2/pkg/metricshandler"
"k8s.io/kube-state-metrics/v2/pkg/options"
)
@@ -69,7 +76,7 @@ func BenchmarkKubeStateMetrics(b *testing.B) {
builder.WithSharding(0, 1)
builder.WithContext(ctx)
builder.WithNamespaces(options.DefaultNamespaces, "")
- builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc(), false)
+ builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc())
allowDenyListFilter, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{})
if err != nil {
@@ -138,7 +145,7 @@ func TestFullScrapeCycle(t *testing.T) {
builder.WithEnabledResources(options.DefaultResources.AsSlice())
builder.WithKubeClient(kubeClient)
builder.WithNamespaces(options.DefaultNamespaces, "")
- builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc(), false)
+ builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc())
l, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{})
if err != nil {
@@ -407,7 +414,7 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
unshardedBuilder.WithNamespaces(options.DefaultNamespaces, "")
unshardedBuilder.WithFamilyGeneratorFilter(l)
unshardedBuilder.WithAllowLabels(map[string][]string{})
- unshardedBuilder.WithGenerateStoresFunc(unshardedBuilder.DefaultGenerateStoresFunc(), false)
+ unshardedBuilder.WithGenerateStoresFunc(unshardedBuilder.DefaultGenerateStoresFunc())
unshardedHandler := metricshandler.New(&options.Options{}, kubeClient, unshardedBuilder, false)
unshardedHandler.ConfigureSharding(ctx, 0, 1)
@@ -420,7 +427,7 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
shardedBuilder1.WithNamespaces(options.DefaultNamespaces, "")
shardedBuilder1.WithFamilyGeneratorFilter(l)
shardedBuilder1.WithAllowLabels(map[string][]string{})
- shardedBuilder1.WithGenerateStoresFunc(shardedBuilder1.DefaultGenerateStoresFunc(), false)
+ shardedBuilder1.WithGenerateStoresFunc(shardedBuilder1.DefaultGenerateStoresFunc())
shardedHandler1 := metricshandler.New(&options.Options{}, kubeClient, shardedBuilder1, false)
shardedHandler1.ConfigureSharding(ctx, 0, 2)
@@ -433,7 +440,7 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
shardedBuilder2.WithNamespaces(options.DefaultNamespaces, "")
shardedBuilder2.WithFamilyGeneratorFilter(l)
shardedBuilder2.WithAllowLabels(map[string][]string{})
- shardedBuilder2.WithGenerateStoresFunc(shardedBuilder2.DefaultGenerateStoresFunc(), false)
+ shardedBuilder2.WithGenerateStoresFunc(shardedBuilder2.DefaultGenerateStoresFunc())
shardedHandler2 := metricshandler.New(&options.Options{}, kubeClient, shardedBuilder2, false)
shardedHandler2.ConfigureSharding(ctx, 1, 2)
@@ -541,6 +548,121 @@ func TestShardingEquivalenceScrapeCycle(t *testing.T) {
}
}
+// TestCustomResourceExtension is a simple smoke test covering the custom resource metrics collection.
+// We use custom resource object samplev1alpha1.Foo in kubernetes/sample-controller as an example.
+func TestCustomResourceExtension(t *testing.T) {
+ kubeClient := fake.NewSimpleClientset()
+ factories := []customresource.RegistryFactory{new(fooFactory)}
+ resources := options.DefaultResources.AsSlice()
+ customResourceClients := make(map[string]interface{}, len(factories))
+ // enable custom resource
+ for _, f := range factories {
+ resources = append(resources, f.Name())
+ customResourceClient, err := f.CreateClient(nil)
+ if err != nil {
+ t.Fatalf("Failed to create customResourceClient for foo: %v", err)
+ }
+ customResourceClients[f.Name()] = customResourceClient
+ }
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ reg := prometheus.NewRegistry()
+ builder := store.NewBuilder()
+ builder.WithCustomResourceStoreFactories(factories...)
+ builder.WithMetrics(reg)
+ builder.WithEnabledResources(resources)
+ builder.WithKubeClient(kubeClient)
+ builder.WithCustomResourceClients(customResourceClients)
+ builder.WithNamespaces(options.DefaultNamespaces, "")
+ builder.WithGenerateStoresFunc(builder.DefaultGenerateStoresFunc())
+ builder.WithGenerateCustomResourceStoresFunc(builder.DefaultGenerateCustomResourceStoresFunc())
+
+ l, err := allowdenylist.New(map[string]struct{}{}, map[string]struct{}{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ builder.WithFamilyGeneratorFilter(l)
+ builder.WithAllowLabels(map[string][]string{
+ "kube_foo_labels": {
+ "namespace",
+ "foo",
+ "uid",
+ },
+ })
+
+ handler := metricshandler.New(&options.Options{}, kubeClient, builder, false)
+ handler.ConfigureSharding(ctx, 0, 1)
+
+ // Wait for caches to fill
+ time.Sleep(time.Second)
+
+ req := httptest.NewRequest("GET", "http://localhost:8080/metrics", nil)
+
+ w := httptest.NewRecorder()
+ handler.ServeHTTP(w, req)
+
+ resp := w.Result()
+ if resp.StatusCode != 200 {
+ t.Fatalf("expected 200 status code but got %v", resp.StatusCode)
+ }
+
+ body, _ := io.ReadAll(resp.Body)
+
+ expected := `# HELP kube_foo_spec_replicas Number of desired replicas for a foo.
+# HELP kube_foo_status_replicas_available The number of available replicas per foo.
+# TYPE kube_foo_spec_replicas gauge
+# TYPE kube_foo_status_replicas_available gauge
+kube_foo_spec_replicas{namespace="default",foo="foo0"} 0
+kube_foo_spec_replicas{namespace="default",foo="foo1"} 1
+kube_foo_spec_replicas{namespace="default",foo="foo2"} 2
+kube_foo_spec_replicas{namespace="default",foo="foo3"} 3
+kube_foo_spec_replicas{namespace="default",foo="foo4"} 4
+kube_foo_spec_replicas{namespace="default",foo="foo5"} 5
+kube_foo_spec_replicas{namespace="default",foo="foo6"} 6
+kube_foo_spec_replicas{namespace="default",foo="foo7"} 7
+kube_foo_spec_replicas{namespace="default",foo="foo8"} 8
+kube_foo_spec_replicas{namespace="default",foo="foo9"} 9
+kube_foo_status_replicas_available{namespace="default",foo="foo0"} 0
+kube_foo_status_replicas_available{namespace="default",foo="foo1"} 1
+kube_foo_status_replicas_available{namespace="default",foo="foo2"} 2
+kube_foo_status_replicas_available{namespace="default",foo="foo3"} 3
+kube_foo_status_replicas_available{namespace="default",foo="foo5"} 5
+kube_foo_status_replicas_available{namespace="default",foo="foo6"} 6
+kube_foo_status_replicas_available{namespace="default",foo="foo7"} 7
+kube_foo_status_replicas_available{namespace="default",foo="foo8"} 8
+kube_foo_status_replicas_available{namespace="default",foo="foo4"} 4
+kube_foo_status_replicas_available{namespace="default",foo="foo9"} 9
+`
+
+ expectedSplit := strings.Split(strings.TrimSpace(expected), "\n")
+ sort.Strings(expectedSplit)
+
+ gotSplit := strings.Split(strings.TrimSpace(string(body)), "\n")
+
+ gotFiltered := []string{}
+ for _, l := range gotSplit {
+ if strings.Contains(l, "kube_foo_") {
+ gotFiltered = append(gotFiltered, l)
+ }
+ }
+
+ sort.Strings(gotFiltered)
+
+ if len(expectedSplit) != len(gotFiltered) {
+ fmt.Println(len(expectedSplit))
+ fmt.Println(len(gotFiltered))
+ t.Fatalf("expected different output length, expected \n\n%s\n\ngot\n\n%s", expected, strings.Join(gotFiltered, "\n"))
+ }
+
+ for i := 0; i < len(expectedSplit); i++ {
+ if expectedSplit[i] != gotFiltered[i] {
+ t.Fatalf("expected:\n\n%v\n, but got:\n\n%v", expectedSplit[i], gotFiltered[i])
+ }
+ }
+}
+
func injectFixtures(client *fake.Clientset, multiplier int) error {
creators := []func(*fake.Clientset, int) error{
configMap,
@@ -673,3 +795,116 @@ func pod(client *fake.Clientset, index int) error {
_, err := client.CoreV1().Pods(metav1.NamespaceDefault).Create(context.TODO(), &pod, metav1.CreateOptions{})
return err
}
+
+func foo(client *samplefake.Clientset, index int) error {
+ i := strconv.Itoa(index)
+ desiredReplicas := int32(index)
+
+ foo := samplev1alpha1.Foo{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "foo" + i,
+ CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)},
+ UID: types.UID("abc-" + i),
+ },
+ Spec: samplev1alpha1.FooSpec{
+ DeploymentName: "foo" + i,
+ Replicas: &desiredReplicas,
+ },
+ Status: samplev1alpha1.FooStatus{
+ AvailableReplicas: desiredReplicas,
+ },
+ }
+
+ _, err := client.SamplecontrollerV1alpha1().Foos(metav1.NamespaceDefault).Create(context.TODO(), &foo, metav1.CreateOptions{})
+ return err
+}
+
+var (
+ descFooLabelsDefaultLabels = []string{"namespace", "foo"}
+)
+
+type fooFactory struct{}
+
+func (f *fooFactory) Name() string {
+ return "foos"
+}
+
+// CreateClient use fake client set to establish 10 foos.
+func (f *fooFactory) CreateClient(cfg *rest.Config) (interface{}, error) {
+ fooClient := samplefake.NewSimpleClientset()
+ for i := 0; i < 10; i++ {
+ err := foo(fooClient, i)
+ if err != nil {
+ return nil, fmt.Errorf("failed to insert sample pod %v", err)
+ }
+ }
+ return fooClient, nil
+}
+
+func (f *fooFactory) MetricFamilyGenerators(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator {
+ return []generator.FamilyGenerator{
+ *generator.NewFamilyGenerator(
+ "kube_foo_spec_replicas",
+ "Number of desired replicas for a foo.",
+ metric.Gauge,
+ "",
+ wrapFooFunc(func(f *samplev1alpha1.Foo) *metric.Family {
+ return &metric.Family{
+ Metrics: []*metric.Metric{
+ {
+ Value: float64(*f.Spec.Replicas),
+ },
+ },
+ }
+ }),
+ ),
+ *generator.NewFamilyGenerator(
+ "kube_foo_status_replicas_available",
+ "The number of available replicas per foo.",
+ metric.Gauge,
+ "",
+ wrapFooFunc(func(f *samplev1alpha1.Foo) *metric.Family {
+ return &metric.Family{
+ Metrics: []*metric.Metric{
+ {
+ Value: float64(f.Status.AvailableReplicas),
+ },
+ },
+ }
+ }),
+ ),
+ }
+}
+
+func wrapFooFunc(f func(*samplev1alpha1.Foo) *metric.Family) func(interface{}) *metric.Family {
+ return func(obj interface{}) *metric.Family {
+ foo := obj.(*samplev1alpha1.Foo)
+
+ metricFamily := f(foo)
+
+ for _, m := range metricFamily.Metrics {
+ m.LabelKeys = append(descFooLabelsDefaultLabels, m.LabelKeys...)
+ m.LabelValues = append([]string{foo.Namespace, foo.Name}, m.LabelValues...)
+ }
+
+ return metricFamily
+ }
+}
+
+func (f *fooFactory) ExpectedType() interface{} {
+ return &samplev1alpha1.Foo{}
+}
+
+func (f *fooFactory) ListWatch(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher {
+ client := customResourceClient.(*samplefake.Clientset)
+ return &cache.ListWatch{
+ ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
+ opts.FieldSelector = fieldSelector
+ return client.SamplecontrollerV1alpha1().Foos(ns).List(context.Background(), opts)
+ },
+ WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
+ opts.FieldSelector = fieldSelector
+ return client.SamplecontrollerV1alpha1().Foos(ns).Watch(context.Background(), opts)
+ },
+ }
+}
diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go
index cd28bf1a0a..1e2763c455 100644
--- a/pkg/builder/builder.go
+++ b/pkg/builder/builder.go
@@ -28,6 +28,7 @@ import (
internalstore "k8s.io/kube-state-metrics/v2/internal/store"
ksmtypes "k8s.io/kube-state-metrics/v2/pkg/builder/types"
+ "k8s.io/kube-state-metrics/v2/pkg/customresource"
metricsstore "k8s.io/kube-state-metrics/v2/pkg/metrics_store"
"k8s.io/kube-state-metrics/v2/pkg/options"
)
@@ -81,12 +82,27 @@ func (b *Builder) WithVPAClient(c vpaclientset.Interface) {
b.internal.WithVPAClient(c)
}
+// WithCustomResourceClients sets the customResourceClients property of a Builder.
+func (b *Builder) WithCustomResourceClients(cs map[string]interface{}) {
+ b.internal.WithCustomResourceClients(cs)
+}
+
+// WithUsingAPIServerCache configures whether using APIServer cache or not.
+func (b *Builder) WithUsingAPIServerCache(u bool) {
+ b.internal.WithUsingAPIServerCache(u)
+}
+
// WithFamilyGeneratorFilter configures the family generator filter which decides which
// metrics are to be exposed by the store build by the Builder.
func (b *Builder) WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter) {
b.internal.WithFamilyGeneratorFilter(l)
}
+// WithAllowAnnotations configures which annotations can be returned for metrics
+func (b *Builder) WithAllowAnnotations(annotations map[string][]string) {
+ b.internal.WithAllowAnnotations(annotations)
+}
+
// WithAllowLabels configures which labels can be returned for metrics
func (b *Builder) WithAllowLabels(l map[string][]string) {
b.internal.WithAllowLabels(l)
@@ -94,7 +110,12 @@ func (b *Builder) WithAllowLabels(l map[string][]string) {
// WithGenerateStoresFunc configures a custom generate store function
func (b *Builder) WithGenerateStoresFunc(f ksmtypes.BuildStoresFunc) {
- b.internal.WithGenerateStoresFunc(f, false)
+ b.internal.WithGenerateStoresFunc(f)
+}
+
+// WithGenerateCustomResourceStoresFunc configures a custom generate custom resource store function
+func (b *Builder) WithGenerateCustomResourceStoresFunc(f ksmtypes.BuildCustomResourceStoresFunc) {
+ b.internal.WithGenerateCustomResourceStoresFunc(f)
}
// DefaultGenerateStoresFunc returns default buildStore function
@@ -102,6 +123,16 @@ func (b *Builder) DefaultGenerateStoresFunc() ksmtypes.BuildStoresFunc {
return b.internal.DefaultGenerateStoresFunc()
}
+// DefaultGenerateCustomResourceStoresFunc returns default buildStores function
+func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustomResourceStoresFunc {
+ return b.internal.DefaultGenerateCustomResourceStoresFunc()
+}
+
+// WithCustomResourceStoreFactories returns configures a custom resource stores factory
+func (b *Builder) WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory) {
+ b.internal.WithCustomResourceStoreFactories(fs...)
+}
+
// Build initializes and registers all enabled stores.
// Returns metric writers.
func (b *Builder) Build() []metricsstore.MetricsWriter {
diff --git a/pkg/builder/types/interfaces.go b/pkg/builder/types/interfaces.go
index 15995435ad..21b657a24f 100644
--- a/pkg/builder/types/interfaces.go
+++ b/pkg/builder/types/interfaces.go
@@ -26,6 +26,7 @@ import (
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/cache"
+ "k8s.io/kube-state-metrics/v2/pkg/customresource"
generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
"k8s.io/kube-state-metrics/v2/pkg/options"
)
@@ -39,10 +40,16 @@ type BuilderInterface interface {
WithContext(ctx context.Context)
WithKubeClient(c clientset.Interface)
WithVPAClient(c vpaclientset.Interface)
+ WithCustomResourceClients(cs map[string]interface{})
+ WithUsingAPIServerCache(u bool)
WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter)
+ WithAllowAnnotations(a map[string][]string)
WithAllowLabels(l map[string][]string)
- WithGenerateStoresFunc(f BuildStoresFunc, useAPIServerCache bool)
+ WithGenerateStoresFunc(f BuildStoresFunc)
+ WithGenerateCustomResourceStoresFunc(f BuildCustomResourceStoresFunc)
DefaultGenerateStoresFunc() BuildStoresFunc
+ DefaultGenerateCustomResourceStoresFunc() BuildCustomResourceStoresFunc
+ WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory)
Build() []metricsstore.MetricsWriter
BuildStores() [][]cache.Store
}
@@ -54,6 +61,14 @@ type BuildStoresFunc func(metricFamilies []generator.FamilyGenerator,
useAPIServerCache bool,
) []cache.Store
+// BuildCustomResourceStoresFunc function signature that is used to return a list of custom resource cache.Store
+type BuildCustomResourceStoresFunc func(resourceName string,
+ metricFamilies []generator.FamilyGenerator,
+ expectedType interface{},
+ listWatchFunc func(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher,
+ useAPIServerCache bool,
+) []cache.Store
+
// AllowDenyLister interface for AllowDeny lister that can allow or exclude metrics by there names
type AllowDenyLister interface {
IsIncluded(string) bool
diff --git a/pkg/customresource/registry_factory.go b/pkg/customresource/registry_factory.go
new file mode 100644
index 0000000000..e4ccee9b1e
--- /dev/null
+++ b/pkg/customresource/registry_factory.go
@@ -0,0 +1,116 @@
+/*
+Copyright 2021 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package customresource
+
+import (
+ "k8s.io/client-go/rest"
+ "k8s.io/client-go/tools/cache"
+
+ generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
+)
+
+// RegistryFactory is a registry interface for a CustomResourceStore.
+// Users who want to extend the kube-state-metrics to support Custom Resource metrics should
+// implement this interface.
+type RegistryFactory interface {
+ // Name returns the name of custom resource.
+ //
+ // Example:
+ //
+ // func (f *FooFactory) Name() string {
+ // return "foos"
+ // }
+ Name() string
+
+ // CreateClient creates a new custom resource client for the given config.
+ //
+ // Example:
+ //
+ // func (f *FooFactory) CreateClient(cfg *rest.Config) (interface{}, error) {
+ // return clientset.NewForConfig(cfg)
+ // }
+ CreateClient(cfg *rest.Config) (interface{}, error)
+
+ // MetricFamilyGenerators returns the metric family generators to generate metric families with a
+ // Kubernetes custom resource object.
+ //
+ // Example:
+ //
+ // func (f *FooFactory) MetricFamilyGenerators(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator {
+ // return []generator.FamilyGenerator{
+ // *generator.NewFamilyGenerator(
+ // "kube_foo_spec_replicas",
+ // "Number of desired replicas for a foo.",
+ // metric.Gauge,
+ // "",
+ // wrapFooFunc(func(f *samplev1alpha1.Foo) *metric.Family {
+ // return &metric.Family{
+ // Metrics: []*metric.Metric{
+ // {
+ // Value: float64(*f.Spec.Replicas),
+ // },
+ // },
+ // }
+ // }),
+ // ),
+ // *generator.NewFamilyGenerator(
+ // "kube_foo_status_replicas_available",
+ // "The number of available replicas per foo.",
+ // metric.Gauge,
+ // "",
+ // wrapFooFunc(func(f *samplev1alpha1.Foo) *metric.Family {
+ // return &metric.Family{
+ // Metrics: []*metric.Metric{
+ // {
+ // Value: float64(f.Status.AvailableReplicas),
+ // },
+ // },
+ // }
+ // }),
+ // ),
+ // }
+ // }
+ MetricFamilyGenerators(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator
+
+ // ExpectedType returns a pointer to an empty custom resource object.
+ //
+ // Example:
+ //
+ // func (f *FooFactory) ExpectedType() interface{} {
+ // return &samplev1alpha1.Foo{}
+ // }
+ ExpectedType() interface{}
+
+ // ListWatch constructs a cache.ListerWatcher of the custom resource object.
+ //
+ // Example:
+ //
+ // func (f *FooFactory) ListWatch(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher {
+ // client := customResourceClient.(*clientset.Clientset)
+ // return &cache.ListWatch{
+ // ListFunc: func(opts metav1.ListOptions) (runtime.Object, error) {
+ // opts.FieldSelector = fieldSelector
+ // return client.SamplecontrollerV1alpha1().Foos(ns).List(context.Background(), opts)
+ // },
+ // WatchFunc: func(opts metav1.ListOptions) (watch.Interface, error) {
+ // opts.FieldSelector = fieldSelector
+ // return client.SamplecontrollerV1alpha1().Foos(ns).Watch(context.Background(), opts)
+ // },
+ // }
+ // }
+ ListWatch(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher
+}