diff --git a/cmd/katalyst-controller/app/controller/resourcerecommender.go b/cmd/katalyst-controller/app/controller/resourcerecommender.go new file mode 100644 index 000000000..cc6855736 --- /dev/null +++ b/cmd/katalyst-controller/app/controller/resourcerecommender.go @@ -0,0 +1,50 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + "k8s.io/klog/v2" + + katalyst "github.com/kubewharf/katalyst-core/cmd/base" + "github.com/kubewharf/katalyst-core/pkg/config" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/controller" +) + +const ( + ResourceRecommenderControllerName = "resourcerecommender" +) + +func StartResourceRecommenderController( + ctx context.Context, + _ *katalyst.GenericContext, + conf *config.Configuration, + _ interface{}, + _ string, +) (bool, error) { + resourceRecommenderController, err := controller.NewResourceRecommenderController(ctx, + conf.GenericConfiguration, + conf.ControllersConfiguration.ResourceRecommenderConfig) + if err != nil { + klog.Errorf("failed to new ResourceRecommender controller") + return false, err + } + + go resourceRecommenderController.Run() + return true, nil +} diff --git a/cmd/katalyst-controller/app/enablecontrollers.go b/cmd/katalyst-controller/app/enablecontrollers.go index f827e1733..0d74453fc 100644 --- a/cmd/katalyst-controller/app/enablecontrollers.go +++ b/cmd/katalyst-controller/app/enablecontrollers.go @@ -54,6 +54,7 @@ func init() { controllerInitializers.Store(controller.MonitorControllerName, ControllerStarter{Starter: controller.StartMonitorController}) controllerInitializers.Store(controller.OvercommitControllerName, ControllerStarter{Starter: controller.StartOvercommitController}) controllerInitializers.Store(controller.TideControllerName, ControllerStarter{Starter: controller.StartTideController}) + controllerInitializers.Store(controller.ResourceRecommenderControllerName, ControllerStarter{Starter: controller.StartResourceRecommenderController}) } // RegisterControllerInitializer is used to register user-defined controllers diff --git a/cmd/katalyst-controller/app/options/controller.go b/cmd/katalyst-controller/app/options/controller.go index e662e99d0..d915dddb9 100644 --- a/cmd/katalyst-controller/app/options/controller.go +++ b/cmd/katalyst-controller/app/options/controller.go @@ -30,16 +30,18 @@ type ControllersOptions struct { *LifeCycleOptions *MonitorOptions *OvercommitOptions + *ResourceRecommenderOptions } func NewControllersOptions() *ControllersOptions { return &ControllersOptions{ - VPAOptions: NewVPAOptions(), - KCCOptions: NewKCCOptions(), - SPDOptions: NewSPDOptions(), - LifeCycleOptions: NewLifeCycleOptions(), - MonitorOptions: NewMonitorOptions(), - OvercommitOptions: NewOvercommitOptions(), + VPAOptions: NewVPAOptions(), + KCCOptions: NewKCCOptions(), + SPDOptions: NewSPDOptions(), + LifeCycleOptions: NewLifeCycleOptions(), + MonitorOptions: NewMonitorOptions(), + OvercommitOptions: NewOvercommitOptions(), + ResourceRecommenderOptions: NewResourceRecommenderOptions(), } } @@ -50,6 +52,7 @@ func (o *ControllersOptions) AddFlags(fss *cliflag.NamedFlagSets) { o.LifeCycleOptions.AddFlags(fss) o.MonitorOptions.AddFlags(fss) o.OvercommitOptions.AddFlags(fss) + o.ResourceRecommenderOptions.AddFlags(fss) } // ApplyTo fills up config with options @@ -62,6 +65,7 @@ func (o *ControllersOptions) ApplyTo(c *controllerconfig.ControllersConfiguratio errList = append(errList, o.LifeCycleOptions.ApplyTo(c.LifeCycleConfig)) errList = append(errList, o.MonitorOptions.ApplyTo(c.MonitorConfig)) errList = append(errList, o.OvercommitOptions.ApplyTo(c.OvercommitConfig)) + errList = append(errList, o.ResourceRecommenderOptions.ApplyTo(c.ResourceRecommenderConfig)) return errors.NewAggregate(errList) } diff --git a/cmd/katalyst-controller/app/options/resourcerecommender.go b/cmd/katalyst-controller/app/options/resourcerecommender.go new file mode 100644 index 000000000..7ec8f3ff6 --- /dev/null +++ b/cmd/katalyst-controller/app/options/resourcerecommender.go @@ -0,0 +1,100 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "time" + + cliflag "k8s.io/component-base/cli/flag" + + "github.com/kubewharf/katalyst-core/pkg/config/controller" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus" +) + +type ResourceRecommenderOptions struct { + OOMRecordMaxNumber int `desc:"max number for oom record"` + + HealthProbeBindPort string `desc:"The port the health probe binds to."` + MetricsBindPort string `desc:"The port the metric endpoint binds to."` + + // available datasource: prom + DataSource []string + // DataSourcePromConfig is the prometheus datasource config + DataSourcePromConfig prometheus.PromConfig + + // LogVerbosityLevel to specify log verbosity level. (The default level is 4) + // Set it to something larger than 4 if more detailed logs are needed. + LogVerbosityLevel string +} + +// NewResourceRecommenderOptions creates a new Options with a default config. +func NewResourceRecommenderOptions() *ResourceRecommenderOptions { + return &ResourceRecommenderOptions{ + OOMRecordMaxNumber: 5000, + HealthProbeBindPort: "8080", + MetricsBindPort: "8081", + DataSource: []string{"prom"}, + DataSourcePromConfig: prometheus.PromConfig{ + KeepAlive: 60 * time.Second, + Timeout: 3 * time.Minute, + BRateLimit: false, + MaxPointsLimitPerTimeSeries: 11000, + }, + LogVerbosityLevel: "4", + } +} + +// AddFlags adds flags to the specified FlagSet +func (o *ResourceRecommenderOptions) AddFlags(fss *cliflag.NamedFlagSets) { + fs := fss.FlagSet("resource-recommend") + fs.IntVar(&o.OOMRecordMaxNumber, "oom-record-max-number", 5000, "Max number for oom records to store in configmap") + + fs.StringVar(&o.HealthProbeBindPort, "resourcerecommend-health-probe-bind-port", "8080", "The port the health probe binds to.") + fs.StringVar(&o.MetricsBindPort, "resourcerecommend-metrics-bind-port", "8081", "The port the metric endpoint binds to.") + + fs.StringSliceVar(&o.DataSource, "resourcerecommend-datasource", []string{"prom"}, "available datasource: prom") + fs.StringVar(&o.DataSourcePromConfig.Address, "resourcerecommend-prometheus-address", "", "prometheus address") + fs.StringVar(&o.DataSourcePromConfig.Auth.Type, "resourcerecommend-prometheus-auth-type", "", "prometheus auth type") + fs.StringVar(&o.DataSourcePromConfig.Auth.Username, "resourcerecommend-prometheus-auth-username", "", "prometheus auth username") + fs.StringVar(&o.DataSourcePromConfig.Auth.Password, "resourcerecommend-prometheus-auth-password", "", "prometheus auth password") + fs.StringVar(&o.DataSourcePromConfig.Auth.BearerToken, "resourcerecommend-prometheus-auth-bearertoken", "", "prometheus auth bearertoken") + fs.DurationVar(&o.DataSourcePromConfig.KeepAlive, "resourcerecommend-prometheus-keepalive", 60*time.Second, "prometheus keep alive") + fs.DurationVar(&o.DataSourcePromConfig.Timeout, "resourcerecommend-prometheus-timeout", 3*time.Minute, "prometheus timeout") + fs.BoolVar(&o.DataSourcePromConfig.BRateLimit, "resourcerecommend-prometheus-bratelimit", false, "prometheus bratelimit") + fs.IntVar(&o.DataSourcePromConfig.MaxPointsLimitPerTimeSeries, "resourcerecommend-prometheus-maxpoints", 11000, "prometheus max points limit per time series") + fs.StringVar(&o.DataSourcePromConfig.BaseFilter, "resourcerecommend-prometheus-promql-base-filter", "", ""+ + "Get basic filters in promql for historical usage data. This filter is added to all promql statements. "+ + "Supports filters format of promql, e.g: group=\\\"Katalyst\\\",cluster=\\\"cfeaf782fasdfe\\\"") +} + +func (o *ResourceRecommenderOptions) ApplyTo(c *controller.ResourceRecommenderConfig) error { + c.OOMRecordMaxNumber = o.OOMRecordMaxNumber + c.HealthProbeBindPort = o.HealthProbeBindPort + c.MetricsBindPort = o.MetricsBindPort + c.DataSource = o.DataSource + c.DataSourcePromConfig = o.DataSourcePromConfig + c.LogVerbosityLevel = o.LogVerbosityLevel + return nil +} + +func (o *ResourceRecommenderOptions) Config() (*controller.ResourceRecommenderConfig, error) { + c := &controller.ResourceRecommenderConfig{} + if err := o.ApplyTo(c); err != nil { + return nil, err + } + return c, nil +} diff --git a/go.mod b/go.mod index 817fd57df..1cb2ec21c 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/kubewharf/katalyst-core go 1.18 require ( + bou.ke/monkey v1.0.2 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d github.com/cespare/xxhash v1.1.0 github.com/cilium/ebpf v0.7.0 @@ -12,17 +13,18 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.5.2 github.com/google/cadvisor v0.44.2 + github.com/google/uuid v1.3.0 github.com/klauspost/cpuid/v2 v2.2.6 github.com/kubewharf/katalyst-api v0.4.1-0.20240222122824-be538f641f58 github.com/montanaflynn/stats v0.7.1 github.com/opencontainers/runc v1.1.6 github.com/opencontainers/selinux v1.10.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.12.1 - github.com/prometheus/client_model v0.2.0 - github.com/prometheus/common v0.32.1 + github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_model v0.3.0 + github.com/prometheus/common v0.37.0 github.com/slok/kubewebhook v0.11.0 - github.com/spf13/cobra v1.4.0 + github.com/spf13/cobra v1.6.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.1 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae @@ -32,25 +34,27 @@ require ( go.opentelemetry.io/otel/sdk v0.20.0 go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go.opentelemetry.io/otel/sdk/metric v0.20.0 - go.uber.org/atomic v1.7.0 + go.uber.org/atomic v1.9.0 golang.org/x/sys v0.7.0 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 - gonum.org/v1/gonum v0.6.2 + gonum.org/v1/gonum v0.8.2 google.golang.org/grpc v1.51.0 + gopkg.in/yaml.v3 v3.0.1 gotest.tools/v3 v3.0.3 - k8s.io/api v0.24.16 - k8s.io/apimachinery v0.24.16 + k8s.io/api v0.26.1 + k8s.io/apimachinery v0.26.1 k8s.io/apiserver v0.24.16 k8s.io/autoscaler/cluster-autoscaler v0.0.0-20231010095923-8a61add71154 - k8s.io/client-go v0.24.16 - k8s.io/component-base v0.24.16 + k8s.io/autoscaler/vertical-pod-autoscaler v1.0.0 + k8s.io/client-go v0.26.1 + k8s.io/component-base v0.25.0 k8s.io/component-helpers v0.24.16 k8s.io/cri-api v0.24.6 k8s.io/klog/v2 v2.80.1 k8s.io/kube-aggregator v0.24.6 k8s.io/kubelet v0.24.6 k8s.io/kubernetes v1.24.16 - k8s.io/metrics v0.24.6 + k8s.io/metrics v0.25.0 k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2 sigs.k8s.io/controller-runtime v0.11.2 sigs.k8s.io/custom-metrics-apiserver v1.24.0 @@ -62,8 +66,6 @@ require ( github.com/Microsoft/go-winio v0.4.17 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/OneOfOne/xxhash v1.2.5 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect @@ -78,23 +80,22 @@ require ( github.com/felixge/httpsnoop v1.0.1 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.19.15 // indirect github.com/godbus/dbus/v5 v5.0.6 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/gnostic v0.6.9 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/imdario/mergo v0.3.12 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.6 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/moby/sys/mountinfo v0.6.0 // indirect @@ -107,8 +108,9 @@ require ( github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect - github.com/sirupsen/logrus v1.8.1 // indirect + github.com/prometheus/procfs v0.8.0 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect go.etcd.io/etcd/api/v3 v3.5.4 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect go.etcd.io/etcd/client/v3 v3.5.4 // indirect @@ -118,27 +120,28 @@ require ( go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect go.opentelemetry.io/otel/trace v0.20.0 // indirect go.opentelemetry.io/proto/otlp v0.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect + go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.19.1 // indirect golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be // indirect golang.org/x/net v0.9.0 // indirect - golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect + golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/term v0.7.0 // indirect golang.org/x/text v0.9.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect gomodules.xyz/jsonpatch/v3 v3.0.1 // indirect gomodules.xyz/orderedmap v0.1.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect - google.golang.org/protobuf v1.28.0 // indirect - gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.24.2 // indirect k8s.io/cloud-provider v0.24.16 // indirect k8s.io/csi-translation-lib v0.24.16 // indirect - k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect + k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect k8s.io/kube-scheduler v0.24.6 // indirect k8s.io/mount-utils v0.24.16 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37 // indirect diff --git a/go.sum b/go.sum index 251b21e04..c7f9ca603 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= bitbucket.org/bertimus9/systemstat v0.0.0-20180207000608-0eeff89b0690/go.mod h1:Ulb78X89vxKYgdL24HMTiXYHlyHEvruOj1ZPlqeNEZM= +bou.ke/monkey v1.0.2 h1:kWcnsrCNUatbxncxR/ThdYqbytgOIArtYWqcQLQzKLI= +bou.ke/monkey v1.0.2/go.mod h1:OqickVX3tNx6t33n1xvtTtu85YN5s6cKwVug+oHMaIA= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -74,9 +76,7 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -191,6 +191,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= @@ -240,6 +241,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -278,9 +280,11 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -295,8 +299,8 @@ github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUe github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= -github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= @@ -408,8 +412,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= @@ -495,10 +499,12 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5/go.mod h1:DM4VvS+hD/kDi1U1QsX2fnZowwBhqD0Dk3bRPKF/Oc8= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s= github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -568,8 +574,9 @@ github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPK github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU= github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -585,8 +592,9 @@ github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= +github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= @@ -641,7 +649,6 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nishanths/exhaustive v0.0.0-20200811152831-6cf413ae40e0/go.mod h1:wBEpHwM2OdmeNpdCvRPUlkEbBuaFmcK4Wv8Q7FuGW3c= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -717,15 +724,17 @@ github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0 github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -735,8 +744,9 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -746,8 +756,9 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/quasilyte/go-ruleguard v0.2.0/go.mod h1:2RT/tf0Ce0UDj5y243iWKosQogJd8+1G3Rs2fxmlYnw= @@ -784,8 +795,9 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/slok/kubewebhook v0.11.0 h1:mRUOHXpMNxROTcqGVq06BQX2r13cT1Kjw0ylcJhTg0g= github.com/slok/kubewebhook v0.11.0/go.mod h1:HWkaQH3ZbQpLeP3ylW/NPhOaYByxCIRU36HPmUEoqyo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -808,8 +820,9 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= -github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= +github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -950,15 +963,17 @@ go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0H go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= @@ -1077,7 +1092,6 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1085,6 +1099,7 @@ golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= @@ -1103,8 +1118,9 @@ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1191,7 +1207,6 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1208,6 +1223,7 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1327,14 +1343,16 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= +gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= gomodules.xyz/jsonpatch/v3 v3.0.1 h1:Te7hKxV52TKCbNYq3t84tzKav3xhThdvSsSp/W89IyI= gomodules.xyz/jsonpatch/v3 v3.0.1/go.mod h1:CBhndykehEwTOlEfnsfJwvkFQbSN8YZFr9M+cIHAJto= gomodules.xyz/orderedmap v0.1.0 h1:fM/+TGh/O1KkqGR5xjTKg6bU8OKBkg7p0Y+x/J9m8Os= gomodules.xyz/orderedmap v0.1.0/go.mod h1:g9/TPUCm1t2gwD3j3zfV8uylyYhVdCNSi+xCEIu7yTU= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= -gonum.org/v1/gonum v0.6.2 h1:4r+yNT0+8SWcOkXP+63H2zQbN+USnC73cjGUxnDF94Q= gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e h1:jRyg0XfpwWlhEV8mDfdNGBeSJM2fuyh9Yjrnd8kF2Ts= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= @@ -1466,15 +1484,16 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1529,6 +1548,8 @@ k8s.io/apiserver v0.24.6 h1:LEVuJb3bwjbZSIA8Ltm6iNBJiFawJtEGmAP7bgMljIE= k8s.io/apiserver v0.24.6/go.mod h1:ZmMXlYsNUhWzEOMJv01JqjL/psUzmvcIF70yDnzen/A= k8s.io/autoscaler/cluster-autoscaler v0.0.0-20231010095923-8a61add71154 h1:UxhksRGQqiuHhuQsY935KuUAaRMG7yAFLz1BhrSbtWE= k8s.io/autoscaler/cluster-autoscaler v0.0.0-20231010095923-8a61add71154/go.mod h1:Yipa7m+SwYJwkNvtcxeI1SboxM5b8yecXvUw/pezXfk= +k8s.io/autoscaler/vertical-pod-autoscaler v1.0.0 h1:y0TgWoHaeYEv3L1MfLC+D2WVxyN1fGr6axURHXq+wHE= +k8s.io/autoscaler/vertical-pod-autoscaler v1.0.0/go.mod h1:w6/LjLR3DPQd57vlgvgbpzpuJKsCiily0+OzQI+nyfI= k8s.io/cli-runtime v0.24.6/go.mod h1:dZghAkzYOsDUDqz1pjFi1KDo+1ly1gKGrIPaX4HMHPs= k8s.io/client-go v0.24.6 h1:q7gZYyGL0Iv9zynYOFi5DHc3NFZ2aA0P56QpWFXbEyE= k8s.io/client-go v0.24.6/go.mod h1:qaJRTFlI24ONWGplf+j8IgTyb6ztpwS6SGfjcNCRpQ8= diff --git a/pkg/config/controller/controller_base.go b/pkg/config/controller/controller_base.go index 54f4b2c50..0d434bebf 100644 --- a/pkg/config/controller/controller_base.go +++ b/pkg/config/controller/controller_base.go @@ -49,6 +49,7 @@ type ControllersConfiguration struct { *MonitorConfig *OvercommitConfig *TideConfig + *ResourceRecommenderConfig } func NewGenericControllerConfiguration() *GenericControllerConfiguration { @@ -57,12 +58,13 @@ func NewGenericControllerConfiguration() *GenericControllerConfiguration { func NewControllersConfiguration() *ControllersConfiguration { return &ControllersConfiguration{ - VPAConfig: NewVPAConfig(), - KCCConfig: NewKCCConfig(), - SPDConfig: NewSPDConfig(), - LifeCycleConfig: NewLifeCycleConfig(), - MonitorConfig: NewMonitorConfig(), - OvercommitConfig: NewOvercommitConfig(), - TideConfig: NewTideConfig(), + VPAConfig: NewVPAConfig(), + KCCConfig: NewKCCConfig(), + SPDConfig: NewSPDConfig(), + LifeCycleConfig: NewLifeCycleConfig(), + MonitorConfig: NewMonitorConfig(), + OvercommitConfig: NewOvercommitConfig(), + TideConfig: NewTideConfig(), + ResourceRecommenderConfig: NewResourceRecommenderConfig(), } } diff --git a/pkg/config/controller/resourcerecommender.go b/pkg/config/controller/resourcerecommender.go new file mode 100644 index 000000000..8cc88d953 --- /dev/null +++ b/pkg/config/controller/resourcerecommender.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus" + +type ResourceRecommenderConfig struct { + OOMRecordMaxNumber int + + HealthProbeBindPort string + MetricsBindPort string + + // available datasource: prom + DataSource []string + // DataSourcePromConfig is the prometheus datasource config + DataSourcePromConfig prometheus.PromConfig + + // LogVerbosityLevel to specify log verbosity level. (The default level is 4) + // Set it to something larger than 4 if more detailed logs are needed. + LogVerbosityLevel string +} + +func NewResourceRecommenderConfig() *ResourceRecommenderConfig { + return &ResourceRecommenderConfig{} +} diff --git a/pkg/controller/resource-recommend/controller/controller.go b/pkg/controller/resource-recommend/controller/controller.go new file mode 100644 index 000000000..0c65fb119 --- /dev/null +++ b/pkg/controller/resource-recommend/controller/controller.go @@ -0,0 +1,193 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/config/controller" + "github.com/kubewharf/katalyst-core/pkg/config/generic" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/oom" + processormanager "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/manager" + recommendermanager "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/recommender/manager" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + + preCacheObjects = []client.Object{ + &v1alpha1.ResourceRecommend{}, + } +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(v1alpha1.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme +} + +type ResourceRecommender struct { + ctx context.Context + genericConf *generic.GenericConfiguration + opts *controller.ResourceRecommenderConfig +} + +func NewResourceRecommenderController(ctx context.Context, genericConf *generic.GenericConfiguration, opts *controller.ResourceRecommenderConfig) (*ResourceRecommender, error) { + return &ResourceRecommender{ + ctx: ctx, + genericConf: genericConf, + opts: opts, + }, nil +} + +func (r ResourceRecommender) Run() { + config := ctrl.GetConfigOrDie() + config.QPS = r.genericConf.ClientConnection.QPS + config.Burst = int(r.genericConf.ClientConnection.Burst) + + ctrlOptions := ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: ":" + r.opts.MetricsBindPort, + HealthProbeBindAddress: ":" + r.opts.HealthProbeBindPort, + Port: 9443, + } + + mgr, err := ctrl.NewManager(config, ctrlOptions) + if err != nil { + klog.Fatal(fmt.Sprintf("unable to start manager: %s", err)) + os.Exit(1) + } + + if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + klog.Fatal(fmt.Sprintf("unable to set up health check: %s", err)) + os.Exit(1) + } + if err = mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + klog.Fatal(fmt.Sprintf("unable to set up ready check: %s", err)) + os.Exit(1) + } + + dataProxy := initDataSources(r.opts) + klog.Infof("successfully init data proxy %v", *dataProxy) + + // Processor Manager + processorManager := processormanager.NewManager(dataProxy, mgr.GetClient()) + go func() { + defer func() { + if r := recover(); r != nil { + err = errors.Errorf("start processor panic: %v", r.(error)) + klog.Error(err) + panic(err) + } + }() + processorManager.StartProcess(r.ctx) + }() + + // OOM Recorder + podOOMRecorder := &PodOOMRecorderController{ + PodOOMRecorder: &oom.PodOOMRecorder{ + Client: mgr.GetClient(), + OOMRecordMaxNumber: r.opts.OOMRecordMaxNumber, + }, + } + if err = podOOMRecorder.SetupWithManager(mgr); err != nil { + klog.Fatal(fmt.Sprintf("Unable to create controller: %s", err)) + os.Exit(1) + } + go func() { + defer func() { + if r := recover(); r != nil { + err = errors.Errorf("Run oom recorder panic: %v", r.(error)) + klog.Error(err) + panic(err) + } + }() + for count := 0; count < 100; count++ { + cacheReady := mgr.GetCache().WaitForCacheSync(r.ctx) + if cacheReady { + break + } + time.Sleep(100 * time.Millisecond) + } + if err = podOOMRecorder.Run(r.ctx.Done()); err != nil { + klog.Warningf("Run oom recorder failed: %v", err) + } + }() + + recommenderManager := recommendermanager.NewManager(*processorManager, podOOMRecorder) + + for _, obj := range preCacheObjects { + _, _ = mgr.GetCache().GetInformer(context.TODO(), obj) + } + + // Resource Recommend Controller + resourceRecommendController := &ResourceRecommendController{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + ProcessorManager: processorManager, + RecommenderManager: recommenderManager, + } + + if err = resourceRecommendController.SetupWithManager(mgr); err != nil { + klog.Fatal(fmt.Sprintf("ResourceRecommend Controller unable to SetupWithManager, err: %s", err)) + os.Exit(1) + } + + //+kubebuilder:scaffold:builder + + setupLog.Info("starting manager") + if err = mgr.Start(r.ctx); err != nil { + klog.Fatal(fmt.Sprintf("problem running manager: %s", err)) + os.Exit(1) + } +} + +func initDataSources(opts *controller.ResourceRecommenderConfig) *datasource.Proxy { + dataProxy := datasource.NewProxy() + for _, datasourceProvider := range opts.DataSource { + switch datasourceProvider { + case string(datasource.PrometheusDatasource): + fallthrough + default: + // default is prom + prometheusProvider, err := prometheus.NewPrometheus(&opts.DataSourcePromConfig) + if err != nil { + klog.Exitf("unable to create datasource provider %v, err: %v", prometheusProvider, err) + panic(err) + } + dataProxy.RegisterDatasource(datasource.PrometheusDatasource, prometheusProvider) + } + } + return dataProxy +} diff --git a/pkg/controller/resource-recommend/controller/controller_test.go b/pkg/controller/resource-recommend/controller/controller_test.go new file mode 100644 index 000000000..f2332a463 --- /dev/null +++ b/pkg/controller/resource-recommend/controller/controller_test.go @@ -0,0 +1,86 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "reflect" + "testing" + "time" + + "bou.ke/monkey" + promapiv1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/stretchr/testify/mock" + + "github.com/kubewharf/katalyst-core/pkg/config/controller" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus" + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" +) + +type MockDatasource struct { + mock.Mock +} + +func (m *MockDatasource) ConvertMetricToQuery(metric datasourcetypes.Metric) (*datasourcetypes.Query, error) { + args := m.Called(metric) + return args.Get(0).(*datasourcetypes.Query), args.Error(1) +} + +func (m *MockDatasource) QueryTimeSeries(query *datasourcetypes.Query, start, end time.Time, step time.Duration) (*datasourcetypes.TimeSeries, error) { + args := m.Called(query, start, end, step) + return args.Get(0).(*datasourcetypes.TimeSeries), args.Error(1) +} + +func (m *MockDatasource) GetPromClient() promapiv1.API { + args := m.Called() + return args.Get(0).(promapiv1.API) +} + +func Test_initDataSources(t *testing.T) { + proxy := datasource.NewProxy() + mockDatasource := MockDatasource{} + proxy.RegisterDatasource(datasource.PrometheusDatasource, &mockDatasource) + type args struct { + opts *controller.ResourceRecommenderConfig + } + tests := []struct { + name string + args args + want *datasource.Proxy + }{ + { + name: "return_Datasource", + args: args{ + opts: &controller.ResourceRecommenderConfig{ + DataSource: []string{string(datasource.PrometheusDatasource)}, + }, + }, + want: proxy, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer monkey.UnpatchAll() + + monkey.Patch(prometheus.NewPrometheus, func(config *prometheus.PromConfig) (prometheus.PromDatasource, error) { return &mockDatasource, nil }) + + if got := initDataSources(tt.args.opts); !reflect.DeepEqual(got, tt.want) { + t.Errorf("initDataSources() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/controller/oom_recorder_controller.go b/pkg/controller/resource-recommend/controller/oom_recorder_controller.go new file mode 100644 index 000000000..400cf9e7f --- /dev/null +++ b/pkg/controller/resource-recommend/controller/oom_recorder_controller.go @@ -0,0 +1,97 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/oom" +) + +// PodOOMRecorderController reconciles a PodOOMRecorder object +type PodOOMRecorderController struct { + *oom.PodOOMRecorder +} + +//+kubebuilder:rbac:groups=recommendation.katalyst.kubewharf.io,resources=podOOMRecorder,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=recommendation.katalyst.kubewharf.io,resources=podOOMRecorder/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=recommendation.katalyst.kubewharf.io,resources=podOOMRecorder/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// reconcile takes an pod resource.It records information when a Pod experiences an OOM (Out of Memory) state. +// the ResourceRecommend object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.2/pkg/reconcile +func (r *PodOOMRecorderController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + klog.V(5).InfoS("Get pods", "NamespacedName", req.NamespacedName) + pod := &v1.Pod{} + err := r.Client.Get(ctx, req.NamespacedName, pod) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + for _, containerStatus := range pod.Status.ContainerStatuses { + if containerStatus.RestartCount > 0 && + containerStatus.LastTerminationState.Terminated != nil && + containerStatus.LastTerminationState.Terminated.Reason == "OOMKilled" { + if container := GetContainer(pod, containerStatus.Name); container != nil { + if memory, ok := container.Resources.Requests[v1.ResourceMemory]; ok { + r.Queue.Add(oom.OOMRecord{ + Namespace: pod.Namespace, + Pod: pod.Name, + Container: containerStatus.Name, + Memory: memory, + OOMAt: containerStatus.LastTerminationState.Terminated.FinishedAt.Time, + }) + klog.V(2).InfoS("Last termination state of the pod is oom", "namespace", pod.Namespace, + "pod", pod.Name, "container", containerStatus.Name, "MemoryRequest", memory) + } + } + } + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *PodOOMRecorderController) SetupWithManager(mgr ctrl.Manager) error { + r.Queue = workqueue.New() + return ctrl.NewControllerManagedBy(mgr). + WithOptions(controller.Options{ + RecoverPanic: true, + }). + For(&v1.Pod{}). + Complete(r) +} + +func GetContainer(pod *v1.Pod, containerName string) *v1.Container { + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == containerName { + return &pod.Spec.Containers[i] + } + } + return nil +} diff --git a/pkg/controller/resource-recommend/controller/oom_recorder_controller_test.go b/pkg/controller/resource-recommend/controller/oom_recorder_controller_test.go new file mode 100644 index 000000000..662ee5fe4 --- /dev/null +++ b/pkg/controller/resource-recommend/controller/oom_recorder_controller_test.go @@ -0,0 +1,80 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "reflect" + "testing" + + v1 "k8s.io/api/core/v1" +) + +func TestGetContainer(t *testing.T) { + type args struct { + pod *v1.Pod + containerName string + } + tests := []struct { + name string + args args + want *v1.Container + }{ + { + name: "notFount", + args: args{ + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "c1", + }, + }, + }, + }, + containerName: "cx", + }, + want: nil, + }, + { + name: "Got", + args: args{ + pod: &v1.Pod{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "c1", + Image: "image1", + }, + }, + }, + }, + containerName: "c1", + }, + want: &v1.Container{ + Name: "c1", + Image: "image1", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetContainer(tt.args.pod, tt.args.containerName); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetContainer() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/controller/resourcerecommend_controller.go b/pkg/controller/resource-recommend/controller/resourcerecommend_controller.go new file mode 100644 index 000000000..d9979abcd --- /dev/null +++ b/pkg/controller/resource-recommend/controller/resourcerecommend_controller.go @@ -0,0 +1,216 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "runtime/debug" + "time" + + "golang.org/x/time/rate" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + processormanager "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/manager" + recommendermanager "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/recommender/manager" + resourceutils "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/resource" + conditionstypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/conditions" + errortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" + recommendationtypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/recommendation" +) + +const ( + ExponentialFailureRateLimiterBaseDelay = time.Minute + ExponentialFailureRateLimiterMaxDelay = 30 * time.Minute + DefaultRecommendInterval = 24 * time.Hour +) + +// ResourceRecommendController reconciles a ResourceRecommend object +type ResourceRecommendController struct { + client.Client + Scheme *runtime.Scheme + ProcessorManager *processormanager.Manager + RecommenderManager *recommendermanager.Manager +} + +//+kubebuilder:rbac:groups=recommendation.katalyst.kubewharf.io,resources=resourcerecommends,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=recommendation.katalyst.kubewharf.io,resources=resourcerecommends/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=recommendation.katalyst.kubewharf.io,resources=resourcerecommends/finalizers,verbs=update + +// SetupWithManager sets up the controller with the Manager. +func (r *ResourceRecommendController) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&v1alpha1.ResourceRecommend{}). + // We will only focus on the event of the Spec update, filter update events for status and meta + WithEventFilter(predicate.GenerationChangedPredicate{}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: 10, + RecoverPanic: true, + RateLimiter: workqueue.NewMaxOfRateLimiter( + // For reconcile failures(i.e. reconcile return err), the retry time is (2*minutes)*2^ + // The maximum retry time is 24 hours + workqueue.NewItemExponentialFailureRateLimiter(ExponentialFailureRateLimiterBaseDelay, ExponentialFailureRateLimiterMaxDelay), + // 10 qps, 100 bucket size. This is only for retry speed and its only the overall factor (not per item) + &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)}, + ), + }). + Complete(r) +} + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// compare the state specified by the ResourceRecommend object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.2/pkg/reconcile +func (r *ResourceRecommendController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + klog.InfoS("Get resourceRecommend to reconcile", "req", req) + resourceRecommend := &v1alpha1.ResourceRecommend{} + err := r.Get(ctx, req.NamespacedName, resourceRecommend) + if err != nil { + if k8serrors.IsNotFound(err) { + // Object not found, return + klog.V(2).InfoS("ResourceRecommend has been deleted.", "req", req) + // CancelTasks err dno‘t need to be processed, because the Processor side has gc logic + _ = r.CancelTasks(req.NamespacedName) + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + if recommender := resourceRecommend.Spec.ResourcePolicy.AlgorithmPolicy.Recommender; recommender != "" && + recommender != recommendationtypes.DefaultRecommenderType { + klog.InfoS("ResourceRecommend is not controlled by the default controller", "req", req) + return ctrl.Result{}, nil + } + + if lastRecommendationTime := resourceRecommend.Status.LastRecommendationTime; lastRecommendationTime != nil { + requeueAfter := time.Until(lastRecommendationTime.Add(DefaultRecommendInterval)) + observedGeneration := resourceRecommend.Status.ObservedGeneration + if requeueAfter > time.Duration(0) && observedGeneration == resourceRecommend.GetGeneration() { + klog.InfoS("no spec change and not time to reconcile, skipping this reconcile", "requeueAfter", requeueAfter, "observedGeneration", observedGeneration, "generation", resourceRecommend.GetGeneration(), "resourceRecommendName", resourceRecommend.GetName()) + return ctrl.Result{RequeueAfter: requeueAfter}, nil + } + } + + err = r.doReconcile(ctx, req.NamespacedName, resourceRecommend) + if err != nil { + klog.ErrorS(err, "Failed to reconcile", "req", req) + return ctrl.Result{}, err + } + + klog.InfoS("reconcile succeeded, requeue after "+DefaultRecommendInterval.String(), "req", req) + //reconcile succeeded, requeue after 24hours + return ctrl.Result{RequeueAfter: DefaultRecommendInterval}, nil +} + +func (r *ResourceRecommendController) doReconcile(ctx context.Context, namespacedName k8stypes.NamespacedName, + resourceRecommend *v1alpha1.ResourceRecommend) (err error) { + recommendation := recommendationtypes.NewRecommendation(resourceRecommend) + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("%v", r) + klog.ErrorS(err, "doReconcile panic", "resourceRecommend", namespacedName, "stack", string(debug.Stack())) + return + } + updateStatusErr := r.UpdateRecommendationStatus(namespacedName, recommendation) + if err == nil { + err = updateStatusErr + } + }() + + //conduct validation + validationError := recommendation.SetConfig(ctx, r.Client, resourceRecommend) + if validationError != nil { + klog.ErrorS(validationError, "Failed to get Recommendation", "resourceRecommend", namespacedName) + recommendation.Conditions.Set(*conditionstypes.ConvertCustomErrorToCondition(*validationError)) + return validationError + } + //set the condition of the validation step to be true + recommendation.Conditions.Set(*conditionstypes.ValidationSucceededCondition()) + + //Initialization + if registerTaskErr := r.RegisterTasks(*recommendation); registerTaskErr != nil { + klog.ErrorS(registerTaskErr, "Failed to register process task", "resourceRecommend", namespacedName) + recommendation.Conditions.Set(*conditionstypes.ConvertCustomErrorToCondition(*registerTaskErr)) + return registerTaskErr + } + //set the condition of the initialization step to be true + recommendation.Conditions.Set(*conditionstypes.InitializationSucceededCondition()) + + //recommendation logic + defaultRecommender := r.RecommenderManager.NewRecommender(recommendation.AlgorithmPolicy.Algorithm) + if recommendationError := defaultRecommender.Recommend(recommendation); recommendationError != nil { + klog.ErrorS(recommendationError, "error when getting recommendation for resource", "resourceRecommend", namespacedName) + recommendation.Conditions.Set(*conditionstypes.ConvertCustomErrorToCondition(*recommendationError)) + return recommendationError + } + //set the condition of the recommendation step to be true + recommendation.Conditions.Set(*conditionstypes.RecommendationReadyCondition()) + return nil +} + +// RegisterTasks Register all process task +func (r *ResourceRecommendController) RegisterTasks(recommendation recommendationtypes.Recommendation) *errortypes.CustomError { + processor := r.ProcessorManager.GetProcessor(recommendation.AlgorithmPolicy.Algorithm) + for _, container := range recommendation.Containers { + for _, containerConfig := range container.ContainerConfigs { + processConfig := processortypes.NewProcessConfig(recommendation.NamespacedName, recommendation.Config.TargetRef, container.ContainerName, containerConfig.ControlledResource, "") + if err := processor.Register(processConfig); err != nil { + return errortypes.DataProcessRegisteredFailedError(err.Error()) + } + } + } + + return nil +} + +// CancelTasks Cancel all process task +func (r *ResourceRecommendController) CancelTasks(namespacedName k8stypes.NamespacedName) *errortypes.CustomError { + processor := r.ProcessorManager.GetProcessor(v1alpha1.AlgorithmPercentile) + err := processor.Cancel(&processortypes.ProcessKey{ResourceRecommendNamespacedName: namespacedName}) + if err != nil { + klog.ErrorS(err, "cancel processor task failed", "namespacedName", namespacedName) + } + return err +} + +func (r *ResourceRecommendController) UpdateRecommendationStatus(namespaceName k8stypes.NamespacedName, + recommendation *recommendationtypes.Recommendation) error { + updateStatus := &v1alpha1.ResourceRecommend{ + Status: recommendation.AsStatus(), + } + //record generation + updateStatus.Status.ObservedGeneration = recommendation.ObservedGeneration + + err := resourceutils.PatchUpdateResourceRecommend(r.Client, namespaceName, updateStatus) + if err != nil { + klog.ErrorS(err, "Update resourceRecommend status error") + } + return err +} diff --git a/pkg/controller/resource-recommend/controller/resourcerecommend_controller_test.go b/pkg/controller/resource-recommend/controller/resourcerecommend_controller_test.go new file mode 100644 index 000000000..f64ca79b7 --- /dev/null +++ b/pkg/controller/resource-recommend/controller/resourcerecommend_controller_test.go @@ -0,0 +1,236 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "bou.ke/monkey" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + processormanager "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/manager" + resourceutils "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/resource" + conditionstypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/conditions" + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" + errortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" + "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" + recommendationtypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/recommendation" +) + +func TestResourceRecommendController_UpdateRecommendationStatus(t *testing.T) { + type args struct { + namespaceName types.NamespacedName + recommendation *recommendationtypes.Recommendation + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "test_ObservedGeneration", + args: args{ + recommendation: &recommendationtypes.Recommendation{ + Conditions: &conditionstypes.ResourceRecommendConditionsMap{ + v1alpha1.Validated: { + Type: v1alpha1.Validated, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Date(2023, 3, 3, 3, 0, 0, 0, time.UTC)), + }, + v1alpha1.Initialized: { + Type: v1alpha1.Initialized, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Date(2023, 4, 4, 4, 0, 0, 0, time.UTC)), + Reason: "reason4", + Message: "test msg4", + }, + }, + Recommendations: []v1alpha1.ContainerResources{ + { + ContainerName: "c1", + }, + }, + ObservedGeneration: 543123451423, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer monkey.UnpatchAll() + + monkey.Patch(resourceutils.PatchUpdateResourceRecommend, func(client k8sclient.Client, namespaceName types.NamespacedName, + resourceRecommend *v1alpha1.ResourceRecommend) error { + if resourceRecommend.Status.ObservedGeneration != tt.args.recommendation.ObservedGeneration { + return errors.New("ObservedGeneration not update") + } + return nil + }) + + r := &ResourceRecommendController{} + if err := r.UpdateRecommendationStatus(tt.args.namespaceName, tt.args.recommendation); (err != nil) != tt.wantErr { + t.Errorf("UpdateRecommendationStatus() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +var gotProcessConfigList []processor.ProcessConfig + +type mockProcessor struct { + algorithm v1alpha1.Algorithm + runMark string +} + +func (p *mockProcessor) Run(_ context.Context) { return } + +func (p *mockProcessor) Register(processConfig *processor.ProcessConfig) *errortypes.CustomError { + gotProcessConfigList = append(gotProcessConfigList, *processConfig) + return nil +} + +func (p *mockProcessor) Cancel(_ *processor.ProcessKey) *errortypes.CustomError { return nil } + +func (p *mockProcessor) QueryProcessedValues(_ *processor.ProcessKey) (float64, error) { return 0, nil } + +func TestResourceRecommendController_RegisterTasks_AssignmentTest(t *testing.T) { + recommendation := recommendationtypes.Recommendation{ + NamespacedName: types.NamespacedName{ + Name: "rec1", + Namespace: "default", + }, + Config: recommendationtypes.Config{ + TargetRef: v1alpha1.CrossVersionObjectReference{ + Name: "demo", + Kind: "deployment", + APIVersion: "app/v1", + }, + Containers: []recommendationtypes.Container{ + { + ContainerName: "c1", + ContainerConfigs: []recommendationtypes.ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: 34, + }, + { + ControlledResource: v1.ResourceMemory, + ResourceBufferPercent: 43, + }, + }, + }, + { + ContainerName: "c2", + ContainerConfigs: []recommendationtypes.ContainerConfig{ + { + ControlledResource: v1.ResourceMemory, + ResourceBufferPercent: 53, + }, + }, + }, + }, + }, + } + + wantProcessConfigList := []processor.ProcessConfig{ + { + ProcessKey: processor.ProcessKey{ + ResourceRecommendNamespacedName: types.NamespacedName{ + Name: "rec1", + Namespace: "default", + }, + Metric: &datasourcetypes.Metric{ + Namespace: "default", + WorkloadName: "demo", + Kind: "deployment", + APIVersion: "app/v1", + ContainerName: "c1", + Resource: v1.ResourceCPU, + }, + }, + }, + { + ProcessKey: processor.ProcessKey{ + ResourceRecommendNamespacedName: types.NamespacedName{ + Name: "rec1", + Namespace: "default", + }, + Metric: &datasourcetypes.Metric{ + Namespace: "default", + WorkloadName: "demo", + Kind: "deployment", + APIVersion: "app/v1", + ContainerName: "c1", + Resource: v1.ResourceMemory, + }, + }, + }, + { + ProcessKey: processor.ProcessKey{ + ResourceRecommendNamespacedName: types.NamespacedName{ + Name: "rec1", + Namespace: "default", + }, + Metric: &datasourcetypes.Metric{ + Namespace: "default", + WorkloadName: "demo", + Kind: "deployment", + APIVersion: "app/v1", + ContainerName: "c2", + Resource: v1.ResourceMemory, + }, + }, + }, + } + + t.Run("AssignmentTest", func(t *testing.T) { + defer monkey.UnpatchAll() + + processor1 := &mockProcessor{} + manager := &processormanager.Manager{} + manager.ProcessorRegister(v1alpha1.AlgorithmPercentile, processor1) + r := &ResourceRecommendController{ + ProcessorManager: manager, + } + + if err := r.RegisterTasks(recommendation); err != nil { + t.Errorf("RegisterTasks Assignment failed") + } + + got, err1 := json.Marshal(gotProcessConfigList) + if err1 != nil { + t.Errorf("RegisterTasks Assignment got json.Marshal failed") + } + want, err2 := json.Marshal(wantProcessConfigList) + if err2 != nil { + t.Errorf("RegisterTasks Assignment want json.Marshal failed") + } + + if string(got) != string(want) { + t.Errorf("RegisterTasks Assignment processConfig failed, got: %s, want: %s", string(got), string(want)) + } + }) + +} diff --git a/pkg/controller/resource-recommend/datasource/datasource_proxy.go b/pkg/controller/resource-recommend/datasource/datasource_proxy.go new file mode 100644 index 000000000..b18353e4f --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/datasource_proxy.go @@ -0,0 +1,68 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package datasource + +import ( + "errors" + "time" + + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" +) + +type DatasourceType string + +const ( + PrometheusDatasource DatasourceType = "Prometheus" +) + +type Datasource interface { + QueryTimeSeries(query *datasourcetypes.Query, start time.Time, end time.Time, step time.Duration) (*datasourcetypes.TimeSeries, error) + ConvertMetricToQuery(metric datasourcetypes.Metric) (*datasourcetypes.Query, error) +} + +type Proxy struct { + datasourceMap map[DatasourceType]Datasource +} + +func NewProxy() *Proxy { + return &Proxy{ + datasourceMap: make(map[DatasourceType]Datasource), + } +} + +func (p *Proxy) RegisterDatasource(name DatasourceType, datasource Datasource) { + p.datasourceMap[name] = datasource +} + +func (p *Proxy) getDatasource(name DatasourceType) (Datasource, error) { + if datasource, ok := p.datasourceMap[name]; ok { + return datasource, nil + } + return nil, errors.New("datasource not found") +} + +func (p *Proxy) QueryTimeSeries(DatasourceName DatasourceType, metric datasourcetypes.Metric, start time.Time, end time.Time, step time.Duration) (*datasourcetypes.TimeSeries, error) { + datasource, err := p.getDatasource(DatasourceName) + if err != nil { + return nil, err + } + query, err := datasource.ConvertMetricToQuery(metric) + if err != nil { + return nil, err + } + return datasource.QueryTimeSeries(query, start, end, step) +} diff --git a/pkg/controller/resource-recommend/datasource/datasource_proxy_test.go b/pkg/controller/resource-recommend/datasource/datasource_proxy_test.go new file mode 100644 index 000000000..49e1943b6 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/datasource_proxy_test.go @@ -0,0 +1,109 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package datasource + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" +) + +type MockDatasource struct { + mock.Mock +} + +func (m *MockDatasource) ConvertMetricToQuery(metric datasourcetypes.Metric) (*datasourcetypes.Query, error) { + args := m.Called(metric) + return args.Get(0).(*datasourcetypes.Query), args.Error(1) +} + +func (m *MockDatasource) QueryTimeSeries(query *datasourcetypes.Query, start, end time.Time, step time.Duration) (*datasourcetypes.TimeSeries, error) { + args := m.Called(query, start, end, step) + return args.Get(0).(*datasourcetypes.TimeSeries), args.Error(1) +} + +func TestProxy_QueryTimeSeries(t *testing.T) { + // Create mock objects + mockDatasource := &MockDatasource{} + mockTimeSeries := &datasourcetypes.TimeSeries{} + + // Create Proxy with mock datasource + proxy := &Proxy{ + datasourceMap: map[DatasourceType]Datasource{ + DatasourceType("mock"): mockDatasource, + }, + } + + // Define test cases + testCases := []struct { + name string + datasource DatasourceType + metric datasourcetypes.Metric + start time.Time + end time.Time + step time.Duration + expectedTS *datasourcetypes.TimeSeries + expectedErr error + }{ + { + name: "Valid query", + datasource: "mock", + metric: datasourcetypes.Metric{}, + start: time.Now(), + end: time.Now(), + step: time.Second, + expectedTS: mockTimeSeries, + }, + { + name: "Error getting datasource", + datasource: "invalid", + metric: datasourcetypes.Metric{}, + start: time.Now(), + end: time.Now(), + step: time.Second, + expectedErr: errors.New("datasource not found"), + }, + } + + // Set up mock behaviors and perform tests + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Set up mock behaviors + mockDatasource.On("ConvertMetricToQuery", tc.metric).Return(&datasourcetypes.Query{}, nil) + if tc.expectedErr == nil { + mockDatasource.On("QueryTimeSeries", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(mockTimeSeries, nil) + } else { + mockDatasource.On("QueryTimeSeries", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, tc.expectedErr) + } + + // Call the QueryTimeSeries method + result, err := proxy.QueryTimeSeries(tc.datasource, tc.metric, tc.start, tc.end, tc.step) + + // Verify the results + assert.Equal(t, tc.expectedTS, result) + assert.Equal(t, tc.expectedErr, err) + + // Verify mock invocations + mockDatasource.AssertExpectations(t) + }) + } +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/auth/basic_auth.go b/pkg/controller/resource-recommend/datasource/prometheus/auth/basic_auth.go new file mode 100644 index 000000000..6777771e9 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/auth/basic_auth.go @@ -0,0 +1,43 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "fmt" + "net/http" + + "github.com/prometheus/common/config" + "k8s.io/klog/v2" +) + +type basicAuthRoundTripperProvider struct { + next http.RoundTripper +} + +func NewBasicAuthRoundTripperProvider(next http.RoundTripper) (RoundTripperFactory, error) { + return &basicAuthRoundTripperProvider{ + next: next, + }, nil +} + +func (p *basicAuthRoundTripperProvider) GetAuthRoundTripper(c *ClientAuth) (http.RoundTripper, error) { + klog.Info("prom auth using basic auth") + if len(c.Username) == 0 || len(c.Password) == 0 { + return nil, fmt.Errorf("invalid user name or password for basic auth") + } + return config.NewBasicAuthRoundTripper(c.Username, config.Secret(c.Password), "", p.next), nil +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/auth/bearer_token_auth.go b/pkg/controller/resource-recommend/datasource/prometheus/auth/bearer_token_auth.go new file mode 100644 index 000000000..2112757f6 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/auth/bearer_token_auth.go @@ -0,0 +1,43 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "fmt" + "net/http" + + "github.com/prometheus/common/config" + "k8s.io/klog/v2" +) + +type tokenAuthRoundTripperProvider struct { + next http.RoundTripper +} + +func NewTokenAuthRoundTripperProvider(next http.RoundTripper) (RoundTripperFactory, error) { + return &tokenAuthRoundTripperProvider{ + next: next, + }, nil +} + +func (p *tokenAuthRoundTripperProvider) GetAuthRoundTripper(c *ClientAuth) (http.RoundTripper, error) { + klog.Info("prom auth using bearer token") + if len(c.BearerToken) == 0 { + return nil, fmt.Errorf("invalid bearer token for bearer token auth") + } + return config.NewAuthorizationCredentialsRoundTripper("Bearer", config.Secret(c.BearerToken), p.next), nil +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/auth/factory.go b/pkg/controller/resource-recommend/datasource/prometheus/auth/factory.go new file mode 100644 index 000000000..cf8d53cba --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/auth/factory.go @@ -0,0 +1,87 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package auth + +import ( + "net/http" + "sync" + + "k8s.io/klog/v2" +) + +type AuthType string + +const ( + BasicAuth AuthType = "Basic" + BearerTokenAuth AuthType = "BearerToken" +) + +// ClientAuth holds the HTTP client identity info. +type ClientAuth struct { + Type string + Username string + BearerToken string + Password string +} + +func init() { + RegisterRoundTripperInitializer(BasicAuth, NewBasicAuthRoundTripperProvider) + RegisterRoundTripperInitializer(BearerTokenAuth, NewTokenAuthRoundTripperProvider) +} + +type RoundTripperFactory interface { + GetAuthRoundTripper(c *ClientAuth) (http.RoundTripper, error) +} + +var roundTripperInitializers sync.Map + +// RoundTripperFactoryInitFunc is the function to initialize a authRoundTripper +type RoundTripperFactoryInitFunc func(next http.RoundTripper) (RoundTripperFactory, error) + +// RegisterRoundTripperInitializer registers a authRoundTripper initializer function +// for a specific prometheus AuthType, the function will be called when the prometheus client +// is created. The function should return a RoundTripperFactory for the prometheus AuthType. If the +// function is called multiple times for the same AuthType, the last one will be used. +func RegisterRoundTripperInitializer(authType AuthType, initFunc RoundTripperFactoryInitFunc) { + roundTripperInitializers.Store(authType, initFunc) +} + +func GetRegisteredRoundTripperFactoryInitializers() map[AuthType]RoundTripperFactoryInitFunc { + roundTrippers := make(map[AuthType]RoundTripperFactoryInitFunc) + roundTripperInitializers.Range(func(key, value interface{}) bool { + roundTrippers[key.(AuthType)] = value.(RoundTripperFactoryInitFunc) + return true + }) + return roundTrippers +} + +func GetRoundTripper(c *ClientAuth, next http.RoundTripper, + initializers map[AuthType]RoundTripperFactoryInitFunc) (http.RoundTripper, error) { + + initFunc := initializers[AuthType(c.Type)] + roundTripperFactory, err := initFunc(next) + if err != nil { + klog.ErrorS(err, "get prometheus auth round tripper factory err", "AuthType", c.Type) + return nil, err + } + roundTripper, err := roundTripperFactory.GetAuthRoundTripper(c) + if err != nil { + klog.ErrorS(err, "get prometheus auth round tripper err", "AuthType", c.Type) + return nil, err + } + return roundTripper, nil +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/mock_prometheus_client.go b/pkg/controller/resource-recommend/datasource/prometheus/mock_prometheus_client.go new file mode 100644 index 000000000..1387f111f --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/mock_prometheus_client.go @@ -0,0 +1,114 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "context" + "time" + + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" +) + +// mockPromAPIClient is a mock implementation of v1.API for testing purposes. +type mockPromAPIClient struct { + QueryRangeFunc func(ctx context.Context, query string, r v1.Range) (model.Value, v1.Warnings, error) +} + +func (m *mockPromAPIClient) Alerts(ctx context.Context) (v1.AlertsResult, error) { + return v1.AlertsResult{}, nil +} + +func (m *mockPromAPIClient) AlertManagers(ctx context.Context) (v1.AlertManagersResult, error) { + return v1.AlertManagersResult{}, nil +} + +func (m *mockPromAPIClient) CleanTombstones(ctx context.Context) error { + return nil +} + +func (m *mockPromAPIClient) Config(ctx context.Context) (v1.ConfigResult, error) { + return v1.ConfigResult{}, nil +} + +func (m *mockPromAPIClient) DeleteSeries(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) error { + return nil +} + +func (m *mockPromAPIClient) Flags(ctx context.Context) (v1.FlagsResult, error) { + return nil, nil +} + +func (m *mockPromAPIClient) LabelNames(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]string, v1.Warnings, error) { + return nil, nil, nil +} + +func (m *mockPromAPIClient) LabelValues(ctx context.Context, label string, matches []string, startTime time.Time, endTime time.Time) (model.LabelValues, v1.Warnings, error) { + return nil, nil, nil +} + +func (m *mockPromAPIClient) Query(ctx context.Context, query string, ts time.Time, opts ...v1.Option) (model.Value, v1.Warnings, error) { + return nil, nil, nil +} + +func (m *mockPromAPIClient) QueryExemplars(ctx context.Context, query string, startTime time.Time, endTime time.Time) ([]v1.ExemplarQueryResult, error) { + return nil, nil +} + +func (m *mockPromAPIClient) Buildinfo(ctx context.Context) (v1.BuildinfoResult, error) { + return v1.BuildinfoResult{}, nil +} + +func (m *mockPromAPIClient) Runtimeinfo(ctx context.Context) (v1.RuntimeinfoResult, error) { + return v1.RuntimeinfoResult{}, nil +} + +func (m *mockPromAPIClient) Series(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]model.LabelSet, v1.Warnings, error) { + return nil, nil, nil +} + +func (m *mockPromAPIClient) Snapshot(ctx context.Context, skipHead bool) (v1.SnapshotResult, error) { + return v1.SnapshotResult{}, nil +} + +func (m *mockPromAPIClient) Rules(ctx context.Context) (v1.RulesResult, error) { + return v1.RulesResult{}, nil +} + +func (m *mockPromAPIClient) Targets(ctx context.Context) (v1.TargetsResult, error) { + return v1.TargetsResult{}, nil +} + +func (m *mockPromAPIClient) TargetsMetadata(ctx context.Context, matchTarget string, metric string, limit string) ([]v1.MetricMetadata, error) { + return nil, nil +} + +func (m *mockPromAPIClient) Metadata(ctx context.Context, metric string, limit string) (map[string][]v1.Metadata, error) { + return nil, nil +} + +func (m *mockPromAPIClient) TSDB(ctx context.Context) (v1.TSDBResult, error) { + return v1.TSDBResult{}, nil +} + +func (m *mockPromAPIClient) WalReplay(ctx context.Context) (v1.WalReplayStatus, error) { + return v1.WalReplayStatus{}, nil +} + +func (m *mockPromAPIClient) QueryRange(ctx context.Context, query string, r v1.Range, opts ...v1.Option) (model.Value, v1.Warnings, error) { + return m.QueryRangeFunc(ctx, query, r) +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/prometheus_client.go b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_client.go new file mode 100644 index 000000000..90e603ca9 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_client.go @@ -0,0 +1,100 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "context" + "crypto/tls" + "net" + "net/http" + "net/url" + "time" + + prometheusapi "github.com/prometheus/client_golang/api" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus/auth" +) + +// PromConfig represents the config of prometheus +type PromConfig struct { + Address string + Timeout time.Duration + KeepAlive time.Duration + InsecureSkipVerify bool + Auth auth.ClientAuth + + QueryConcurrency int + BRateLimit bool + MaxPointsLimitPerTimeSeries int + TLSHandshakeTimeoutInSecond time.Duration + + BaseFilter string +} + +type prometheusAuthClient struct { + client prometheusapi.Client +} + +// URL implements prometheus client interface +func (p *prometheusAuthClient) URL(ep string, args map[string]string) *url.URL { + return p.client.URL(ep, args) +} + +// Do implements prometheus client interface, wrapped with an auth info +func (p *prometheusAuthClient) Do(ctx context.Context, request *http.Request) (*http.Response, []byte, error) { + return p.client.Do(ctx, request) +} + +// NewPrometheusClient returns a prometheus.Client +func NewPrometheusClient(config *PromConfig) (prometheusapi.Client, error) { + + tlsConfig := &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify} + + var rt http.RoundTripper = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: config.Timeout, + KeepAlive: config.KeepAlive, + }).DialContext, + TLSHandshakeTimeout: config.TLSHandshakeTimeoutInSecond, + TLSClientConfig: tlsConfig, + } + + t, err := auth.GetRoundTripper(&config.Auth, rt, auth.GetRegisteredRoundTripperFactoryInitializers()) + if err != nil { + return nil, err + } + + pc := prometheusapi.Config{ + Address: config.Address, + RoundTripper: t, + } + return newPrometheusAuthClient(pc) +} + +func newPrometheusAuthClient(config prometheusapi.Config) (prometheusapi.Client, error) { + c, err := prometheusapi.NewClient(config) + if err != nil { + return nil, err + } + + client := &prometheusAuthClient{ + client: c, + } + + return client, nil +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/prometheus_provider.go b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_provider.go new file mode 100644 index 000000000..c48dc717e --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_provider.go @@ -0,0 +1,151 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "context" + "fmt" + "time" + + promapiv1 "github.com/prometheus/client_golang/api/prometheus/v1" + datamodel "github.com/prometheus/common/model" + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/util/general" + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" +) + +type prometheus struct { + promAPIClient promapiv1.API + config *PromConfig +} + +type PromDatasource interface { + datasource.Datasource + GetPromClient() promapiv1.API +} + +// NewPrometheus return a prometheus data source +func NewPrometheus(config *PromConfig) (PromDatasource, error) { + + client, err := NewPrometheusClient(config) + if err != nil { + return nil, err + } + promAPIClient := promapiv1.NewAPI(client) + + return &prometheus{promAPIClient: promAPIClient, config: config}, nil +} + +func (p *prometheus) GetPromClient() promapiv1.API { + return p.promAPIClient +} + +func (p *prometheus) ConvertMetricToQuery(metric datasourcetypes.Metric) (*datasourcetypes.Query, error) { + extraFilters := GetExtraFilters(metric.Selectors, p.config.BaseFilter) + var queryExpr string + switch metric.Resource { + case v1.ResourceCPU: + queryExpr = GetContainerCpuUsageQueryExp(metric.Namespace, metric.WorkloadName, metric.Kind, metric.ContainerName, extraFilters) + case v1.ResourceMemory: + queryExpr = GetContainerMemUsageQueryExp(metric.Namespace, metric.WorkloadName, metric.Kind, metric.ContainerName, extraFilters) + default: + return nil, fmt.Errorf("query for resource type %v is not supported", metric.Resource) + } + convertedQuery := datasourcetypes.PrometheusQuery{ + Query: queryExpr, + } + return &datasourcetypes.Query{ + Prometheus: &convertedQuery, + }, nil +} + +func (p *prometheus) QueryTimeSeries(query *datasourcetypes.Query, start time.Time, end time.Time, step time.Duration) (*datasourcetypes.TimeSeries, error) { + klog.InfoS("QueryTimeSeries", "query", general.StructToString(query), "start", start, "end", end) + timeoutCtx, cancelFunc := context.WithTimeout(context.Background(), p.config.Timeout) + defer cancelFunc() + timeSeries, err := p.queryRangeSync(timeoutCtx, query.Prometheus.Query, start, end, step) + if err != nil { + klog.ErrorS(err, "query", query, "start", start, "end", end) + return nil, err + } + return timeSeries, nil +} + +// queryRangeSync range query prometheus in sync way +func (p *prometheus) queryRangeSync(ctx context.Context, query string, start, end time.Time, step time.Duration) (*datasourcetypes.TimeSeries, error) { + r := promapiv1.Range{ + Start: start, + End: end, + Step: step, + } + klog.InfoS("Prom query", "query", query) + var ts *datasourcetypes.TimeSeries + results, warnings, err := p.promAPIClient.QueryRange(ctx, query, r) + if len(warnings) != 0 { + klog.InfoS("Prom query range warnings", "warnings", warnings) + } + if err != nil { + return ts, err + } + klog.V(5).InfoS("Prom query range result", "query", query, "result", results.String(), "resultsType", results.Type()) + + return p.convertPromResultsToTimeSeries(results) +} + +func (p *prometheus) convertPromResultsToTimeSeries(value datamodel.Value) (*datasourcetypes.TimeSeries, error) { + results := datasourcetypes.NewTimeSeries() + typeValue := value.Type() + switch typeValue { + case datamodel.ValMatrix: + if matrix, ok := value.(datamodel.Matrix); ok { + for _, sampleStream := range matrix { + if sampleStream == nil { + continue + } + for key, val := range sampleStream.Metric { + results.AppendLabel(string(key), string(val)) + } + for _, pair := range sampleStream.Values { + results.AppendSample(int64(pair.Timestamp/1000), float64(pair.Value)) + } + } + return results, nil + } else { + return results, fmt.Errorf("prometheus value type is %v, but assert failed", typeValue) + } + + case datamodel.ValVector: + if vector, ok := value.(datamodel.Vector); ok { + for _, sample := range vector { + if sample == nil { + continue + } + for key, val := range sample.Metric { + results.AppendLabel(string(key), string(val)) + } + results.AppendSample(int64(sample.Timestamp/1000), float64(sample.Value)) + } + return results, nil + } else { + return results, fmt.Errorf("prometheus value type is %v, but assert failed", typeValue) + } + } + return results, fmt.Errorf("prometheus return unsupported model value type %v", typeValue) +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/prometheus_provider_test.go b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_provider_test.go new file mode 100644 index 000000000..403080675 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_provider_test.go @@ -0,0 +1,455 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "context" + "fmt" + "reflect" + "testing" + "time" + + promapiv1 "github.com/prometheus/client_golang/api/prometheus/v1" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" + corev1 "k8s.io/api/core/v1" + + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" +) + +func Test_prometheus_ConvertMetricToQuery(t *testing.T) { + p := &prometheus{ + config: &PromConfig{}, + } + tests := []struct { + name string + metric datasourcetypes.Metric + expectedQuery *datasourcetypes.Query + expectedError error + }{ + { + name: "CPU metric", + metric: datasourcetypes.Metric{ + Resource: corev1.ResourceCPU, + Namespace: "my-namespace", + WorkloadName: "my-workload", + Kind: "Deployment", + ContainerName: "my-container", + }, + expectedQuery: &datasourcetypes.Query{ + Prometheus: &datasourcetypes.PrometheusQuery{ + Query: GetContainerCpuUsageQueryExp("my-namespace", "my-workload", "Deployment", "my-container", ""), + }, + }, + expectedError: nil, + }, + { + name: "Memory metric", + metric: datasourcetypes.Metric{ + Resource: corev1.ResourceMemory, + Namespace: "my-namespace", + WorkloadName: "my-workload", + Kind: "Deployment", + ContainerName: "my-container", + }, + expectedQuery: &datasourcetypes.Query{ + Prometheus: &datasourcetypes.PrometheusQuery{ + Query: GetContainerMemUsageQueryExp("my-namespace", "my-workload", "Deployment", "my-container", ""), + }, + }, + expectedError: nil, + }, + { + name: "Unsupported metric", + metric: datasourcetypes.Metric{ + Resource: "Unsupported", + Namespace: "my-namespace", + WorkloadName: "my-workload", + Kind: "Deployment", + ContainerName: "my-container", + }, + expectedQuery: nil, + expectedError: fmt.Errorf("query for resource type Unsupported is not supported"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotQuery, gotErr := p.ConvertMetricToQuery(tt.metric) + + if !reflect.DeepEqual(gotQuery, tt.expectedQuery) { + t.Errorf("ConvertMetricToQuery() gotQuery = %v, want %v", gotQuery, tt.expectedQuery) + } + + if (gotErr == nil && tt.expectedError != nil) || (gotErr != nil && tt.expectedError == nil) || (gotErr != nil && tt.expectedError != nil && gotErr.Error() != tt.expectedError.Error()) { + t.Errorf("ConvertMetricToQuery() gotErr = %v, wantErr %v", gotErr, tt.expectedError) + } + }) + } +} + +func Test_prometheus_QueryTimeSeries(t *testing.T) { + type fields struct { + promAPIClient v1.API + config *PromConfig + } + type args struct { + query *datasourcetypes.Query + start time.Time + end time.Time + step time.Duration + } + tests := []struct { + name string + fields fields + args args + want *datasourcetypes.TimeSeries + wantErr bool + }{ + { + name: "Successful query time series", + fields: fields{ + promAPIClient: &mockPromAPIClient{ + QueryRangeFunc: func(ctx context.Context, query string, r promapiv1.Range) (model.Value, promapiv1.Warnings, error) { + matrix := model.Matrix{ + &model.SampleStream{ + Metric: model.Metric{ + "label1": "value1", + "label2": "value2", + }, + Values: []model.SamplePair{ + { + Timestamp: 1500000000, + Value: 10, + }, + { + Timestamp: 1500001000, + Value: 20, + }, + }, + }, + } + return matrix, nil, nil + }, + }, + config: &PromConfig{ + Timeout: time.Second * 5, + }, + }, + args: args{ + query: &datasourcetypes.Query{ + Prometheus: &datasourcetypes.PrometheusQuery{ + Query: "my_query", + }, + }, + start: time.Unix(1500000000, 0), + end: time.Unix(1500002000, 0), + step: time.Second, + }, + want: &datasourcetypes.TimeSeries{ + Samples: []datasourcetypes.Sample{ + { + Value: 10, + Timestamp: 1500000, + }, + { + Value: 20, + Timestamp: 1500001, + }, + }, + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + }, + wantErr: false, + }, + { + name: "Query time series error", + fields: fields{ + promAPIClient: &mockPromAPIClient{ + QueryRangeFunc: func(ctx context.Context, query string, r promapiv1.Range) (model.Value, promapiv1.Warnings, error) { + return nil, nil, fmt.Errorf("query error") + }, + }, + config: &PromConfig{ + Timeout: time.Second * 5, + }, + }, + args: args{ + query: &datasourcetypes.Query{ + Prometheus: &datasourcetypes.PrometheusQuery{ + Query: "my_query", + }, + }, + start: time.Unix(1500000000, 0), + end: time.Unix(1500002000, 0), + step: time.Second, + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &prometheus{ + promAPIClient: tt.fields.promAPIClient, + config: tt.fields.config, + } + got, err := p.QueryTimeSeries(tt.args.query, tt.args.start, tt.args.end, tt.args.step) + if (err != nil) != tt.wantErr { + t.Errorf("QueryTimeSeries() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("QueryTimeSeries() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_prometheus_convertPromResultsToTimeSeries(t *testing.T) { + type fields struct { + promAPIClient v1.API + config *PromConfig + } + type args struct { + value model.Value + } + tests := []struct { + name string + fields fields + args args + want *datasourcetypes.TimeSeries + wantErr bool + }{ + { + name: "Matrix value", + fields: fields{ + promAPIClient: nil, + config: nil, + }, + args: args{ + value: model.Matrix{ + &model.SampleStream{ + Metric: model.Metric{ + "label1": "value1", + "label2": "value2", + }, + Values: []model.SamplePair{ + { + Timestamp: 1500000000, + Value: 10, + }, + { + Timestamp: 1500001000, + Value: 20, + }, + }, + }, + }, + }, + want: &datasourcetypes.TimeSeries{ + Samples: []datasourcetypes.Sample{ + { + Value: 10, + Timestamp: 1500000, + }, + { + Value: 20, + Timestamp: 1500001, + }, + }, + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + }, + wantErr: false, + }, + { + name: "Vector value", + fields: fields{ + promAPIClient: nil, + config: nil, + }, + args: args{ + value: model.Vector{ + &model.Sample{ + Metric: model.Metric{ + "label1": "value1", + "label2": "value2", + }, + Timestamp: 1500000000, + Value: 10, + }, + }, + }, + want: &datasourcetypes.TimeSeries{ + Samples: []datasourcetypes.Sample{ + { + Value: 10, + Timestamp: 1500000, + }, + }, + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + }, + wantErr: false, + }, + { + name: "Unsupported value type", + fields: fields{ + promAPIClient: nil, + config: nil, + }, + args: args{ + value: &model.Scalar{}, + }, + want: datasourcetypes.NewTimeSeries(), + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &prometheus{ + promAPIClient: tt.fields.promAPIClient, + config: tt.fields.config, + } + got, err := p.convertPromResultsToTimeSeries(tt.args.value) + if (err != nil) != tt.wantErr { + t.Errorf("convertPromResultsToTimeSeries() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("convertPromResultsToTimeSeries() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_prometheus_queryRangeSync(t *testing.T) { + type fields struct { + promAPIClient v1.API + config *PromConfig + } + type args struct { + ctx context.Context + query string + start time.Time + end time.Time + step time.Duration + } + tests := []struct { + name string + fields fields + args args + want *datasourcetypes.TimeSeries + wantErr bool + }{ + { + name: "Successful query range", + fields: fields{ + promAPIClient: &mockPromAPIClient{ + QueryRangeFunc: func(ctx context.Context, query string, r promapiv1.Range) (model.Value, promapiv1.Warnings, error) { + matrix := model.Matrix{ + &model.SampleStream{ + Metric: model.Metric{ + "label1": "value1", + "label2": "value2", + }, + Values: []model.SamplePair{ + { + Timestamp: 1500000000, + Value: 10, + }, + { + Timestamp: 1500001000, + Value: 20, + }, + }, + }, + } + return matrix, nil, nil + }, + }, + config: nil, + }, + args: args{ + ctx: context.TODO(), + query: "my_query", + start: time.Unix(1500000000, 0), + end: time.Unix(1500002000, 0), + step: time.Second, + }, + want: &datasourcetypes.TimeSeries{ + Samples: []datasourcetypes.Sample{ + { + Value: 10, + Timestamp: 1500000, + }, + { + Value: 20, + Timestamp: 1500001, + }, + }, + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + }, + wantErr: false, + }, + { + name: "Error in query range", + fields: fields{ + promAPIClient: &mockPromAPIClient{ + QueryRangeFunc: func(ctx context.Context, query string, r promapiv1.Range) (model.Value, promapiv1.Warnings, error) { + return nil, nil, fmt.Errorf("query error") + }, + }, + config: nil, + }, + args: args{ + ctx: context.TODO(), + query: "my_query", + start: time.Unix(1500000000, 0), + end: time.Unix(1500002000, 0), + step: time.Second, + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &prometheus{ + promAPIClient: tt.fields.promAPIClient, + config: tt.fields.config, + } + got, err := p.queryRangeSync(tt.args.ctx, tt.args.query, tt.args.start, tt.args.end, tt.args.step) + if (err != nil) != tt.wantErr { + t.Errorf("queryRangeSync() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("queryRangeSync() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/prometheus_query.go b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_query.go new file mode 100644 index 000000000..27e5c7524 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_query.go @@ -0,0 +1,68 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "fmt" + + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" +) + +const ( + // ContainerCpuUsageQueryExpr is used to query container cpu usage by promql + ContainerCpuUsageQueryExpr = `rate(container_cpu_usage_seconds_total{container!="POD",namespace="%s",pod=~"%s",container="%s"%s}[30s])` + // ContainerMemUsageQueryExpr is used to query container cpu usage by promql + ContainerMemUsageQueryExpr = `container_memory_working_set_bytes{container!="POD",namespace="%s",pod=~"%s",container="%s"%s}` +) + +const ( + WorkloadSuffixRuleForDeployment = `[a-z0-9]+-[a-z0-9]{5}$` +) + +func GetContainerCpuUsageQueryExp(namespace string, workloadName string, kind string, containerName string, extraFilters string) string { + if len(extraFilters) != 0 { + extraLabels := fmt.Sprintf(",%s", extraFilters) + return fmt.Sprintf(ContainerCpuUsageQueryExpr, namespace, convertWorkloadNameToPods(workloadName, kind), containerName, extraLabels) + } + return fmt.Sprintf(ContainerCpuUsageQueryExpr, namespace, convertWorkloadNameToPods(workloadName, kind), containerName, extraFilters) +} + +func GetContainerMemUsageQueryExp(namespace string, workloadName string, kind string, containerName string, extraFilters string) string { + if len(extraFilters) != 0 { + extraLabels := fmt.Sprintf(",%s", extraFilters) + return fmt.Sprintf(ContainerMemUsageQueryExpr, namespace, convertWorkloadNameToPods(workloadName, kind), containerName, extraLabels) + } + return fmt.Sprintf(ContainerMemUsageQueryExpr, namespace, convertWorkloadNameToPods(workloadName, kind), containerName, extraFilters) +} + +func convertWorkloadNameToPods(workloadName string, workloadKind string) string { + switch workloadKind { + case string(datasourcetypes.WorkloadDeployment): + return fmt.Sprintf("^%s-%s", workloadName, WorkloadSuffixRuleForDeployment) + } + return fmt.Sprintf("^%s-%s", workloadName, `.*`) +} + +func GetExtraFilters(extraFilters string, baseFilter string) string { + if extraFilters == "" { + return baseFilter + } + if baseFilter == "" { + return extraFilters + } + return extraFilters + "," + baseFilter +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/prometheus_query_test.go b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_query_test.go new file mode 100644 index 000000000..bb5052b9d --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_query_test.go @@ -0,0 +1,188 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "testing" + + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" +) + +func TestGetContainerCpuUsageQueryExp(t *testing.T) { + type args struct { + namespace string + workloadName string + kind string + containerName string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "Deployment workload kind", + args: args{ + namespace: "my-namespace", + workloadName: "my-deployment", + kind: string(datasourcetypes.WorkloadDeployment), + containerName: "test", + }, + want: "rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=\"my-namespace\",pod=~\"^my-deployment-[a-z0-9]+-[a-z0-9]{5}$\",container=\"test\"}[30s])", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetContainerCpuUsageQueryExp(tt.args.namespace, tt.args.workloadName, tt.args.kind, tt.args.containerName, ""); got != tt.want { + t.Errorf("GetContainerCpuUsageQueryExp() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetContainerMemUsageQueryExp(t *testing.T) { + type args struct { + namespace string + workloadName string + kind string + containerName string + extraFilter string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "Deployment workload kind", + args: args{ + namespace: "my-namespace", + workloadName: "my-deployment", + kind: string(datasourcetypes.WorkloadDeployment), + containerName: "test", + extraFilter: "", + }, + want: "container_memory_working_set_bytes{container!=\"POD\",namespace=\"my-namespace\",pod=~\"^my-deployment-[a-z0-9]+-[a-z0-9]{5}$\",container=\"test\"}", + }, + { + name: "Deployment workload kind", + args: args{ + namespace: "my-namespace", + workloadName: "my-deployment", + kind: string(datasourcetypes.WorkloadDeployment), + containerName: "test2", + extraFilter: "key1=\"value\"", + }, + want: "container_memory_working_set_bytes{container!=\"POD\",namespace=\"my-namespace\",pod=~\"^my-deployment-[a-z0-9]+-[a-z0-9]{5}$\",container=\"test2\",key1=\"value\"}", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetContainerMemUsageQueryExp(tt.args.namespace, tt.args.workloadName, tt.args.kind, tt.args.containerName, tt.args.extraFilter); got != tt.want { + t.Errorf("GetContainerMemUsageQueryExp() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_convertWorkloadNameToPods(t *testing.T) { + type args struct { + workloadName string + workloadKind string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "Deployment workload kind", + args: args{ + workloadName: "my-deployment", + workloadKind: string(datasourcetypes.WorkloadDeployment), + }, + want: "^my-deployment-[a-z0-9]+-[a-z0-9]{5}$", + }, + { + name: "Unknown workload kind", + args: args{ + workloadName: "my-workload", + workloadKind: "unknown", + }, + want: "^my-workload-.*", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := convertWorkloadNameToPods(tt.args.workloadName, tt.args.workloadKind); got != tt.want { + t.Errorf("convertWorkloadNameToPods() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetExtraFilters(t *testing.T) { + type args struct { + extraFilters string + baseFilter string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "all is empty", + args: args{ + extraFilters: "", + baseFilter: "", + }, + want: "", + }, + { + name: "extraFilters is empty", + args: args{ + extraFilters: "", + baseFilter: "cluster=\"cfeaf782fasdfe\"", + }, + want: "cluster=\"cfeaf782fasdfe\"", + }, + { + name: "baseFilter is empty", + args: args{ + extraFilters: "service=\"Katalyst\"", + baseFilter: "", + }, + want: "service=\"Katalyst\"", + }, + { + name: "all is not empty", + args: args{ + extraFilters: "service=\"Katalyst\"", + baseFilter: "cluster=\"cfeaf782fasdfe\"", + }, + want: "service=\"Katalyst\",cluster=\"cfeaf782fasdfe\"", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetExtraFilters(tt.args.extraFilters, tt.args.baseFilter); got != tt.want { + t.Errorf("GetExtraFilters() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/oom/oom_recorder.go b/pkg/controller/resource-recommend/oom/oom_recorder.go new file mode 100644 index 000000000..b875603eb --- /dev/null +++ b/pkg/controller/resource-recommend/oom/oom_recorder.go @@ -0,0 +1,222 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oom + +import ( + "context" + "encoding/json" + "sort" + "sync" + "time" + + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + ConfigMapOOMRecordName = "oom-record" + ConfigMapDataOOMRecord = "oom-data" + ConfigMapOOMRecordNameSpace = "kube-system" + CacheCleanTimeDurationHour = 12 + DataRetentionHour = 168 +) + +type Recorder interface { + ListOOMRecords() []OOMRecord +} + +type OOMRecord struct { + Namespace string + Pod string + Container string + Memory resource.Quantity + OOMAt time.Time +} + +type PodOOMRecorder struct { + client.Client + + mu sync.Mutex + + OOMRecordMaxNumber int + cache []OOMRecord + Queue workqueue.Interface +} + +func (r *PodOOMRecorder) initOOMCacheFromConfigmap() { + r.mu.Lock() + defer r.mu.Unlock() + + oomRecords, err := r.ListOOMRecordsFromConfigmap() + if err != nil { + // TODO: add monitor metric + klog.ErrorS(err, "init cache from configmap failed") + } + r.cache = oomRecords +} + +func (r *PodOOMRecorder) ListOOMRecords() []OOMRecord { + return r.cache +} + +func (r *PodOOMRecorder) cleanOOMRecord() { + r.mu.Lock() + defer r.mu.Unlock() + oomCache := r.ListOOMRecords() + sort.Slice(oomCache, func(i, j int) bool { + return oomCache[i].OOMAt.Before(oomCache[j].OOMAt) + }) + now := time.Now() + index := 0 + for i := len(oomCache) - 1; i >= 0; i-- { + if oomCache[i].OOMAt.Before(now.Add(-DataRetentionHour * time.Hour)) { + break + } + index++ + if index >= r.OOMRecordMaxNumber { + break + } + } + r.cache = oomCache[len(oomCache)-index:] +} + +func (r *PodOOMRecorder) updateOOMRecordCache(oomRecord OOMRecord) bool { + r.mu.Lock() + defer r.mu.Unlock() + + oomCache := r.ListOOMRecords() + if oomCache == nil { + oomCache = []OOMRecord{} + } + + isFound := false + isUpdated := false + for i := range oomCache { + if oomCache[i].Namespace == oomRecord.Namespace && oomCache[i].Pod == oomRecord.Pod && oomCache[i].Container == oomRecord.Container { + if oomRecord.Memory.Value() >= oomCache[i].Memory.Value() && !oomRecord.OOMAt.Equal(oomCache[i].OOMAt) { + oomCache[i].Memory = oomRecord.Memory + oomCache[i].OOMAt = oomRecord.OOMAt + isUpdated = true + } + isFound = true + break + } + } + + if !isFound { + oomCache = append(oomCache, oomRecord) + isUpdated = true + } + if isUpdated { + r.cache = oomCache + } + return isUpdated +} + +func (r *PodOOMRecorder) updateOOMRecordConfigMap() error { + r.mu.Lock() + defer r.mu.Unlock() + + oomCache := r.ListOOMRecords() + cacheData, err := json.Marshal(oomCache) + if err != nil { + return err + } + oomConfigMap := &v1.ConfigMap{} + err = r.Client.Get(context.TODO(), types.NamespacedName{Namespace: ConfigMapOOMRecordNameSpace, + Name: ConfigMapOOMRecordName}, oomConfigMap) + if err != nil { + if client.IgnoreNotFound(err) != nil { + return err + } + oomConfigMap.Name = ConfigMapOOMRecordName + oomConfigMap.Namespace = ConfigMapOOMRecordNameSpace + oomConfigMap.Data = map[string]string{ + ConfigMapDataOOMRecord: string(cacheData), + } + return r.Client.Create(context.TODO(), oomConfigMap) + } + oomConfigMap.Data = map[string]string{ + ConfigMapDataOOMRecord: string(cacheData), + } + return r.Client.Update(context.TODO(), oomConfigMap) +} + +func (r *PodOOMRecorder) Run(stopCh <-chan struct{}) error { + r.initOOMCacheFromConfigmap() + cleanTicker := time.NewTicker(time.Duration(CacheCleanTimeDurationHour) * time.Hour) + go func() { + defer func() { + if r := recover(); r != nil { + if r := recover(); r != nil { + err := errors.Errorf("Run clean oom recorder panic: %v", r.(error)) + klog.Error(err) + panic(err) + } + } + }() + for range cleanTicker.C { + r.cleanOOMRecord() + } + }() + for { + select { + case <-stopCh: + return nil + default: + } + + record, shutdown := r.Queue.Get() + if shutdown { + return errors.New("queue of OOMRecord recorder is shutting down ! ") + } + oomRecord, ok := record.(OOMRecord) + if !ok { + klog.Error("type conversion failed") + r.Queue.Done(record) + continue + } + isUpdated := r.updateOOMRecordCache(oomRecord) + if !isUpdated { + r.Queue.Done(record) + continue + } + + err := r.updateOOMRecordConfigMap() + if err != nil { + klog.ErrorS(err, "Update oomRecord failed") + } + r.Queue.Done(record) + } +} + +func (r *PodOOMRecorder) ListOOMRecordsFromConfigmap() ([]OOMRecord, error) { + oomConfigMap := &v1.ConfigMap{} + err := r.Client.Get(context.TODO(), types.NamespacedName{Namespace: ConfigMapOOMRecordNameSpace, + Name: ConfigMapOOMRecordName}, oomConfigMap) + if err != nil { + return nil, client.IgnoreNotFound(err) + } + oomRecords := make([]OOMRecord, 0) + err = json.Unmarshal([]byte(oomConfigMap.Data[ConfigMapDataOOMRecord]), &oomRecords) + return oomRecords, err +} diff --git a/pkg/controller/resource-recommend/oom/oom_recorder_test.go b/pkg/controller/resource-recommend/oom/oom_recorder_test.go new file mode 100644 index 000000000..ccfe77c74 --- /dev/null +++ b/pkg/controller/resource-recommend/oom/oom_recorder_test.go @@ -0,0 +1,321 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oom + +import ( + "context" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestCleanOOMRecord(t *testing.T) { + + oomRecordsList := []PodOOMRecorder{ + { + OOMRecordMaxNumber: 4, + cache: []OOMRecord{ + { + OOMAt: time.Now().Add(-140 * time.Hour), + }, + { + OOMAt: time.Now().Add(-150 * time.Hour), + }, + { + OOMAt: time.Now().Add(-150 * time.Hour), + }, + { + OOMAt: time.Now().Add(-150 * time.Hour), + }, + { + OOMAt: time.Now().Add(-150 * time.Hour), + }, + { + OOMAt: time.Now().Add(-160 * time.Hour), + }, + { + OOMAt: time.Now().Add(-170 * time.Hour), + }, + { + OOMAt: time.Now().Add(-180 * time.Hour), + }, + { + OOMAt: time.Now().Add(-190 * time.Hour), + }, + }, + }, + { + OOMRecordMaxNumber: 4, + cache: []OOMRecord{ + { + OOMAt: time.Now().Add(-150 * time.Hour), + }, + { + OOMAt: time.Now().Add(-140 * time.Hour), + }, + { + OOMAt: time.Now().Add(-130 * time.Hour), + }, + { + OOMAt: time.Now().Add(-120 * time.Hour), + }, + { + OOMAt: time.Now().Add(-110 * time.Hour), + }, + }, + }, + { + OOMRecordMaxNumber: 4, + cache: []OOMRecord{ + { + OOMAt: time.Now().Add(-170 * time.Hour), + }, + { + OOMAt: time.Now().Add(-180 * time.Hour), + }, + { + OOMAt: time.Now().Add(-190 * time.Hour), + }, + { + OOMAt: time.Now().Add(-190 * time.Hour), + }, + { + OOMAt: time.Now().Add(-200 * time.Hour), + }, + { + OOMAt: time.Now().Add(-210 * time.Hour), + }, + }, + }, + { + OOMRecordMaxNumber: 4, + cache: []OOMRecord{ + { + OOMAt: time.Now().Add(-160 * time.Hour), + }, + { + OOMAt: time.Now().Add(-170 * time.Hour), + }, + { + OOMAt: time.Now().Add(-180 * time.Hour), + }, + }, + }, + { + OOMRecordMaxNumber: 4, + cache: []OOMRecord{ + { + OOMAt: time.Now().Add(-170 * time.Hour), + }, + { + OOMAt: time.Now().Add(-180 * time.Hour), + }, + { + OOMAt: time.Now().Add(-190 * time.Hour), + }, + }, + }, + { + OOMRecordMaxNumber: 4, + cache: []OOMRecord{ + { + OOMAt: time.Now().Add(-160 * time.Hour), + }, + { + OOMAt: time.Now().Add(-150 * time.Hour), + }, + { + OOMAt: time.Now().Add(-140 * time.Hour), + }, + }, + }, + { + OOMRecordMaxNumber: 4, + cache: []OOMRecord{}, + }, + } + for index := range oomRecordsList { + splitTimePoints := time.Now().Add(-DataRetentionHour * time.Hour) + oomRecordsList[index].cleanOOMRecord() + if len(oomRecordsList[index].cache) > oomRecordsList[index].OOMRecordMaxNumber { + t.Errorf("Expected oomRecordsList length to be less than or equal to %d, but it is actually %d", + oomRecordsList[index].OOMRecordMaxNumber, len(oomRecordsList[index].cache)) + } + for _, record := range oomRecordsList[index].cache { + if record.OOMAt.Before(splitTimePoints) { + t.Errorf("Expected oomAt to be greater than %v, but it is actually %v", splitTimePoints, record.OOMAt) + } + } + } + +} + +func TestListOOMRecordsFromConfigmap(t *testing.T) { + dummyClient := fake.NewClientBuilder().WithObjects(&v1.ConfigMap{}).Build() + dummyPodOOMRecorder := PodOOMRecorder{ + Client: dummyClient, + } + oomRecords, _ := dummyPodOOMRecorder.ListOOMRecordsFromConfigmap() + if len(oomRecords) > 0 { + t.Errorf("Expected oomRecords length is zero, but actual oomRecords length is %v", len(oomRecords)) + } + oomConfigMap := &v1.ConfigMap{ + Data: map[string]string{ + ConfigMapDataOOMRecord: `[{"Namespace":"dummyNamespace","Pod":"dummyPod","Container":"dummyContainer","Memory":"600Mi","OOMAt":"2023-08-07T16:45:50+08:00"},{"Namespace":"dummyNamespace","Pod":"dummyPod","Container":"dummyContainer","Memory":"600Mi","OOMAt":"2023-08-07T16:46:07+08:00"}]`, + }, + } + oomConfigMap.SetName(ConfigMapOOMRecordName) + oomConfigMap.SetNamespace(ConfigMapOOMRecordNameSpace) + dummyPodOOMRecorder.Client.Create(context.TODO(), oomConfigMap) + oomRecords, err := dummyPodOOMRecorder.ListOOMRecordsFromConfigmap() + if err != nil || len(oomRecords) != 2 { + t.Errorf("Expected oomRecords length is 2 and err is nil, but actual oomRecords length is %v and err is %v", len(oomRecords), err) + } +} + +func TestUpdateOOMRecordCache(t *testing.T) { + now := time.Now() + dummyClient := fake.NewClientBuilder().WithObjects(&v1.ConfigMap{}).Build() + podOOMRecorderList := []PodOOMRecorder{ + { + Client: dummyClient, + cache: []OOMRecord{ + { + Namespace: "dummyNamespace1", + Pod: "dummyPod1", + Container: "dummyContainer1", + Memory: resource.MustParse("6Gi"), + OOMAt: now, + }, + }, + }, + { + Client: dummyClient, + cache: []OOMRecord{ + { + Namespace: "dummyNamespace", + Pod: "dummyPod", + Container: "dummyContainer", + Memory: resource.MustParse("6Gi"), + OOMAt: now, + }, + }, + }, + { + Client: dummyClient, + cache: []OOMRecord{ + { + Namespace: "dummyNamespace", + Pod: "dummyPod", + Container: "dummyContainer", + Memory: resource.MustParse("6Gi"), + OOMAt: now.Add(1 * time.Hour), + }, + }, + }, + { + Client: dummyClient, + cache: []OOMRecord{ + { + Namespace: "dummyNamespace", + Pod: "dummyPod", + Container: "dummyContainer", + Memory: resource.MustParse("7Gi"), + OOMAt: now.Add(1 * time.Hour), + }, + }, + }, + { + Client: dummyClient, + cache: []OOMRecord{ + { + Namespace: "dummyNamespace", + Pod: "dummyPod", + Container: "dummyContainer", + Memory: resource.MustParse("5Gi"), + OOMAt: now.Add(1 * time.Hour), + }, + }, + }, + } + oomRecord := OOMRecord{ + Namespace: "dummyNamespace", + Pod: "dummyPod", + Container: "dummyContainer", + Memory: resource.MustParse("6Gi"), + OOMAt: now, + } + resultList := []bool{true, false, true, false, true} + for index := range podOOMRecorderList { + isUpdated := podOOMRecorderList[index].updateOOMRecordCache(oomRecord) + if isUpdated != resultList[index] { + t.Errorf("Expected isUpdated %v, but it is actually %v", resultList[index], isUpdated) + } + } + +} + +func TestUpdateOOMRecordConfigMap(t *testing.T) { + dummyClient := fake.NewClientBuilder().WithObjects(&v1.ConfigMap{}).Build() + oomConfigMap := &v1.ConfigMap{ + Data: map[string]string{ + ConfigMapDataOOMRecord: `[]`, + }, + } + oomConfigMap.SetName(ConfigMapOOMRecordName) + oomConfigMap.SetNamespace(ConfigMapOOMRecordNameSpace) + dummyPodOOMRecorder := PodOOMRecorder{ + Client: dummyClient, + cache: []OOMRecord{ + { + Namespace: "dummyNamespace", + Pod: "dummyPod", + Container: "dummyContainer", + Memory: resource.MustParse("600Mi"), + OOMAt: time.Date(2012, time.March, 4, 0, 0, 0, 0, time.UTC), + }, + }, + } + err := dummyPodOOMRecorder.updateOOMRecordConfigMap() + if err != nil { + t.Errorf("Expected the configMap was successfully created,but actually an error:%v occurred.", err) + } + oomRecordList, _ := dummyPodOOMRecorder.ListOOMRecordsFromConfigmap() + for index := range oomRecordList { + if dummyPodOOMRecorder.cache[index] != oomRecordList[index] { + t.Errorf("Expected OOMRecord value is %v, actually OOMRecord value is %v", + dummyPodOOMRecorder.cache[index], oomRecordList[index]) + } + } + + dummyPodOOMRecorder.Client.DeleteAllOf(context.TODO(), oomConfigMap) + dummyPodOOMRecorder.Client.Create(context.TODO(), oomConfigMap) + err = dummyPodOOMRecorder.updateOOMRecordConfigMap() + if err != nil { + t.Errorf("Expected the configMap was successfully updated,but actually an error:%v occurred.", err) + } + oomRecordList, _ = dummyPodOOMRecorder.ListOOMRecordsFromConfigmap() + for index := range oomRecordList { + if dummyPodOOMRecorder.cache[index] != oomRecordList[index] { + t.Errorf("Expected OOMRecord value is %v, actually OOMRecord value is %v", + dummyPodOOMRecorder.cache[index], oomRecordList[index]) + } + } +} diff --git a/pkg/controller/resource-recommend/processor/manager/processor_manager.go b/pkg/controller/resource-recommend/processor/manager/processor_manager.go new file mode 100644 index 000000000..259fbe794 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/manager/processor_manager.go @@ -0,0 +1,80 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "context" + "runtime/debug" + "sync" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile" + "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/log" +) + +type Manager struct { + processors map[v1alpha1.Algorithm]processor.Processor +} + +func NewManager(datasourceProxy *datasource.Proxy, c client.Client) *Manager { + percentileProcessor := percentile.NewProcessor(datasourceProxy, c) + return &Manager{ + processors: map[v1alpha1.Algorithm]processor.Processor{ + v1alpha1.AlgorithmPercentile: percentileProcessor, + }, + } +} + +func (m *Manager) ProcessorRegister(algorithm v1alpha1.Algorithm, registerProcessor processor.Processor) { + m.processors = make(map[v1alpha1.Algorithm]processor.Processor) + m.processors[algorithm] = registerProcessor +} + +func (m *Manager) StartProcess(ctx context.Context) { + var wg sync.WaitGroup + for processorName, dataProcessor := range m.processors { + wg.Add(1) + processorCtx := log.SetKeysAndValues(log.InitContext(ctx), "processor", processorName) + go func(ctx context.Context, processor processor.Processor) { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + log.ErrorS(ctx, r.(error), "processor panic", "stack", string(debug.Stack())) + panic("processor panic") + } + }() + processor.Run(ctx) + }(processorCtx, dataProcessor) + } + + log.InfoS(ctx, "ProcessorManager started, all Processor running") + + wg.Wait() + + log.InfoS(ctx, "ProcessorManager stopped, all Processor end") +} + +func (m *Manager) GetProcessor(algorithm v1alpha1.Algorithm) processor.Processor { + if dataProcessor, ok := m.processors[algorithm]; ok { + return dataProcessor + } + return m.processors[v1alpha1.AlgorithmPercentile] +} diff --git a/pkg/controller/resource-recommend/processor/manager/processor_manager_test.go b/pkg/controller/resource-recommend/processor/manager/processor_manager_test.go new file mode 100644 index 000000000..23e6db1d8 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/manager/processor_manager_test.go @@ -0,0 +1,118 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "context" + "reflect" + "testing" + "time" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor" + "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/log" + errortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" +) + +type mockProcessor struct { + algorithm v1alpha1.Algorithm + runMark string +} + +func (p *mockProcessor) Run(ctx context.Context) { + time.Sleep(1 * time.Second) + p.runMark = string(p.algorithm) + "processor" + "-run" + log.InfoS(ctx, "mockProcessor Run", "algorithm", p.algorithm) +} + +func (p *mockProcessor) Register(_ *processortypes.ProcessConfig) *errortypes.CustomError { + return nil +} + +func (p *mockProcessor) Cancel(_ *processortypes.ProcessKey) *errortypes.CustomError { return nil } + +func (p *mockProcessor) QueryProcessedValues(_ *processortypes.ProcessKey) (float64, error) { + return 0, nil +} + +func TestManager_StartProcess(t *testing.T) { + mockProcessor1 := mockProcessor{algorithm: "mockAlgorithm1"} + mockProcessor2 := mockProcessor{algorithm: "mockAlgorithm2"} + mockProcessor3 := mockProcessor{algorithm: "mockAlgorithm3"} + tests := []struct { + name string + processors []*mockProcessor + }{ + { + name: "case1", + processors: []*mockProcessor{&mockProcessor1, &mockProcessor2, &mockProcessor3}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manager := &Manager{processors: map[v1alpha1.Algorithm]processor.Processor{}} + for _, p := range tt.processors { + manager.processors[p.algorithm] = p + } + manager.StartProcess(context.Background()) + for _, p := range tt.processors { + if p.runMark != string(p.algorithm)+"processor"+"-run" { + t.Errorf("StartProcess() failed, %s processor not run", p.algorithm) + } + } + }) + } +} + +func TestManager_GetProcessor(t *testing.T) { + manager1 := NewManager(nil, nil) + type args struct { + algorithm v1alpha1.Algorithm + } + tests := []struct { + name string + manager *Manager + args args + want processor.Processor + }{ + { + name: "Unknown_Algorithm", + manager: manager1, + args: args{ + algorithm: "Unknown", + }, + want: manager1.processors[v1alpha1.AlgorithmPercentile], + }, + { + name: "get_AlgorithmPercentile_processor", + manager: manager1, + args: args{ + algorithm: v1alpha1.AlgorithmPercentile, + }, + want: manager1.processors[v1alpha1.AlgorithmPercentile], + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.manager.GetProcessor(tt.args.algorithm) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetProcessor() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/processor/percentile/process_gc.go b/pkg/controller/resource-recommend/processor/percentile/process_gc.go new file mode 100644 index 000000000..bc8407910 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/process_gc.go @@ -0,0 +1,128 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "context" + "runtime/debug" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task" + "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/log" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" +) + +func (p *Processor) GarbageCollector(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + errMsg := "garbage collect goroutine panic" + log.ErrorS(ctx, r.(error), errMsg, "stack", string(debug.Stack())) + panic(errMsg) + } + }() + + ticker := time.Tick(DefaultGarbageCollectInterval) + for range ticker { + if err := p.garbageCollect(NewContext()); err != nil { + log.ErrorS(ctx, err, "garbageCollect failed") + } + } +} + +func (p *Processor) garbageCollect(ctx context.Context) error { + defer func() { + if r := recover(); r != nil { + errMsg := "garbage collect run panic" + log.ErrorS(ctx, r.(error), errMsg, "stack", string(debug.Stack())) + } + }() + + log.InfoS(ctx, "garbage collect start") + + p.mutex.Lock() + defer p.mutex.Unlock() + + // Tasks that have not been running for too long are considered invalid and will be cleared + p.AggregateTasks.Range(func(ki, vi any) bool { + taskID := ki.(processortypes.TaskID) + t, ok := vi.(*task.HistogramTask) + if !ok { + p.AggregateTasks.Delete(taskID) + return true + } + if t.IsTimeoutNotExecute() { + p.AggregateTasks.Delete(taskID) + } + return true + }) + + // List all ResourceRecommend CR up to now + resourceRecommendList := &v1alpha1.ResourceRecommendList{} + err := p.Client.List(ctx, resourceRecommendList, &k8sclient.ListOptions{Raw: &metav1.ListOptions{ResourceVersion: "0"}}) + if err != nil { + log.ErrorS(ctx, err, "garbage collect list all ResourceRecommend failed") + return err + } + + //p.ClearingNoAttributionTask(resourceRecommendList) + klog.InfoS("percentile processor garbage collect list ResourceRecommend", + "ResourceRecommend Count", len(resourceRecommendList.Items)) + // Convert the ResourceRecommend list to map for quick check whether it exists + existResourceRecommends := make(map[types.NamespacedName]v1alpha1.CrossVersionObjectReference) + for _, existResourceRecommend := range resourceRecommendList.Items { + namespacedName := types.NamespacedName{ + Name: existResourceRecommend.Name, + Namespace: existResourceRecommend.Namespace, + } + existResourceRecommends[namespacedName] = existResourceRecommend.Spec.TargetRef + } + + // Delete tasks belong to not exist ResourceRecommend or task not match the targetRef in the ResourceRecommend + for namespacedName, tasks := range p.ResourceRecommendTaskIDsMap { + if tasks == nil { + delete(p.ResourceRecommendTaskIDsMap, namespacedName) + continue + } + targetRef, isExist := existResourceRecommends[namespacedName] + for metrics, taskID := range *tasks { + if !isExist || + targetRef.Kind != metrics.Kind || + targetRef.Name != metrics.WorkloadName || + targetRef.APIVersion != metrics.APIVersion { + p.AggregateTasks.Delete(taskID) + delete(*tasks, metrics) + } + // timeout task in p.AggregateTasks will clean, + // If taskID does not exist in p.AggregateTasks，mean is the task is cleared, this needs to be deleted from tasks + if _, found := p.AggregateTasks.Load(taskID); !found { + delete(*tasks, metrics) + } + } + if !isExist { + delete(p.ResourceRecommendTaskIDsMap, namespacedName) + } + } + + log.InfoS(ctx, "garbage collect end") + return nil +} diff --git a/pkg/controller/resource-recommend/processor/percentile/process_gc_test.go b/pkg/controller/resource-recommend/processor/percentile/process_gc_test.go new file mode 100644 index 000000000..b5e2215f1 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/process_gc_test.go @@ -0,0 +1,120 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "sync" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-api/pkg/client/clientset/versioned/scheme" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task" + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" +) + +func TestProcessor_garbageCollect(t *testing.T) { + s := scheme.Scheme + s.AddKnownTypes(v1alpha1.SchemeGroupVersion, &v1alpha1.ResourceRecommend{}) + client := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() + processor := Processor{ + Client: client, + AggregateTasks: &sync.Map{}, + ResourceRecommendTaskIDsMap: make(map[types.NamespacedName]*map[datasourcetypes.Metric]processortypes.TaskID), + } + + namespacedName0 := types.NamespacedName{ + Name: "testGC0", + Namespace: "testGC", + } + + // case1: clean TypeIllegal task + taskIDTaskTypeIllegal := processortypes.TaskID("taskID_TaskTypeIllegal") + metricTaskTypeIllegal := datasourcetypes.Metric{ + Namespace: "testGCTaskTypeIllegal", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testGCTaskTypeIllegal", + ContainerName: "testGCTaskTypeIllegal-1", + Resource: v1.ResourceCPU, + } + + // case2: clean timeout task + taskID0 := processortypes.TaskID("taskID0") + metric0 := datasourcetypes.Metric{ + Namespace: "testGC", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testGC0", + ContainerName: "testGC0-1", + Resource: v1.ResourceCPU, + } + task0, _ := task.NewTask(metric0, "") + task0.AddSample(time.Unix(1693572914, 0), 20, 1000) + + // case3: clean resourceRecommend cr not exist task + taskID1 := processortypes.TaskID("taskID1") + metric1 := datasourcetypes.Metric{ + Namespace: "testGC", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testGC1", + ContainerName: "testGC1-1", + Resource: v1.ResourceCPU, + } + task1, _ := task.NewTask(metric1, "") + + processor.AggregateTasks.LoadOrStore(taskIDTaskTypeIllegal, 3) + processor.AggregateTasks.LoadOrStore(taskID0, task0) + processor.AggregateTasks.LoadOrStore(taskID1, task1) + processor.ResourceRecommendTaskIDsMap[namespacedName0] = &map[datasourcetypes.Metric]processortypes.TaskID{ + metric0: taskID0, + metric1: taskID1, + metricTaskTypeIllegal: taskIDTaskTypeIllegal, + } + + tests := []struct { + name string + wantErr bool + }{ + {}, + } + for _, tt := range tests { + err := processor.garbageCollect(NewContext()) + if (err != nil) != tt.wantErr { + t.Errorf("garbageCollect() error = %v, wantErr %v", err, tt.wantErr) + return + } + if _, ok := processor.AggregateTasks.Load(taskIDTaskTypeIllegal); ok { + t.Errorf("garbageCollect() type tllegal task not cleaned up") + } + if _, ok := processor.AggregateTasks.Load(taskID0); ok { + t.Errorf("garbageCollect() timeout task not cleaned up") + } + if _, ok := processor.AggregateTasks.Load(taskID1); ok { + t.Errorf("garbageCollect() resourceRecommend cr not exist tasks(in AggregateTasks) not cleaned up") + } + if _, ok := processor.ResourceRecommendTaskIDsMap[namespacedName0]; ok { + t.Errorf("garbageCollect() resourceRecommend cr not exist tasks(in ResourceRecommendTaskIDsMap) not cleaned up") + } + } +} diff --git a/pkg/controller/resource-recommend/processor/percentile/process_tasks.go b/pkg/controller/resource-recommend/processor/percentile/process_tasks.go new file mode 100644 index 000000000..ed9891acf --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/process_tasks.go @@ -0,0 +1,99 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "context" + "runtime/debug" + "sync" + + "github.com/pkg/errors" + + "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/log" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" +) + +func (p *Processor) ProcessTasks(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + errMsg := "process tasks goroutine run panic" + log.ErrorS(ctx, r.(error), errMsg, "stack", string(debug.Stack())) + panic(errMsg) + } + }() + + wg := &sync.WaitGroup{} + wg.Add(DefaultConcurrentTaskNum) + for i := 0; i < DefaultConcurrentTaskNum; i++ { + go func() { + defer wg.Done() + // Run a worker thread that just dequeues items, processes them, and marks them done. + for p.processTask(NewContext()) { + } + }() + } + + log.InfoS(ctx, "process workers running") + wg.Wait() + log.InfoS(ctx, "all process workers finished") +} + +func (p *Processor) processTask(ctx context.Context) bool { + obj, shutdown := p.TaskQueue.Get() + log.InfoS(ctx, "process task dequeue", "obj", obj, "shutdown", shutdown) + if shutdown { + err := errors.New("task queue is shutdown") + log.ErrorS(ctx, err, "process task failed") + // Stop working + return false + } + + defer p.TaskQueue.Done(obj) + + p.taskHandler(ctx, obj) + return true +} + +func (p *Processor) taskHandler(ctx context.Context, obj interface{}) { + log.InfoS(ctx, "task handler begin") + taskID, ok := obj.(processortypes.TaskID) + if !ok { + err := errors.New("task key type error, drop") + log.ErrorS(ctx, err, "task err", "obj", obj) + p.TaskQueue.Forget(obj) + return + } + ctx = log.SetKeysAndValues(ctx, "taskID", taskID) + + task, err := p.getTaskForTaskID(taskID) + if err != nil { + log.ErrorS(ctx, err, "task not found, drop it") + p.TaskQueue.Forget(obj) + return + } + + if nextRunInterval, err := task.Run(ctx, p.DatasourceProxy); err != nil { + log.ErrorS(ctx, err, "task handler err") + p.TaskQueue.AddRateLimited(taskID) + } else { + log.InfoS(ctx, "task handler finished") + p.TaskQueue.Forget(obj) + if nextRunInterval > 0 { + p.TaskQueue.AddAfter(taskID, nextRunInterval) + } + } +} diff --git a/pkg/controller/resource-recommend/processor/percentile/process_tasks_test.go b/pkg/controller/resource-recommend/processor/percentile/process_tasks_test.go new file mode 100644 index 000000000..6a8e3a615 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/process_tasks_test.go @@ -0,0 +1,290 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "golang.org/x/time/rate" + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/util/workqueue" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task" + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" +) + +func TestProcessor_processTask(t *testing.T) { + taskQueue1 := workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName) + //processor1 := Processor{ + // TaskQueue: taskQueue1, + //} + + taskQueue2 := workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName) + //processor2 := Processor{ + // TaskQueue: taskQueue2, + //} + taskIDTaskTypeIllegal := 3 + + taskQueue3 := workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName) + //processor3 := Processor{ + // TaskQueue: taskQueue3, + // AggregateTasks: &sync.Map{}, + //} + taskID3 := processortypes.TaskID("case3") + + type testFunc func() + type checkFunc func() + tests := []struct { + name string + processor Processor + testFunc + isContinueToRun bool + checkFunc + }{ + { + name: "queue_shutdown", + processor: Processor{ + TaskQueue: taskQueue1, + }, + testFunc: func() { + taskQueue1.ShutDown() + }, + isContinueToRun: false, + checkFunc: func() {}, + }, + { + name: "task_type_illegal", + processor: Processor{ + TaskQueue: taskQueue2, + }, + testFunc: func() { + taskQueue2.Add(taskIDTaskTypeIllegal) + }, + isContinueToRun: true, + checkFunc: func() { + if taskQueue2.Len() != 0 { + t.Errorf("task type illegal not forget") + } + }, + }, + { + name: "task_type_illegal", + processor: Processor{ + TaskQueue: taskQueue3, + AggregateTasks: &sync.Map{}, + }, + testFunc: func() { + taskQueue3.Add(taskID3) + }, + isContinueToRun: true, + checkFunc: func() { + if taskQueue3.Len() != 0 { + t.Errorf("not found task not forget") + } + }, + }, + } + for i := 0; i < len(tests); i++ { + t.Run(tests[i].name, func(t *testing.T) { + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + isContinueToRun := tests[i].processor.processTask(NewContext()) + if tests[i].isContinueToRun != isContinueToRun { + t.Errorf("processTask() isShutdown = %v, want %v\n", isContinueToRun, tests[i].isContinueToRun) + return + } + }() + go func() { + defer wg.Done() + tests[i].testFunc() + }() + wg.Wait() + tests[i].checkFunc() + }) + } +} + +type MockDatasourceForProcessTasks struct{} + +func (m1 *MockDatasourceForProcessTasks) QueryTimeSeries(_ *datasourcetypes.Query, _ time.Time, _ time.Time, _ time.Duration) (*datasourcetypes.TimeSeries, error) { + return &datasourcetypes.TimeSeries{Samples: []datasourcetypes.Sample{ + { + Timestamp: 1694270256, + Value: 1, + }, + }}, nil +} + +func (m1 *MockDatasourceForProcessTasks) ConvertMetricToQuery(metric datasourcetypes.Metric) (*datasourcetypes.Query, error) { + return nil, nil +} + +func TestProcessor_processTask1(t *testing.T) { + queueRateLimiter := workqueue.NewMaxOfRateLimiter( + workqueue.NewItemExponentialFailureRateLimiter(time.Second, 32*time.Second), + // 10 qps, 100 bucket size. This is only for retry speed and its only the overall factor (not per item) + &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)}, + ) + taskQueue1 := workqueue.NewNamedRateLimitingQueue(queueRateLimiter, ProcessorName) + taskID1 := processortypes.TaskID("task_run_err") + aggregateTasks1 := &sync.Map{} + aggregateTasks1.Store(taskID1, &task.HistogramTask{}) + + taskQueue2 := workqueue.NewNamedRateLimitingQueue(queueRateLimiter, ProcessorName) + taskID := processortypes.TaskID("task_run_err") + mockTask, _ := task.NewTask(datasourcetypes.Metric{Resource: v1.ResourceCPU}, "") + mockTask.ProcessInterval = time.Second * 2 + aggregateTasks2 := &sync.Map{} + aggregateTasks2.Store(taskID, mockTask) + + proxy := datasource.NewProxy() + proxy.RegisterDatasource(datasource.PrometheusDatasource, &MockDatasourceForProcessTasks{}) + + type testFunc func() + type runFunc func() + tests := []struct { + name string + processor Processor + testFunc + runFunc + runTimes int + wantRunSecond int64 + }{ + { + name: "run_err", + processor: Processor{ + TaskQueue: taskQueue1, + AggregateTasks: aggregateTasks1, + }, + testFunc: func() { + taskQueue1.Add(taskID1) + }, + runTimes: 4, + wantRunSecond: 7, + }, + { + name: "run", + processor: Processor{ + TaskQueue: taskQueue2, + AggregateTasks: aggregateTasks2, + DatasourceProxy: proxy, + }, + testFunc: func() { + taskQueue2.Add(taskID) + }, + runTimes: 3, + wantRunSecond: 4, + }, + } + for i := 0; i < len(tests); i++ { + t.Run(tests[i].name, func(t *testing.T) { + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + beginTime := time.Now() + for j := 0; j < tests[i].runTimes; j++ { + _ = tests[i].processor.processTask(NewContext()) + } + runtime := int64(time.Now().Sub(beginTime) / time.Second) + if runtime != tests[i].wantRunSecond { + t.Errorf("processTask() runtime = %ds, want %ds", runtime, tests[i].wantRunSecond) + } + }() + go func() { + defer wg.Done() + tests[i].testFunc() + }() + wg.Wait() + }) + } +} + +type MockDatasource1ForProcessTasks struct{} + +func (m1 *MockDatasource1ForProcessTasks) QueryTimeSeries(_ *datasourcetypes.Query, _ time.Time, _ time.Time, _ time.Duration) (*datasourcetypes.TimeSeries, error) { + time.Sleep(2 * time.Second) + return &datasourcetypes.TimeSeries{Samples: []datasourcetypes.Sample{ + { + Timestamp: 1694270256, + Value: 1, + }, + }}, nil +} + +func (m1 *MockDatasource1ForProcessTasks) ConvertMetricToQuery(metric datasourcetypes.Metric) (*datasourcetypes.Query, error) { + return nil, nil +} + +func TestProcessor_ProcessTasks(t *testing.T) { + q := workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName) + aggregateTasks := &sync.Map{} + proxy := datasource.NewProxy() + proxy.RegisterDatasource(datasource.PrometheusDatasource, &MockDatasource1ForProcessTasks{}) + + taskIDList := make([]processortypes.TaskID, 0) + for i := 0; i < DefaultConcurrentTaskNum+1; i++ { + taskID := processortypes.TaskID(fmt.Sprintf("task-%d", i)) + taskIDList = append(taskIDList, taskID) + mockTask, _ := task.NewTask(datasourcetypes.Metric{Resource: v1.ResourceCPU}, "") + aggregateTasks.Store(taskID, mockTask) + } + mockP := Processor{ + TaskQueue: q, + AggregateTasks: aggregateTasks, + DatasourceProxy: proxy, + } + type testFunc func() + tests := []struct { + name string + testFunc + }{ + { + name: "case1", + testFunc: func() { + for _, id := range taskIDList { + q.Add(id) + } + time.Sleep(5 * time.Second) + q.ShutDown() + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + mockP.ProcessTasks(context.Background()) + }() + go func() { + defer wg.Done() + tt.testFunc() + }() + wg.Wait() + }) + } +} diff --git a/pkg/controller/resource-recommend/processor/percentile/process_util.go b/pkg/controller/resource-recommend/processor/percentile/process_util.go new file mode 100644 index 000000000..9b80981e8 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/process_util.go @@ -0,0 +1,62 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "context" + + "github.com/pkg/errors" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task" + "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/log" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" +) + +func NewContext() context.Context { + return log.SetKeysAndValues(log.InitContext(context.Background()), "processor", ProcessorName) +} + +func (p *Processor) getTaskForProcessKey(processKey *processortypes.ProcessKey) (*task.HistogramTask, error) { + if processKey == nil { + return nil, errors.Errorf("ProcessKey is nil") + } + if processKey.Metric == nil { + return nil, errors.Errorf("Metric is nil") + } + if tasks, ok := p.ResourceRecommendTaskIDsMap[processKey.ResourceRecommendNamespacedName]; !ok { + return nil, errors.Errorf("not found percentile process task ID for ResourceRecommend(%s)", + processKey.ResourceRecommendNamespacedName) + } else { + if taskID, exist := (*tasks)[*processKey.Metric]; exist { + return p.getTaskForTaskID(taskID) + } + return nil, errors.Errorf("not found process task ID for container(%s) in ResourceRecommend(%s)", + processKey.ContainerName, processKey.ResourceRecommendNamespacedName) + } +} + +func (p *Processor) getTaskForTaskID(taskID processortypes.TaskID) (*task.HistogramTask, error) { + data, found := p.AggregateTasks.Load(taskID) + if !found { + return nil, errors.New("process task not found") + } + t, ok := data.(*task.HistogramTask) + if !ok { + return nil, errors.New("process task type illegal") + } + return t, nil +} diff --git a/pkg/controller/resource-recommend/processor/percentile/process_util_test.go b/pkg/controller/resource-recommend/processor/percentile/process_util_test.go new file mode 100644 index 000000000..9a5bd8626 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/process_util_test.go @@ -0,0 +1,91 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "reflect" + "testing" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" +) + +func TestProcessor_getTaskForProcessKey(t *testing.T) { + type newProcessor func() *Processor + tests := []struct { + name string + newProcessor newProcessor + processKey *processortypes.ProcessKey + want *task.HistogramTask + wantErr bool + }{ + { + name: "processConfig is nil", + processKey: nil, + want: nil, + wantErr: true, + }, + { + name: "processConfig.Metric is nil", + processKey: &metricIsNilProcessorKey, + want: nil, + wantErr: true, + }, + { + name: "not found NamespacedName", + processKey: ¬ExistNamespacedNameProcessorKey, + want: nil, + wantErr: true, + }, + { + name: "not found Metric", + processKey: ¬ExistMetricProcessorKey, + want: nil, + wantErr: true, + }, + { + name: "not found Task", + processKey: ¬FoundTaskProcessorKey, + want: nil, + wantErr: true, + }, + { + name: "task type illegal", + processKey: &valueTypeIllegalProcessorKey, + want: nil, + wantErr: true, + }, + { + name: "success", + processKey: &mockProcessKey1, + want: mockTask1, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := mockProcessor.getTaskForProcessKey(tt.processKey) + if (err != nil) != tt.wantErr { + t.Errorf("getTaskForProcessKey() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getTaskForProcessKey() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/processor/percentile/processor.go b/pkg/controller/resource-recommend/processor/percentile/processor.go new file mode 100644 index 000000000..aae9e879f --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/processor.go @@ -0,0 +1,220 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "context" + "runtime/debug" + "sync" + "time" + + "github.com/pkg/errors" + "golang.org/x/time/rate" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task" + "github.com/kubewharf/katalyst-core/pkg/util/general" + "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/log" + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" + errortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" +) + +const ( + ProcessorName = "percentile" + // DefaultConcurrentTaskNum is num of default concurrent task + DefaultConcurrentTaskNum = 100 + DefaultPercentile = 0.9 + DefaultGarbageCollectInterval = 1 * time.Hour + ExceptionRequeueBaseDelay = time.Minute + ExceptionRequeueMaxDelay = 30 * time.Minute +) + +type Processor struct { + mutex sync.Mutex + + client.Client + + DatasourceProxy *datasource.Proxy + + TaskQueue workqueue.RateLimitingInterface + + AggregateTasks *sync.Map + + // Stores taskID corresponding to Metrics in the ResourceRecommend + ResourceRecommendTaskIDsMap map[types.NamespacedName]*map[datasourcetypes.Metric]processortypes.TaskID +} + +var DefaultQueueRateLimiter = workqueue.NewMaxOfRateLimiter( + workqueue.NewItemExponentialFailureRateLimiter(ExceptionRequeueBaseDelay, ExceptionRequeueMaxDelay), + // 10 qps, 100 bucket size. This is only for retry speed and its only the overall factor (not per item) + &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)}, +) + +func NewProcessor(datasourceProxy *datasource.Proxy, c client.Client) processor.Processor { + return &Processor{ + DatasourceProxy: datasourceProxy, + TaskQueue: workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName), + Client: c, + AggregateTasks: &sync.Map{}, + ResourceRecommendTaskIDsMap: make(map[types.NamespacedName]*map[datasourcetypes.Metric]processortypes.TaskID), + } +} + +func (p *Processor) Register(processConfig *processortypes.ProcessConfig) (cErr *errortypes.CustomError) { + defer func() { + if cErr != nil { + klog.ErrorS(cErr, "Percentile task register failed", "ResourceRecommend", processConfig.ResourceRecommendNamespacedName) + } + if r := recover(); r != nil { + errMsg := "percentile process register panic" + klog.ErrorS(r.(error), errMsg, "stack", string(debug.Stack())) + cErr = errortypes.RegisterProcessTaskPanic() + } + }() + + if err := processConfig.Validate(); err != nil { + return errortypes.RegisterProcessTaskValidateError(err) + } + + taskID := processConfig.GenerateTaskID() + + // Check whether a task has been registered and avoid repeated registration + _, ok := p.AggregateTasks.Load(taskID) + if ok { + klog.V(4).InfoS("The Percentile Processor task already registered", "processConfig", general.StructToString(processConfig)) + return nil + } + + p.mutex.Lock() + defer p.mutex.Unlock() + + klog.InfoS("Register Percentile Processor Task", "processConfig", general.StructToString(processConfig)) + + metric := *processConfig.Metric + + t, err := task.NewTask(metric, processConfig.Config) + if err != nil { + cErr := errortypes.NewProcessTaskError(err) + return cErr + } + + _, loaded := p.AggregateTasks.LoadOrStore(taskID, t) + if !loaded { + p.TaskQueue.Add(taskID) + } + + // Record the taskID corresponding to the Metric with the same ResourceRecommendID into ResourceRecommendTaskIDsMap + // To get the taskID from the ResourceRecommendID and Metric + if tasks, ok := p.ResourceRecommendTaskIDsMap[processConfig.ResourceRecommendNamespacedName]; !ok { + p.ResourceRecommendTaskIDsMap[processConfig.ResourceRecommendNamespacedName] = &map[datasourcetypes.Metric]processortypes.TaskID{ + metric: taskID, + } + } else { + if existingTaskID, exist := (*tasks)[metric]; exist { + if existingTaskID == taskID { + return nil + } + // existingTaskID != taskID means that the config of the task has changed. + // Need to delete the old task of config + p.AggregateTasks.Delete(existingTaskID) + } + (*tasks)[metric] = taskID + } + + return nil +} + +func (p *Processor) Cancel(processKey *processortypes.ProcessKey) (cErr *errortypes.CustomError) { + if processKey == nil { + return nil + } + + defer func() { + if cErr != nil { + klog.ErrorS(cErr, "Percentile task cancel failed", "ResourceRecommend", processKey.ResourceRecommendNamespacedName) + } + if r := recover(); r != nil { + errMsg := "percentile process cancel panic" + klog.ErrorS(r.(error), errMsg, "stack", string(debug.Stack())) + cErr = errortypes.CancelProcessTaskPanic() + } + }() + + p.mutex.Lock() + defer p.mutex.Unlock() + + tasks, ok := p.ResourceRecommendTaskIDsMap[processKey.ResourceRecommendNamespacedName] + if !ok { + klog.InfoS("Cancel task failed, percentile process task not found", "ResourceRecommend", + processKey.ResourceRecommendNamespacedName) + return errortypes.NotFoundTasksError(processKey.ResourceRecommendNamespacedName) + } + if processKey.Metric == nil { + klog.InfoS("delete percentile process tasks", "processConfig", processKey) + for _, taskID := range *tasks { + p.AggregateTasks.Delete(taskID) + } + delete(p.ResourceRecommendTaskIDsMap, processKey.ResourceRecommendNamespacedName) + } else { + if taskID, exist := (*tasks)[*processKey.Metric]; exist { + klog.InfoS("percentile process task delete for", "processConfig", processKey) + p.AggregateTasks.Delete(taskID) + delete(*tasks, *processKey.Metric) + } else { + klog.InfoS("task for metric cannot be found, don't deleted", "processConfig", processKey, "metric", processKey.Metric) + } + if tasks == nil || len(*tasks) == 0 { + delete(p.ResourceRecommendTaskIDsMap, processKey.ResourceRecommendNamespacedName) + } + } + + return nil +} + +func (p *Processor) Run(ctx context.Context) { + log.InfoS(ctx, "percentile processor starting") + + // Get task from queue and run it + go p.ProcessTasks(ctx) + + // Garbage collect every hour. Clearing timeout or no attribution task + go p.GarbageCollector(ctx) + + log.InfoS(ctx, "percentile processor running") + + <-ctx.Done() + + log.InfoS(ctx, "percentile processor end") +} + +func (p *Processor) QueryProcessedValues(processKey *processortypes.ProcessKey) (float64, error) { + t, err := p.getTaskForProcessKey(processKey) + if err != nil { + return 0, errors.Wrapf(err, "internal err, process task not found") + } + percentileValue, err := t.QueryPercentileValue(NewContext(), DefaultPercentile) + if err != nil { + return 0, err + } + return percentileValue, nil +} diff --git a/pkg/controller/resource-recommend/processor/percentile/processor_mock.go b/pkg/controller/resource-recommend/processor/percentile/processor_mock.go new file mode 100644 index 000000000..644b4fe95 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/processor_mock.go @@ -0,0 +1,131 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "sync" + "time" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task" + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" +) + +var mockNamespacedName1 = types.NamespacedName{ + Namespace: "testNamespace1", + Name: "testName1", +} +var mockNamespacedName2 = types.NamespacedName{ + Namespace: "testNamespace2", + Name: "testName2", +} +var notExistNamespacedName = types.NamespacedName{ + Namespace: "notExistNamespace", + Name: "notExistName", +} +var mockMetric1 = datasourcetypes.Metric{ + Namespace: "testNamespace1", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testWorkload1", + ContainerName: "testContainer1", + Resource: "cpu", +} +var emptyTaskMetric = datasourcetypes.Metric{ + Namespace: "testNamespace1", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testWorkload1", + ContainerName: "testContainer1", + Resource: "memory", +} +var notExistTaskIDMetric = datasourcetypes.Metric{ + Namespace: "testNamespace1", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testWorkload2", + ContainerName: "testContainer123", + Resource: "cpu", +} +var valueTypeIllegalMetric = datasourcetypes.Metric{ + Namespace: "testNamespace1", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testWorkload2", + ContainerName: "testContainer123", + Resource: "memory", +} +var notExistMetric = datasourcetypes.Metric{ + Namespace: "testNamespace1", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testWorkload2", + ContainerName: "testContainer876", + Resource: "memory", +} +var mockProcessKey1 = processortypes.ProcessKey{ResourceRecommendNamespacedName: mockNamespacedName1, Metric: &mockMetric1} +var emptyTaskProcessorKey = processortypes.ProcessKey{ResourceRecommendNamespacedName: mockNamespacedName1, Metric: &emptyTaskMetric} +var metricIsNilProcessorKey = processortypes.ProcessKey{ResourceRecommendNamespacedName: mockNamespacedName1, Metric: nil} +var notExistNamespacedNameProcessorKey = processortypes.ProcessKey{ResourceRecommendNamespacedName: notExistNamespacedName, Metric: ¬ExistTaskIDMetric} +var notExistMetricProcessorKey = processortypes.ProcessKey{ResourceRecommendNamespacedName: mockNamespacedName1, Metric: ¬ExistMetric} +var notFoundTaskProcessorKey = processortypes.ProcessKey{ResourceRecommendNamespacedName: mockNamespacedName2, Metric: ¬ExistTaskIDMetric} +var valueTypeIllegalProcessorKey = processortypes.ProcessKey{ResourceRecommendNamespacedName: mockNamespacedName2, Metric: &valueTypeIllegalMetric} + +var mockProcessConfig1 = processortypes.ProcessConfig{ProcessKey: mockProcessKey1} +var emptyProcessConfig = processortypes.ProcessConfig{ProcessKey: emptyTaskProcessorKey} +var valueTypeIllegalProcessConfig = processortypes.ProcessConfig{ProcessKey: valueTypeIllegalProcessorKey} + +var mockTaskID1 = mockProcessConfig1.GenerateTaskID() +var emptyTaskID = emptyProcessConfig.GenerateTaskID() +var notExistTaskID = "NotExistTaskID" +var valueTypeIllegalTaskID = valueTypeIllegalProcessConfig.GenerateTaskID() + +var mockTask1, _ = task.NewTask(mockMetric1, "") +var mockTask195PercentileValue = 20.40693 +var emptyTask, _ = task.NewTask(emptyTaskMetric, "") + +var mockAggregateTasks = sync.Map{} + +var mockResourceRecommendTaskIDsMap = map[types.NamespacedName]*map[datasourcetypes.Metric]processortypes.TaskID{} + +var mockProcessor = Processor{ + AggregateTasks: &mockAggregateTasks, + TaskQueue: workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName), + ResourceRecommendTaskIDsMap: mockResourceRecommendTaskIDsMap, +} + +func init() { + mockAggregateTasks.LoadOrStore(mockTaskID1, mockTask1) + mockAggregateTasks.LoadOrStore(emptyTaskID, emptyTask) + mockAggregateTasks.LoadOrStore(valueTypeIllegalTaskID, 3) + + mockResourceRecommendTaskIDsMap[mockNamespacedName1] = &map[datasourcetypes.Metric]processortypes.TaskID{ + mockMetric1: mockTaskID1, + emptyTaskMetric: emptyTaskID, + } + mockResourceRecommendTaskIDsMap[mockNamespacedName2] = &map[datasourcetypes.Metric]processortypes.TaskID{ + notExistTaskIDMetric: processortypes.TaskID(notExistTaskID), + valueTypeIllegalMetric: valueTypeIllegalTaskID, + } + + sampleTime1 := time.Now() + mockTask1.AddSample(sampleTime1.Add(-time.Hour*24), 20, 1000) + mockTask1.AddSample(sampleTime1, 10, 1) +} diff --git a/pkg/controller/resource-recommend/processor/percentile/processor_test.go b/pkg/controller/resource-recommend/processor/percentile/processor_test.go new file mode 100644 index 000000000..3c7a0a5b1 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/processor_test.go @@ -0,0 +1,370 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" + errortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" +) + +func TestProcessor_Register(t *testing.T) { + tests := []struct { + name string + processConfig *processortypes.ProcessConfig + wantCErr *errortypes.CustomError + }{ + { + name: "validate err", + processConfig: &processortypes.ProcessConfig{ + ProcessKey: processortypes.ProcessKey{ + ResourceRecommendNamespacedName: types.NamespacedName{ + Namespace: "ns1", + Name: "n1", + }, + }, + }, + wantCErr: errortypes.RegisterProcessTaskValidateError(errors.New("")), + }, + { + name: "loaded", + processConfig: &mockProcessConfig1, + wantCErr: nil, + }, + { + name: "case1", + processConfig: &processortypes.ProcessConfig{ProcessKey: processortypes.ProcessKey{ + ResourceRecommendNamespacedName: types.NamespacedName{ + Namespace: "testRegisterNamespace1", + Name: "testRegisterName1", + }, + Metric: &datasourcetypes.Metric{ + Namespace: "testRegister", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testRegister", + ContainerName: "testRegister", + Resource: "cpu", + }, + }, + }, + wantCErr: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotCErr := mockProcessor.Register(tt.processConfig) + if gotCErr != nil { + if gotCErr.Phase != tt.wantCErr.Phase || gotCErr.Code != tt.wantCErr.Code { + t.Errorf("Register() = %v, want %v", gotCErr, tt.wantCErr) + } + return + } + tasks, ok := mockProcessor.ResourceRecommendTaskIDsMap[tt.processConfig.ResourceRecommendNamespacedName] + if !ok { + t.Errorf("Register() failed, namespaceName not exist") + return + } + id := (*tasks)[*tt.processConfig.Metric] + if id != tt.processConfig.GenerateTaskID() { + t.Errorf("Register() failed, Metric not exist") + return + } + + if _, ok := mockProcessor.AggregateTasks.Load(id); !ok { + t.Errorf("Register() failed, task not store") + return + } + + }) + } +} + +func TestProcessor_Cancel(t *testing.T) { + processor := Processor{ + TaskQueue: workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName), + AggregateTasks: &sync.Map{}, + ResourceRecommendTaskIDsMap: make(map[types.NamespacedName]*map[datasourcetypes.Metric]processortypes.TaskID), + } + + ns0 := types.NamespacedName{Namespace: "CancelNamespace0", Name: "CancelName0"} + ns1 := types.NamespacedName{Namespace: "CancelNamespace1", Name: "CancelName1"} + ns2 := types.NamespacedName{Namespace: "CancelNamespace2", Name: "CancelName2"} + ns3 := types.NamespacedName{Namespace: "CancelNamespace3", Name: "CancelName3"} + notFoundNs := types.NamespacedName{Namespace: "NamespacedNameNotFound", Name: "NamespacedNameNotFound"} + + pk0 := processortypes.ProcessKey{ + ResourceRecommendNamespacedName: ns0, + Metric: &datasourcetypes.Metric{ + Namespace: "testCancel", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testCancel0", + ContainerName: "testCancel0-1", + Resource: v1.ResourceCPU, + }, + } + pk1 := processortypes.ProcessKey{ + ResourceRecommendNamespacedName: ns1, + Metric: &datasourcetypes.Metric{ + Namespace: "testCancel1", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testCancel1", + ContainerName: "testCancel1-1", + Resource: v1.ResourceCPU, + }, + } + pk2 := processortypes.ProcessKey{ + ResourceRecommendNamespacedName: ns1, + Metric: &datasourcetypes.Metric{ + Namespace: "testCancel1", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testCancel1", + ContainerName: "testCancel1-1", + Resource: v1.ResourceMemory, + }, + } + pk3 := processortypes.ProcessKey{ + ResourceRecommendNamespacedName: ns2, + Metric: &datasourcetypes.Metric{ + Namespace: "testCancel", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testCancel2", + ContainerName: "testCancel2-1", + Resource: v1.ResourceMemory, + }, + } + pk4 := processortypes.ProcessKey{ + ResourceRecommendNamespacedName: ns2, + Metric: &datasourcetypes.Metric{ + Namespace: "testCancel", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testCancel2", + ContainerName: "testCancel2-2", + Resource: v1.ResourceMemory, + }, + } + pk5 := processortypes.ProcessKey{ + ResourceRecommendNamespacedName: ns3, + Metric: &datasourcetypes.Metric{ + Namespace: "testCancel", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testCancel3", + ContainerName: "testCancel3-1", + Resource: v1.ResourceMemory, + }, + } + processConfig0 := processortypes.ProcessConfig{ProcessKey: pk0} + processConfig1 := processortypes.ProcessConfig{ProcessKey: pk1} + processConfig2 := processortypes.ProcessConfig{ProcessKey: pk2} + processConfig3 := processortypes.ProcessConfig{ProcessKey: pk3} + processConfig4 := processortypes.ProcessConfig{ProcessKey: pk4} + processConfig5 := processortypes.ProcessConfig{ProcessKey: pk5} + + _ = processor.Register(&processConfig0) + _ = processor.Register(&processConfig1) + _ = processor.Register(&processConfig2) + _ = processor.Register(&processConfig3) + _ = processor.Register(&processConfig4) + _ = processor.Register(&processConfig5) + + type want struct { + existTaskIDs []processortypes.TaskID + notExistTaskIDs []processortypes.TaskID + namespaceTaskIsNil bool + } + tests := []struct { + name string + testProcessKey *processortypes.ProcessKey + wantCErr *errortypes.CustomError + want want + }{ + { + name: "case1", + testProcessKey: nil, + wantCErr: nil, + }, + { + name: "NamespacedName not found", + testProcessKey: &processortypes.ProcessKey{ResourceRecommendNamespacedName: notFoundNs}, + wantCErr: errortypes.NotFoundTasksError(notFoundNs), + }, + { + name: "metric_not_found", + testProcessKey: &processortypes.ProcessKey{ + ResourceRecommendNamespacedName: ns0, + Metric: &datasourcetypes.Metric{WorkloadName: "metricNotFound"}, + }, + want: want{ + existTaskIDs: []processortypes.TaskID{processConfig0.GenerateTaskID()}, + notExistTaskIDs: []processortypes.TaskID{}, + namespaceTaskIsNil: false, + }, + }, + { + name: "delete_task", + testProcessKey: &pk1, + wantCErr: nil, + want: want{ + existTaskIDs: []processortypes.TaskID{processConfig2.GenerateTaskID()}, + notExistTaskIDs: []processortypes.TaskID{processConfig1.GenerateTaskID()}, + namespaceTaskIsNil: false, + }, + }, + { + name: "delete_all_task_belong_to_NamespacedName", + testProcessKey: &processortypes.ProcessKey{ResourceRecommendNamespacedName: ns2}, + wantCErr: nil, + want: want{ + existTaskIDs: []processortypes.TaskID{}, + notExistTaskIDs: []processortypes.TaskID{processConfig3.GenerateTaskID(), processConfig4.GenerateTaskID()}, + namespaceTaskIsNil: true, + }, + }, + { + name: "delete_task_and_NamespacedName_in_map", + testProcessKey: &pk5, + wantCErr: nil, + want: want{ + existTaskIDs: []processortypes.TaskID{}, + notExistTaskIDs: []processortypes.TaskID{processConfig5.GenerateTaskID()}, + namespaceTaskIsNil: true, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotCErr := processor.Cancel(tt.testProcessKey) + if gotCErr != nil { + if gotCErr.Phase != tt.wantCErr.Phase || gotCErr.Code != tt.wantCErr.Code { + t.Errorf("Cancel() = %v, want %v", gotCErr, tt.wantCErr) + } + return + } + if tt.testProcessKey == nil && tt.wantCErr == nil { + return + } + tasks, ok := processor.ResourceRecommendTaskIDsMap[tt.testProcessKey.ResourceRecommendNamespacedName] + if tt.want.namespaceTaskIsNil || tt.testProcessKey.Metric == nil { + if ok { + t.Errorf("Cancel() failed, tasks for namespaceName(%s) exist", tt.testProcessKey.ResourceRecommendNamespacedName) + return + } + } else { + if !ok || tasks == nil { + t.Errorf("Cancel() failed, tasks for namespaceName(%s) not exist", tt.testProcessKey.ResourceRecommendNamespacedName) + return + } + if _, ok := (*tasks)[*tt.testProcessKey.Metric]; ok { + t.Errorf("Cancel() failed, task for Metric(%s) exist", *tt.testProcessKey.Metric) + } + } + for _, id := range tt.want.notExistTaskIDs { + if _, ok := processor.AggregateTasks.Load(id); ok { + t.Errorf("Cancel() failed, task for taskID(%s) exist", id) + } + } + for _, id := range tt.want.existTaskIDs { + if _, ok := processor.AggregateTasks.Load(id); !ok { + t.Errorf("Cancel() failed, task for taskID(%s) not exist", id) + } + } + }) + } +} + +func TestProcessor_Run(t *testing.T) { + processor := Processor{ + TaskQueue: workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName), + AggregateTasks: &sync.Map{}, + ResourceRecommendTaskIDsMap: make(map[types.NamespacedName]*map[datasourcetypes.Metric]processortypes.TaskID), + } + type args struct { + ctx context.Context + } + + tests := []struct { + name string + args args + }{ + { + name: "case", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx1, cancelFunc := context.WithCancel(context.Background()) + go processor.Run(ctx1) + time.Sleep(2 * time.Second) + cancelFunc() + }) + } +} + +func TestProcessor_QueryProcessedValues(t *testing.T) { + tests := []struct { + name string + processKey *processortypes.ProcessKey + want float64 + wantErr bool + }{ + { + name: "case1", + processKey: &metricIsNilProcessorKey, + wantErr: true, + }, + { + name: "case2", + processKey: &emptyTaskProcessorKey, + wantErr: true, + }, + { + name: "case3", + processKey: &mockProcessKey1, + wantErr: false, + want: mockTask195PercentileValue, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := mockProcessor.QueryProcessedValues(tt.processKey) + if (err != nil) != tt.wantErr { + t.Errorf("QueryProcessedValues() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr { + assert.InEpsilon(t, tt.want, got, 1e-5) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/processor/percentile/task/config.go b/pkg/controller/resource-recommend/processor/percentile/task/config.go new file mode 100644 index 000000000..1762dc460 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/task/config.go @@ -0,0 +1,133 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "time" + + "github.com/pkg/errors" + "gopkg.in/yaml.v3" + v1 "k8s.io/api/core/v1" + vpautil "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/util" + + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" +) + +const ( + // DefaultSampleWeight is the default weight of any sample (prior to including decaying factor) + DefaultSampleWeight = float64(1) + // DefaultMinSampleWeight is the minimal sample weight of any sample (prior to including decaying factor) + DefaultMinSampleWeight = float64(0.1) + // DefaultEpsilon is the minimal weight kept in histograms, it should be small enough that old samples + // (just inside MemoryAggregationWindowLength) added with DefaultMinSampleWeight are still kept + DefaultEpsilon = 0.001 * DefaultMinSampleWeight + // DefaultHistogramBucketSizeGrowth is the default value for HistogramBucketSizeGrowth. + DefaultHistogramBucketSizeGrowth = 0.05 // Make each bucket 5% larger than the previous one. + + // DefaultCPUHistogramMaxValue CPU histograms max bucket size is 1000.0 cores + DefaultCPUHistogramMaxValue = 1000.0 + // DefaultCPUHistogramFirstBucketSize CPU histograms smallest bucket size is 0.01 cores + DefaultCPUHistogramFirstBucketSize = 0.01 + + // DefaultMemHistogramMaxValue Mem histograms max bucket size is 1TB + DefaultMemHistogramMaxValue = 1e12 + // DefaultMemHistogramFirstBucketSize Mem histograms smallest bucket size is 10MB + DefaultMemHistogramFirstBucketSize = 1e7 + + DefaultCPUTaskProcessInterval = time.Minute * 10 + DefaultMemTaskProcessInterval = time.Hour * 1 + + // DefaultHistogramDecayHalfLife is the default value for HistogramDecayHalfLife. + DefaultHistogramDecayHalfLife = time.Hour * 24 + + // DefaultInitDataLength is default data query span for the first run of the task + DefaultInitDataLength = time.Hour * 25 + + // MaxOfAllowNotExecutionTime is maximum non-run tolerance time for a task + MaxOfAllowNotExecutionTime = 50 * time.Hour +) + +func cpuHistogramOptions() vpautil.HistogramOptions { + // CPU histograms use exponential bucketing scheme with the smallest bucket size of 0.01 core, max of 1000.0 cores + options, err := vpautil.NewExponentialHistogramOptions(DefaultCPUHistogramMaxValue, DefaultCPUHistogramFirstBucketSize, + 1.+DefaultHistogramBucketSizeGrowth, DefaultEpsilon) + if err != nil { + panic("Invalid CPU histogram options") // Should not happen. + } + return options +} + +func memoryHistogramOptions() vpautil.HistogramOptions { + // Memory histograms use exponential bucketing scheme with the smallest bucket size of 10MB, max of 1TB + options, err := vpautil.NewExponentialHistogramOptions(DefaultMemHistogramMaxValue, DefaultMemHistogramFirstBucketSize, + 1.+DefaultHistogramBucketSizeGrowth, DefaultEpsilon) + if err != nil { + panic("Invalid memory histogram options") // Should not happen. + } + return options +} + +func HistogramOptionsFactory(resourceName v1.ResourceName) (vpautil.HistogramOptions, error) { + switch resourceName { + case v1.ResourceCPU: + return cpuHistogramOptions(), nil + case v1.ResourceMemory: + return memoryHistogramOptions(), nil + default: + return nil, errors.Errorf("generate histogram options failed, unknow resource: %s", resourceName) + } +} + +func GetDefaultTaskProcessInterval(resourceName v1.ResourceName) (t time.Duration, err error) { + switch resourceName { + case v1.ResourceCPU: + return DefaultCPUTaskProcessInterval, nil + case v1.ResourceMemory: + return DefaultMemTaskProcessInterval, nil + default: + return t, errors.Errorf("get task process interval failed, unknow resource: %s", resourceName) + } +} + +type ProcessConfig struct { + DecayHalfLife time.Duration +} + +const ( + ProcessConfigHalfLifeKey = "decayHalfLife" +) + +func GetTaskConfig(extensions processortypes.TaskConfigStr) (*ProcessConfig, error) { + processConfig := &ProcessConfig{ + DecayHalfLife: DefaultHistogramDecayHalfLife, + } + if extensions == "" { + return processConfig, nil + } + config := make(map[string]interface{}) + if err := yaml.Unmarshal([]byte(extensions), config); err != nil { + return nil, errors.Wrap(err, "unmarshal process config failed") + } + + if value, ok := config[ProcessConfigHalfLifeKey]; ok { + if halfLife, ok := value.(int); ok { + processConfig.DecayHalfLife = time.Hour * time.Duration(halfLife) + } + } + + return processConfig, nil +} diff --git a/pkg/controller/resource-recommend/processor/percentile/task/config_test.go b/pkg/controller/resource-recommend/processor/percentile/task/config_test.go new file mode 100644 index 000000000..261c688c7 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/task/config_test.go @@ -0,0 +1,190 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "fmt" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" +) + +func TestGetProcessConfig(t *testing.T) { + type args struct { + extensions processortypes.TaskConfigStr + } + tests := []struct { + name string + args args + want *ProcessConfig + wantErr bool + }{ + { + name: "case-1", + args: args{ + extensions: processortypes.TaskConfigStr(`{"decayHalfLife":222,"key2":123}`), + }, + want: &ProcessConfig{ + DecayHalfLife: time.Hour * 24, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetTaskConfig(tt.args.extensions) + if (err != nil) != tt.wantErr { + t.Errorf("GetTaskConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + fmt.Printf("%v", got) + }) + } +} + +func TestHistogramOptionsFactory(t *testing.T) { + type want struct { + BucketsNum int + BucketFind struct { + value float64 + bucket int + } + StartBucket struct { + bucket int + value float64 + } + Epsilon float64 + } + tests := []struct { + name string + args v1.ResourceName + want want + wantErr bool + }{ + { + name: "case1", + args: v1.ResourceCPU, + want: want{ + BucketsNum: 176, + BucketFind: struct { + value float64 + bucket int + }{value: 10, bucket: 80}, + StartBucket: struct { + bucket int + value float64 + }{bucket: 10, value: 0.1257789253554882}, + Epsilon: DefaultEpsilon, + }, + wantErr: false, + }, + { + name: "case2", + args: v1.ResourceMemory, + want: want{ + BucketsNum: 176, + BucketFind: struct { + value float64 + bucket int + }{value: 976875765, bucket: 36}, + StartBucket: struct { + bucket int + value float64 + }{bucket: 90, value: 1.5946073009825298e+10}, + Epsilon: DefaultEpsilon, + }, + wantErr: false, + }, + { + name: "case3", + args: v1.ResourceName("errName"), + want: want{}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := HistogramOptionsFactory(tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("HistogramOptionsFactory() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.wantErr { + return + } + if NumBuckets := got.NumBuckets(); NumBuckets != tt.want.BucketsNum { + t.Errorf("HistogramOptionsFactory() NumBuckets gotT = %v, want %v", NumBuckets, tt.want.BucketsNum) + //return + } + if gotT := got.FindBucket(tt.want.BucketFind.value); gotT != tt.want.BucketFind.bucket { + t.Errorf("HistogramOptionsFactory() FindBucket gotT = %v, want %v", gotT, tt.want.BucketFind.bucket) + //return + } + if gotT := got.GetBucketStart(tt.want.StartBucket.bucket); gotT != tt.want.StartBucket.value { + t.Errorf("HistogramOptionsFactory() GetBucketStart gotT = %v, want %v", gotT, tt.want.StartBucket.value) + //return + } + if gotT := got.Epsilon(); gotT != tt.want.Epsilon { + t.Errorf("HistogramOptionsFactory() Epsilon gotT = %v, want %v", gotT, tt.want.Epsilon) + //return + } + }) + } +} + +func TestGetDefaultTaskProcessInterval(t *testing.T) { + + tests := []struct { + name string + args v1.ResourceName + wantT time.Duration + wantErr bool + }{ + { + name: "case1", + args: v1.ResourceCPU, + wantT: DefaultCPUTaskProcessInterval, + wantErr: false, + }, + { + name: "case2", + args: v1.ResourceMemory, + wantT: DefaultMemTaskProcessInterval, + wantErr: false, + }, + { + name: "case3", + args: v1.ResourceName("errName"), + wantT: 0, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotT, err := GetDefaultTaskProcessInterval(tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("GetDefaultTaskProcessInterval() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotT != tt.wantT { + t.Errorf("GetDefaultTaskProcessInterval() gotT = %v, want %v", gotT, tt.wantT) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/processor/percentile/task/histogram_task.go b/pkg/controller/resource-recommend/processor/percentile/task/histogram_task.go new file mode 100644 index 000000000..3bd81b47c --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/task/histogram_task.go @@ -0,0 +1,191 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "context" + "fmt" + "math" + "runtime/debug" + "sync" + "time" + + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + vpautil "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/util" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/log" + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" +) + +var ( + DataPreparingErr = errors.New("data preparing") + SampleExpirationErr = errors.New("no samples in the last 24 hours") + InsufficientSampleErr = errors.New("The data sample is insufficient to obtain the predicted value") + QuerySamplesIsEmptyErr = errors.New("the sample found is empty") + HistogramTaskRunPanicErr = errors.New("histogram task run panic") +) + +type HistogramTask struct { + mutex sync.Mutex + + histogram vpautil.Histogram + decayHalfLife time.Duration + firstSampleTime time.Time + lastSampleTime time.Time + totalSamplesCount int + + createTime time.Time + lastRunTime time.Time + + metric datasourcetypes.Metric + ProcessInterval time.Duration +} + +func NewTask(metric datasourcetypes.Metric, config processortypes.TaskConfigStr) (*HistogramTask, error) { + processConfig, err := GetTaskConfig(config) + if err != nil { + return nil, errors.Wrapf(err, "New histogram task error, get process config failed, config: %s", config) + } + histogramOptions, err := HistogramOptionsFactory(metric.Resource) + if err != nil { + return nil, errors.Wrap(err, "get histogram options failed") + } + taskProcessInterval, err := GetDefaultTaskProcessInterval(metric.Resource) + if err != nil { + return nil, errors.Wrap(err, "get process interval failed") + } + return &HistogramTask{ + mutex: sync.Mutex{}, + metric: metric, + histogram: vpautil.NewDecayingHistogram(histogramOptions, processConfig.DecayHalfLife), + decayHalfLife: processConfig.DecayHalfLife, + createTime: time.Now(), + ProcessInterval: taskProcessInterval, + }, nil +} + +func (t *HistogramTask) AddSample(sampleTime time.Time, sampleValue float64, sampleWeight float64) { + t.histogram.AddSample(sampleValue, sampleWeight, sampleTime) + if t.lastSampleTime.Before(sampleTime) { + t.lastSampleTime = sampleTime + } + if t.firstSampleTime.IsZero() || t.firstSampleTime.After(sampleTime) { + t.firstSampleTime = sampleTime + } + t.totalSamplesCount++ +} + +func (t *HistogramTask) AddRangeSample(firstSampleTime, lastSampleTime time.Time, sampleValue float64, sampleWeight float64) { + t.histogram.AddSample(sampleValue, sampleWeight, lastSampleTime) + if t.lastSampleTime.Before(lastSampleTime) { + t.lastSampleTime = lastSampleTime + } + if t.firstSampleTime.IsZero() || t.firstSampleTime.After(firstSampleTime) { + t.firstSampleTime = firstSampleTime + } + t.totalSamplesCount++ +} + +func (t *HistogramTask) Run(ctx context.Context, datasourceProxy *datasource.Proxy) (nextRunInterval time.Duration, err error) { + defer func() { + if r := recover(); r != nil { + log.ErrorS(ctx, HistogramTaskRunPanicErr, fmt.Sprintf("%v", r), "stack", string(debug.Stack())) + err = HistogramTaskRunPanicErr + } + }() + + t.mutex.Lock() + defer t.mutex.Unlock() + + log.InfoS(ctx, "percentile process task run") + + runSectionBegin := t.lastRunTime + runSectionEnd := time.Now() + t.lastRunTime = runSectionEnd + if runSectionBegin.IsZero() { + runSectionBegin = runSectionEnd.Add(-DefaultInitDataLength) + } + ctx = log.SetKeysAndValues(ctx, "runSectionBegin", runSectionBegin.String(), "runSectionEnd", runSectionEnd.String()) + + timeSeries, err := datasourceProxy.QueryTimeSeries(datasource.PrometheusDatasource, t.metric, runSectionBegin, runSectionEnd, time.Minute) + if err != nil { + log.ErrorS(ctx, err, "task handler error, query samples failed") + return 0, err + } + samplesOverview := datasourcetypes.GetSamplesOverview(timeSeries) + if samplesOverview == nil { + log.ErrorS(ctx, QuerySamplesIsEmptyErr, "No sample found, histogram task run termination") + return 0, QuerySamplesIsEmptyErr + } + log.InfoS(ctx, "percentile process task query samples", "samplesOverview", samplesOverview) + + switch t.metric.Resource { + case v1.ResourceMemory: + lastSampleTime := time.Unix(samplesOverview.LastTimestamp, 0) + firstSampleTime := time.Unix(samplesOverview.FirstTimestamp, 0) + t.AddRangeSample(firstSampleTime, lastSampleTime, samplesOverview.MaxValue, DefaultSampleWeight) + log.V(5).InfoS(ctx, "Mem peak sample added.", + "firstSampleTime", firstSampleTime, "lastSampleTime", lastSampleTime, + "SampleValue", samplesOverview.MaxValue, "SampleNum", len(timeSeries.Samples)) + case v1.ResourceCPU: + for _, sample := range timeSeries.Samples { + sampleTime := time.Unix(sample.Timestamp, 0) + sampleWeight := math.Max(sample.Value, DefaultMinSampleWeight) + t.AddSample(sampleTime, sample.Value, sampleWeight) + log.V(5).InfoS(ctx, "CPU sample added", + "sampleTime", sampleTime, "sampleWeight", sampleWeight, "SampleValue", sample.Value) + } + } + + log.InfoS(ctx, "percentile process task run finished") + return t.ProcessInterval, nil +} + +func (t *HistogramTask) QueryPercentileValue(ctx context.Context, percentile float64) (float64, error) { + t.mutex.Lock() + defer t.mutex.Unlock() + + if t.firstSampleTime.IsZero() { + return 0, DataPreparingErr + } + if time.Now().Sub(t.lastSampleTime) > 24*time.Hour { + return 0, SampleExpirationErr + } + if t.lastSampleTime.Sub(t.firstSampleTime) < time.Hour*24 { + return 0, InsufficientSampleErr + } + //TODO: Check whether the ratio between the count of samples and the running time meets the requirements + + percentileValue := t.histogram.Percentile(percentile) + + log.InfoS(ctx, "Query Processed Values", + "lastSampleTime", t.lastSampleTime, "firstSampleTime", t.firstSampleTime, + "totalSamplesCount", t.totalSamplesCount, "percentileValue", percentileValue) + return percentileValue, nil +} + +func (t *HistogramTask) IsTimeoutNotExecute() bool { + baseTime := t.createTime + if !t.lastRunTime.IsZero() { + baseTime = t.lastRunTime + } + // If there is no execution record within 10 task execution intervals, the task is regarded as invalid + return baseTime.Add(MaxOfAllowNotExecutionTime).Before(time.Now()) +} diff --git a/pkg/controller/resource-recommend/processor/percentile/task/histogram_task_test.go b/pkg/controller/resource-recommend/processor/percentile/task/histogram_task_test.go new file mode 100644 index 000000000..50c4989c3 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/task/histogram_task_test.go @@ -0,0 +1,490 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" +) + +func TestHistogramTask_AddSample(t1 *testing.T) { + type newTask func() *HistogramTask + type sample struct { + sampleTime time.Time + sampleValue float64 + sampleWeight float64 + } + type want struct { + lastSampleTime time.Time + firstSampleTime time.Time + totalSamplesCount int + } + tests := []struct { + name string + newTask newTask + samples []sample + want + }{ + { + name: "case1", + newTask: func() *HistogramTask { + t, _ := NewTask(datasourcetypes.Metric{Resource: v1.ResourceCPU}, "") + return t + }, + samples: []sample{ + { + sampleTime: time.Unix(1694270256, 0), + sampleValue: 1, + sampleWeight: 1, + }, + { + sampleTime: time.Unix(1694270765, 0), + sampleValue: 1, + sampleWeight: 1, + }, + { + sampleTime: time.Unix(1694278758, 0), + sampleValue: 1, + sampleWeight: 1, + }, + { + sampleTime: time.Unix(1694278700, 0), + sampleValue: 1, + sampleWeight: 1, + }, + }, + want: want{ + lastSampleTime: time.Unix(1694278758, 0), + firstSampleTime: time.Unix(1694270256, 0), + totalSamplesCount: 4, + }, + }, + } + for _, tt := range tests { + t1.Run(tt.name, func(t1 *testing.T) { + t := tt.newTask() + for _, sample := range tt.samples { + t.AddSample(sample.sampleTime, sample.sampleValue, sample.sampleWeight) + } + if tt.lastSampleTime != t.lastSampleTime { + t1.Errorf("AddSample() got lastSampleTime=%s, want lastSampleTime=%s", t.lastSampleTime, tt.lastSampleTime) + } + if tt.firstSampleTime != t.firstSampleTime { + t1.Errorf("AddSample() got firstSampleTime=%s, want firstSampleTime=%s", t.firstSampleTime, tt.firstSampleTime) + } + if tt.totalSamplesCount != t.totalSamplesCount { + t1.Errorf("AddSample() got totalSamplesCount=%d, want totalSamplesCount=%d", t.totalSamplesCount, tt.totalSamplesCount) + } + }) + } +} + +func TestHistogramTask_AddRangeSample(t1 *testing.T) { + type newTask func() *HistogramTask + type sample struct { + firstSampleTime time.Time + lastSampleTime time.Time + sampleValue float64 + sampleWeight float64 + } + type want struct { + lastSampleTime time.Time + firstSampleTime time.Time + totalSamplesCount int + } + tests := []struct { + name string + newTask newTask + samples []sample + want + }{ + { + name: "case1", + newTask: func() *HistogramTask { + t, _ := NewTask(datasourcetypes.Metric{Resource: v1.ResourceCPU}, "") + return t + }, + samples: []sample{ + { + firstSampleTime: time.Unix(1694270256, 0), + lastSampleTime: time.Unix(1694270768, 0), + sampleValue: 1, + sampleWeight: 1, + }, + { + firstSampleTime: time.Unix(1694270769, 0), + lastSampleTime: time.Unix(1694271769, 0), + sampleValue: 1, + sampleWeight: 1, + }, + }, + want: want{ + lastSampleTime: time.Unix(1694271769, 0), + firstSampleTime: time.Unix(1694270256, 0), + totalSamplesCount: 2, + }, + }, + } + for _, tt := range tests { + t1.Run(tt.name, func(t1 *testing.T) { + t := tt.newTask() + for _, sample := range tt.samples { + t.AddRangeSample(sample.firstSampleTime, sample.lastSampleTime, sample.sampleValue, sample.sampleWeight) + } + if tt.lastSampleTime != t.lastSampleTime { + t1.Errorf("AddSample() got lastSampleTime=%s, want lastSampleTime=%s", t.lastSampleTime, tt.lastSampleTime) + } + if tt.firstSampleTime != t.firstSampleTime { + t1.Errorf("AddSample() got firstSampleTime=%s, want firstSampleTime=%s", t.firstSampleTime, tt.firstSampleTime) + } + if tt.totalSamplesCount != t.totalSamplesCount { + t1.Errorf("AddSample() got totalSamplesCount=%d, want totalSamplesCount=%d", t.totalSamplesCount, tt.totalSamplesCount) + } + }) + } +} + +type mockDatasourceEmptyQuery struct{} + +func (m1 *mockDatasourceEmptyQuery) QueryTimeSeries(_ *datasourcetypes.Query, _ time.Time, _ time.Time, _ time.Duration) (*datasourcetypes.TimeSeries, error) { + return &datasourcetypes.TimeSeries{Samples: []datasourcetypes.Sample{}}, nil +} + +func (m1 *mockDatasourceEmptyQuery) ConvertMetricToQuery(_ datasourcetypes.Metric) (*datasourcetypes.Query, error) { + return nil, nil +} + +type mockDatasourcePanic struct{} + +func (m1 *mockDatasourcePanic) QueryTimeSeries(_ *datasourcetypes.Query, _ time.Time, _ time.Time, _ time.Duration) (*datasourcetypes.TimeSeries, error) { + panic("test panic") +} + +func (m1 *mockDatasourcePanic) ConvertMetricToQuery(_ datasourcetypes.Metric) (*datasourcetypes.Query, error) { + return nil, nil +} + +type mockDatasource struct{} + +func (m1 *mockDatasource) QueryTimeSeries(_ *datasourcetypes.Query, _ time.Time, _ time.Time, _ time.Duration) (*datasourcetypes.TimeSeries, error) { + return &datasourcetypes.TimeSeries{Samples: []datasourcetypes.Sample{ + { + Timestamp: 1694270256, + Value: 1, + }, + { + Timestamp: 1694270765, + Value: 2, + }, + { + Timestamp: 1694278758, + Value: 3, + }, + { + Timestamp: 1694278700, + Value: 4, + }, + }}, nil +} + +func (m1 *mockDatasource) ConvertMetricToQuery(_ datasourcetypes.Metric) (*datasourcetypes.Query, error) { + return nil, nil +} + +func TestHistogramTask_Run(t1 *testing.T) { + type newTask func() *HistogramTask + type args struct { + ctx context.Context + datasourceProxy func() *datasource.Proxy + } + type want struct { + lastSampleTime time.Time + firstSampleTime time.Time + totalSamplesCount int + } + tests := []struct { + name string + newTask newTask + args args + wantNextRunInterval time.Duration + wantErr error + want want + }{ + { + name: "empty samples", + newTask: func() *HistogramTask { + task, _ := NewTask(datasourcetypes.Metric{Resource: v1.ResourceCPU}, "") + return task + }, + args: args{ + ctx: context.Background(), + datasourceProxy: func() *datasource.Proxy { + proxy := datasource.NewProxy() + proxy.RegisterDatasource(datasource.PrometheusDatasource, &mockDatasourceEmptyQuery{}) + return proxy + }, + }, + wantNextRunInterval: 0, + wantErr: QuerySamplesIsEmptyErr, + }, + { + name: "panic", + newTask: func() *HistogramTask { + task, _ := NewTask(datasourcetypes.Metric{Resource: v1.ResourceCPU}, "") + return task + }, + args: args{ + ctx: context.Background(), + datasourceProxy: func() *datasource.Proxy { + proxy := datasource.NewProxy() + proxy.RegisterDatasource(datasource.PrometheusDatasource, &mockDatasourcePanic{}) + return proxy + }, + }, + wantNextRunInterval: 0, + wantErr: HistogramTaskRunPanicErr, + }, + { + name: "cpu run", + newTask: func() *HistogramTask { + task, _ := NewTask(datasourcetypes.Metric{Resource: v1.ResourceCPU}, "") + return task + }, + args: args{ + ctx: context.Background(), + datasourceProxy: func() *datasource.Proxy { + proxy := datasource.NewProxy() + proxy.RegisterDatasource(datasource.PrometheusDatasource, &mockDatasource{}) + return proxy + }, + }, + wantNextRunInterval: DefaultCPUTaskProcessInterval, + wantErr: nil, + want: want{ + lastSampleTime: time.Unix(1694278758, 0), + firstSampleTime: time.Unix(1694270256, 0), + totalSamplesCount: 4, + }, + }, + { + name: "mem run", + newTask: func() *HistogramTask { + task, _ := NewTask(datasourcetypes.Metric{Resource: v1.ResourceMemory}, "") + return task + }, + args: args{ + ctx: context.Background(), + datasourceProxy: func() *datasource.Proxy { + proxy := datasource.NewProxy() + proxy.RegisterDatasource(datasource.PrometheusDatasource, &mockDatasource{}) + return proxy + }, + }, + wantNextRunInterval: DefaultMemTaskProcessInterval, + wantErr: nil, + want: want{ + lastSampleTime: time.Unix(1694278758, 0), + firstSampleTime: time.Unix(1694270256, 0), + totalSamplesCount: 1, + }, + }, + } + for _, tt := range tests { + t1.Run(tt.name, func(t1 *testing.T) { + t := tt.newTask() + gotNextRunInterval, err := t.Run(tt.args.ctx, tt.args.datasourceProxy()) + if tt.wantErr != nil && err.Error() != tt.wantErr.Error() { + t1.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.wantErr == nil { + if gotNextRunInterval != tt.wantNextRunInterval { + t1.Errorf("Run() gotNextRunInterval = %v, want %v", gotNextRunInterval, tt.wantNextRunInterval) + } + if tt.want.lastSampleTime != t.lastSampleTime { + t1.Errorf("Run() got lastSampleTime=%s, want lastSampleTime=%s", t.lastSampleTime, tt.want.lastSampleTime) + } + if tt.want.firstSampleTime != t.firstSampleTime { + t1.Errorf("Run() got firstSampleTime=%s, want firstSampleTime=%s", t.firstSampleTime, tt.want.firstSampleTime) + } + if tt.want.totalSamplesCount != t.totalSamplesCount { + t1.Errorf("Run() got totalSamplesCount=%d, want totalSamplesCount=%d", t.totalSamplesCount, tt.want.totalSamplesCount) + } + } + }) + } +} + +func TestHistogramTask_QueryPercentileValue(t1 *testing.T) { + type newTask func() *HistogramTask + type args struct { + ctx context.Context + percentile float64 + } + tests := []struct { + name string + newTask newTask + args args + want float64 + wantErr error + }{ + { + name: "case0", + newTask: func() *HistogramTask { + return &HistogramTask{ + lastSampleTime: time.Now().Add(-25 * time.Hour), + } + }, + args: args{ + ctx: context.Background(), + percentile: 0.9, + }, + want: 0.0, + wantErr: DataPreparingErr, + }, + { + name: "case1", + newTask: func() *HistogramTask { + return &HistogramTask{ + firstSampleTime: time.Now().Add(-26 * time.Hour), + lastSampleTime: time.Now().Add(-25 * time.Hour), + } + }, + args: args{ + ctx: context.Background(), + percentile: 0.9, + }, + want: 0.0, + wantErr: SampleExpirationErr, + }, + { + name: "case2", + newTask: func() *HistogramTask { + return &HistogramTask{ + lastSampleTime: time.Now(), + firstSampleTime: time.Now().Add(-20 * time.Hour), + } + }, + args: args{ + ctx: context.Background(), + percentile: 0.9, + }, + want: 0.0, + wantErr: InsufficientSampleErr, + }, + { + name: "case3", + newTask: func() *HistogramTask { + task, _ := NewTask(datasourcetypes.Metric{Resource: v1.ResourceCPU}, "") + sampleTime := time.Now() + task.AddSample(sampleTime.Add(-time.Hour*24), 20, 1000) + task.AddSample(sampleTime, 10, 1) + return task + }, + args: args{ + ctx: context.Background(), + percentile: 0.9, + }, + want: 20.40693, + wantErr: nil, + }, + { + name: "case4", + newTask: func() *HistogramTask { + task, _ := NewTask(datasourcetypes.Metric{Resource: v1.ResourceCPU}, "") + sampleTime := time.Now() + task.AddSample(sampleTime.Add(-time.Hour*24*20), 20, 1000) + task.AddSample(sampleTime, 10, 1) + return task + }, + args: args{ + ctx: context.Background(), + percentile: 0.9, + }, + want: 10.207902, + wantErr: nil, + }, + } + for _, tt := range tests { + t1.Run(tt.name, func(t1 *testing.T) { + t := tt.newTask() + got, err := t.QueryPercentileValue(tt.args.ctx, tt.args.percentile) + if tt.wantErr != nil && err.Error() != tt.wantErr.Error() { + t1.Errorf("QueryPercentileValue() error = %v, WantErr %v", err, tt.wantErr) + return + } + + if tt.wantErr == nil { + assert.InEpsilon(t1, tt.want, got, 1e-5) + } + }) + } +} + +func TestHistogramTask_IsTimeoutNotExecute(t1 *testing.T) { + type newTask func() *HistogramTask + tests := []struct { + name string + newFunc newTask + want bool + }{ + { + name: "case1", + newFunc: func() *HistogramTask { + lastRunTime := time.Now() + return &HistogramTask{ + lastRunTime: lastRunTime, + createTime: lastRunTime.Add(-48 * time.Minute), + } + }, + want: false, + }, + { + name: "case1", + newFunc: func() *HistogramTask { + return &HistogramTask{ + lastRunTime: time.Now().Add(-(MaxOfAllowNotExecutionTime + time.Minute)), + } + }, + want: true, + }, + { + name: "case2", + newFunc: func() *HistogramTask { + return &HistogramTask{ + createTime: time.Now().Add(-(MaxOfAllowNotExecutionTime + time.Minute)), + } + }, + want: true, + }, + } + for _, tt := range tests { + t1.Run(tt.name, func(t1 *testing.T) { + t := tt.newFunc() + if got := t.IsTimeoutNotExecute(); got != tt.want { + t1.Errorf("IsTimeoutNotExecute() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/processor/processor.go b/pkg/controller/resource-recommend/processor/processor.go new file mode 100644 index 000000000..fa9027bfe --- /dev/null +++ b/pkg/controller/resource-recommend/processor/processor.go @@ -0,0 +1,35 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package processor + +import ( + "context" + + errortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" +) + +type Processor interface { + // Run performs the prediction routine. + Run(ctx context.Context) + + Register(processConfig *processortypes.ProcessConfig) *errortypes.CustomError + + Cancel(processKey *processortypes.ProcessKey) *errortypes.CustomError + + QueryProcessedValues(taskKey *processortypes.ProcessKey) (float64, error) +} diff --git a/pkg/controller/resource-recommend/recommender/manager/recommender_manager.go b/pkg/controller/resource-recommend/recommender/manager/recommender_manager.go new file mode 100644 index 000000000..dae6a6c0a --- /dev/null +++ b/pkg/controller/resource-recommend/recommender/manager/recommender_manager.go @@ -0,0 +1,47 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "k8s.io/klog/v2" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/oom" + processormanager "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/manager" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/recommender" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/recommender/recommenders" +) + +type Manager struct { + ProcessorManager processormanager.Manager + OomRecorder oom.Recorder +} + +func NewManager(ProcessorManager processormanager.Manager, OomRecorder oom.Recorder) *Manager { + return &Manager{ + ProcessorManager: ProcessorManager, + OomRecorder: OomRecorder} +} + +func (m *Manager) NewRecommender(algorithm v1alpha1.Algorithm) recommender.Recommender { + switch algorithm { + case v1alpha1.AlgorithmPercentile: + return recommenders.NewPercentileRecommender(m.ProcessorManager.GetProcessor(v1alpha1.AlgorithmPercentile), m.OomRecorder) + } + klog.InfoS("no recommender matched. fall through to default percentile recommender") + return recommenders.NewPercentileRecommender(m.ProcessorManager.GetProcessor(v1alpha1.AlgorithmPercentile), m.OomRecorder) +} diff --git a/pkg/controller/resource-recommend/recommender/recommender.go b/pkg/controller/resource-recommend/recommender/recommender.go new file mode 100644 index 000000000..6869df13a --- /dev/null +++ b/pkg/controller/resource-recommend/recommender/recommender.go @@ -0,0 +1,26 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommender + +import ( + errortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" + recommendationtypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/recommendation" +) + +type Recommender interface { + Recommend(recommendation *recommendationtypes.Recommendation) *errortypes.CustomError +} diff --git a/pkg/controller/resource-recommend/recommender/recommenders/percentile_recommender.go b/pkg/controller/resource-recommend/recommender/recommenders/percentile_recommender.go new file mode 100644 index 000000000..7477628c5 --- /dev/null +++ b/pkg/controller/resource-recommend/recommender/recommenders/percentile_recommender.go @@ -0,0 +1,167 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommenders + +import ( + "strings" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + vpamodel "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + "k8s.io/klog/v2" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/oom" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/recommender" + "github.com/kubewharf/katalyst-core/pkg/util/general" + errortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" + recommendationtype "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/recommendation" +) + +type PercentileRecommender struct { + recommender.Recommender + DataProcessor processor.Processor + OomRecorder oom.Recorder +} + +const ( + // OOMBumpUpRatio specifies how much memory will be added after observing OOM. + OOMBumpUpRatio float64 = 1.2 + // OOMMinBumpUp specifies minimal increase of memory after observing OOM. + OOMMinBumpUp float64 = 100 * 1024 * 1024 // 100MB +) + +// NewPercentileRecommender returns a +func NewPercentileRecommender(DataProcessor processor.Processor, OomRecorder oom.Recorder) *PercentileRecommender { + return &PercentileRecommender{ + DataProcessor: DataProcessor, + OomRecorder: OomRecorder} +} + +func (r *PercentileRecommender) Recommend(recommendation *recommendationtype.Recommendation) *errortypes.CustomError { + klog.InfoS("starting recommenders process", "recommendationConfig", recommendation.Config) + for _, container := range recommendation.Config.Containers { + containerRecommendation := v1alpha1.ContainerResources{ + ContainerName: container.ContainerName, + } + requests := v1alpha1.ContainerResourceList{ + Target: map[v1.ResourceName]resource.Quantity{}, + } + for _, containerConfig := range container.ContainerConfigs { + taskKey := processortypes.GetProcessKey(recommendation.NamespacedName, recommendation.Config.TargetRef, container.ContainerName, containerConfig.ControlledResource) + switch containerConfig.ControlledResource { + case v1.ResourceCPU: + cpuQuantity, err := r.getCpuTargetPercentileEstimationWithUsageBuffer(&taskKey, float64(containerConfig.ResourceBufferPercent)/100) + if err != nil { + return errortypes.RecommendationNotReadyError(err.Error()) + } + klog.InfoS("got recommended cpu for container", "recommendedCPU", cpuQuantity.String(), "container", container.ContainerName) + requests.Target[v1.ResourceCPU] = *cpuQuantity + case v1.ResourceMemory: + memQuantity, err := r.getMemTargetPercentileEstimationWithUsageBuffer(&taskKey, float64(containerConfig.ResourceBufferPercent)/100) + if err != nil { + return errortypes.RecommendationNotReadyError(err.Error()) + } + requests.Target[v1.ResourceMemory] = *memQuantity + } + } + containerRecommendation.Requests = &requests + recommendation.Recommendations = append(recommendation.Recommendations, containerRecommendation) + } + klog.InfoS("recommenders process done", "recommendation", general.StructToString(recommendation.Recommendations)) + return nil +} + +func (r *PercentileRecommender) ScaleOnOOM(oomRecords []oom.OOMRecord, namespace string, workloadName string, containerName string) *resource.Quantity { + klog.InfoS("scaling on oom for namespace, workload, container", "namespace", namespace, "workload", workloadName, "container", containerName) + var oomRecord *oom.OOMRecord + for _, record := range oomRecords { + // use oomRecord for all pods in workload + if strings.HasPrefix(record.Pod, workloadName) && containerName == record.Container && namespace == record.Namespace { + oomRecord = &record + break + } + } + + // ignore too old oom events + if oomRecord != nil && time.Since(oomRecord.OOMAt) <= (time.Hour*24*7) { + memoryOOM := oomRecord.Memory.Value() + var memoryNeeded vpamodel.ResourceAmount + memoryNeeded = vpamodel.ResourceAmountMax(vpamodel.ResourceAmount(memoryOOM)+vpamodel.MemoryAmountFromBytes(OOMMinBumpUp), + vpamodel.ScaleResource(vpamodel.ResourceAmount(memoryOOM), OOMBumpUpRatio)) + + return r.getMemQuantity(float64(memoryNeeded)) + } + + return nil +} + +func (r *PercentileRecommender) getCpuTargetPercentileEstimationWithUsageBuffer(taskKey *processortypes.ProcessKey, resourceBufferPercentage float64) (quantity *resource.Quantity, err error) { + klog.InfoS("getting cpu estimation for namespace, workload, container, with resource buffer", "namespace", taskKey.Namespace, "workload", taskKey.WorkloadName, "container", taskKey.ContainerName, "resourceBuffer", resourceBufferPercentage) + cpuRecommendedValue, err := r.DataProcessor.QueryProcessedValues(taskKey) + if err != nil { + return nil, err + } + klog.InfoS("got cpu recommended value from processor", "cpuRecommendedValue", cpuRecommendedValue) + //scale cpu resource based on usageBuffer + cpuRecommendedValue = cpuRecommendedValue * (1 + resourceBufferPercentage) + klog.InfoS("scaled cpu recommended value for container", "container", taskKey.ContainerName, "resourceBuffer", resourceBufferPercentage, "cpuRecommendedValue", cpuRecommendedValue) + cpuQuantity := resource.NewMilliQuantity(int64(cpuRecommendedValue*1000), resource.DecimalSI) + return cpuQuantity, nil +} + +func (r *PercentileRecommender) getMemTargetPercentileEstimationWithUsageBuffer(taskKey *processortypes.ProcessKey, resourceBufferPercentage float64) (quantity *resource.Quantity, err error) { + klog.InfoS("getting mem estimation for namespace, workload, container, with resource buffer", "namespace", taskKey.Namespace, "workload", taskKey.WorkloadName, "container", taskKey.ContainerName, "resourceBuffer", resourceBufferPercentage) + memRecommendedValue, err := r.DataProcessor.QueryProcessedValues(taskKey) + if err != nil { + return nil, err + } + klog.InfoS("got mem recommended value from processor", "memRecommendedValue", memRecommendedValue) + //scale mem resource based on usageBuffer + memRecommendedValue = memRecommendedValue * (1 + resourceBufferPercentage) + klog.InfoS("scaled mem recommended value for container", "container", taskKey.ContainerName, "resourceBuffer", resourceBufferPercentage, "memRecommendedValue", memRecommendedValue) + memQuantity := r.getMemQuantity(memRecommendedValue) + klog.InfoS("got recommended memory for container", "container", taskKey.ContainerName, "memory", memQuantity.String()) + oomRecords := r.OomRecorder.ListOOMRecords() + oomScaledMem := r.ScaleOnOOM(oomRecords, taskKey.Namespace, taskKey.WorkloadName, taskKey.ContainerName) + if oomScaledMem != nil && !oomScaledMem.IsZero() && oomScaledMem.Cmp(*memQuantity) > 0 { + klog.InfoS("container using oomProtect Memory", "container", taskKey.ContainerName, "oomScaledMem", oomScaledMem.String()) + memQuantity = oomScaledMem + } + return memQuantity, nil +} + +func (r *PercentileRecommender) getMemQuantity(memRecommendedValue float64) (quantity *resource.Quantity) { + scale := int64(1) + quotient := int64(memRecommendedValue) + remainder := int64(0) + for scale < 1024*1024 { + if quotient < 1024 { + break + } + scale *= 1024 + quotient = int64(memRecommendedValue) / scale + remainder = int64(memRecommendedValue) % scale + } + if remainder == 0 { + return resource.NewQuantity(quotient*scale, resource.BinarySI) + } + return resource.NewQuantity((quotient+1)*scale, resource.BinarySI) +} diff --git a/pkg/controller/resource-recommend/recommender/recommenders/percentile_recommender_test.go b/pkg/controller/resource-recommend/recommender/recommenders/percentile_recommender_test.go new file mode 100644 index 000000000..53e318318 --- /dev/null +++ b/pkg/controller/resource-recommend/recommender/recommenders/percentile_recommender_test.go @@ -0,0 +1,242 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommenders + +import ( + "context" + "reflect" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/oom" + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" + customtypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" + processortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/processor" + recommendationtypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/recommendation" +) + +func TestRecommend(t *testing.T) { + recommendation1 := &recommendationtypes.Recommendation{ + NamespacedName: types.NamespacedName{ + Name: "name1", + Namespace: "namespace1", + }, + Config: recommendationtypes.Config{ + Containers: []recommendationtypes.Container{ + { + ContainerName: "container1", + ContainerConfigs: []recommendationtypes.ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: 10, + }, + { + ControlledResource: v1.ResourceMemory, + ResourceBufferPercent: 10, + }, + }, + }, + }, + TargetRef: v1alpha1.CrossVersionObjectReference{ + Kind: "deployment", + Name: "workload1", + }, + }, + } + + r := &PercentileRecommender{ + DataProcessor: dummyDataProcessor{}, + OomRecorder: dummyOomRecorder{}, + } + err := r.Recommend(recommendation1) + + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + if recommendation1.Recommendations[0].Requests.Target.Cpu().String() != "1100" || recommendation1.Recommendations[0].Requests.Target.Memory().String() != "2Ki" { + t.Errorf("Recommendations mismatch.") + } +} + +func TestGetCpuTargetPercentileEstimationWithUsageBuffer(t *testing.T) { + recommender := &PercentileRecommender{ + DataProcessor: dummyDataProcessor{}, + OomRecorder: dummyOomRecorder{}, + } + taskKey := &processortypes.ProcessKey{ + ResourceRecommendNamespacedName: types.NamespacedName{ + Name: "name1", + Namespace: "namespace1", + }, + Metric: &datasourcetypes.Metric{ + Namespace: "namespace1", + Kind: "deployment1", + WorkloadName: "workload1", + ContainerName: "container1", + Resource: v1.ResourceCPU, + }, + } + resourceBufferPercentage := 0.1 + cpuQuantity, err := recommender.getCpuTargetPercentileEstimationWithUsageBuffer(taskKey, resourceBufferPercentage) + if err != nil { + t.Errorf("Expected no error, but got: %v", err) + } + + expectedCpuQuantity := resource.NewMilliQuantity(int64(1100*1000), resource.DecimalSI) // 设置期望的CPU数量 + if !cpuQuantity.Equal(*expectedCpuQuantity) { + t.Errorf("Expected cpu quantity %s, but got %s", cpuQuantity.String(), expectedCpuQuantity.String()) + } +} + +func TestGetMemTargetPercentileEstimationWithUsageBuffer(t *testing.T) { + recommender := &PercentileRecommender{ + DataProcessor: dummyDataProcessor{}, + OomRecorder: dummyOomRecorder{}, + } + taskKey := &processortypes.ProcessKey{ + ResourceRecommendNamespacedName: types.NamespacedName{ + Name: "name1", + Namespace: "namespace1", + }, + Metric: &datasourcetypes.Metric{ + Namespace: "namespace1", + Kind: "deployment1", + WorkloadName: "workload1", + ContainerName: "container1", + Resource: v1.ResourceMemory, + }, + } + resourceBufferPercentage := 0.1 + memQuantity, err := recommender.getMemTargetPercentileEstimationWithUsageBuffer(taskKey, resourceBufferPercentage) + if err != nil { + t.Errorf("Expected no error, but got: %v", err) + } + + expectedMemQuantity := resource.NewQuantity((1100/1024+1)*1024, resource.BinarySI) // 设置期望的内存数量 + if !memQuantity.Equal(*expectedMemQuantity) { + t.Errorf("Expected memory quantity %s, but got %s", expectedMemQuantity.String(), memQuantity.String()) + } +} + +func TestScaleOnOOM(t *testing.T) { + oomRecords := []oom.OOMRecord{ + { + Pod: "workload-name-pod-1", + Container: "container-1", + Namespace: "namespace-1", + OOMAt: time.Now(), + Memory: *resource.NewQuantity(1024, resource.BinarySI), + }, + { + Pod: "workload-name-pod-2", + Container: "container-2", + Namespace: "namespace-2", + OOMAt: time.Now().Add(-time.Hour * 24 * 8), // too old event + Memory: *resource.NewQuantity(2048, resource.BinarySI), + }, + } + + r := &PercentileRecommender{} + namespace := "namespace-1" + workloadName := "workload-name" + containerName := "container-1" + + quantityPointer := r.ScaleOnOOM(oomRecords, namespace, workloadName, containerName) + + if quantityPointer == nil { + t.Errorf("Expected non-nil quantity, got nil") + return + } + quantity := *quantityPointer + + expectedValuePointer := r.getMemQuantity(float64(oomRecords[0].Memory.Value()) + OOMMinBumpUp) + if expectedValuePointer == nil { + t.Errorf("got expectedValuePointer is nil") + return + } + expectedValue := *expectedValuePointer + if !quantity.Equal(expectedValue) { + t.Errorf("Expected value %s, got %s", expectedValue.String(), quantity.String()) + return + } +} + +type dummyDataProcessor struct{} + +func (d dummyDataProcessor) Run(_ context.Context) { + return +} + +func (d dummyDataProcessor) Register(_ *processortypes.ProcessConfig) *customtypes.CustomError { + return nil +} + +func (d dummyDataProcessor) Cancel(_ *processortypes.ProcessKey) *customtypes.CustomError { + return nil +} + +func (d dummyDataProcessor) QueryProcessedValues(_ *processortypes.ProcessKey) (float64, error) { + return 1000, nil +} + +type dummyOomRecorder struct{} + +func (d dummyOomRecorder) ListOOMRecords() []oom.OOMRecord { + return nil +} + +func (d dummyOomRecorder) ScaleOnOOM(_ []oom.OOMRecord, _, _, _ string) *resource.Quantity { + return nil +} + +func TestPercentileRecommender_getMemQuantity(t *testing.T) { + tests := []struct { + name string + memRecommendedValue float64 + wantQuantity *resource.Quantity + }{ + { + name: "testKi", + memRecommendedValue: 1100, + wantQuantity: resource.NewQuantity(2*1024, resource.BinarySI), + }, + { + name: "testMi", + memRecommendedValue: 2 * 1024 * 1024, + wantQuantity: resource.NewQuantity(2*1024*1024, resource.BinarySI), + }, + { + name: "testMiNotDivisible", + memRecommendedValue: 2.66 * 1024 * 1024, + wantQuantity: resource.NewQuantity(3*1024*1024, resource.BinarySI), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &PercentileRecommender{} + if gotQuantity := r.getMemQuantity(tt.memRecommendedValue); !reflect.DeepEqual(gotQuantity, tt.wantQuantity) { + t.Errorf("getMemQuantity() = %v, want %v", gotQuantity, tt.wantQuantity) + } + }) + } +} diff --git a/pkg/util/general/list.go b/pkg/util/general/list.go new file mode 100644 index 000000000..4aa621835 --- /dev/null +++ b/pkg/util/general/list.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package general + +import "reflect" + +// SliceContains returns true if an element is present in a slice. +func SliceContains(list interface{}, elem interface{}) bool { + if list == nil || elem == nil { + return false + } + listV := reflect.ValueOf(list) + + if listV.Kind() == reflect.Slice { + for i := 0; i < listV.Len(); i++ { + item := listV.Index(i).Interface() + + target := reflect.ValueOf(elem).Convert(reflect.TypeOf(item)).Interface() + if ok := reflect.DeepEqual(item, target); ok { + return true + } + } + } + return false +} diff --git a/pkg/util/general/list_test.go b/pkg/util/general/list_test.go new file mode 100644 index 000000000..fd358c34c --- /dev/null +++ b/pkg/util/general/list_test.go @@ -0,0 +1,147 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package general + +import ( + "testing" +) + +func TestSliceContains(t *testing.T) { + type args struct { + list interface{} + elem interface{} + } + type myType struct { + name string + id int + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "int - true", + args: args{ + list: []int{1, 2, 3}, + elem: 1, + }, + want: true, + }, + { + name: "int - false", + args: args{ + list: []int{1, 2, 3}, + elem: 4, + }, + want: false, + }, + { + name: "string - true", + args: args{ + list: []string{"1", "2", "3"}, + elem: "1", + }, + want: true, + }, + { + name: "string - false", + args: args{ + list: []string{"1", "2", "3"}, + elem: "4", + }, + want: false, + }, + { + name: "string - false", + args: args{ + list: []string{"1", "2", "3"}, + elem: 4, + }, + want: false, + }, + { + name: "string - true", + args: args{ + list: []string{"1", "2", "3"}, + elem: "1", + }, + want: true, + }, + { + name: "myType - true", + args: args{ + list: []myType{ + { + name: "name", + id: 1, + }, + { + name: "name", + id: 2, + }, + }, + elem: myType{ + name: "name", + id: 2, + }, + }, + want: true, + }, + { + name: "myType - false", + args: args{ + list: []myType{ + { + name: "name", + id: 1, + }, + { + name: "name", + id: 2, + }, + }, + elem: myType{ + name: "name-1", + id: 2, + }, + }, + want: false, + }, + { + name: "list - nil", + args: args{ + elem: 1, + }, + want: false, + }, + { + name: "elem - nil", + args: args{ + list: []int{1, 2, 3}, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := SliceContains(tt.args.list, tt.args.elem); got != tt.want { + t.Errorf("SliceContains() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/util/general/string.go b/pkg/util/general/string.go new file mode 100644 index 000000000..f03b7c8d1 --- /dev/null +++ b/pkg/util/general/string.go @@ -0,0 +1,37 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package general + +import ( + "encoding/json" // nolint: byted_substitute_packages + "fmt" +) + +func StructToString(val interface{}) string { + if val == nil { + return "" + } + data, err := json.Marshal(val) + if err != nil { + return fmt.Sprintf("%+v", val) + } + return BytesToString(data) +} + +func BytesToString(b []byte) string { + return string(b) +} diff --git a/pkg/util/general/string_test.go b/pkg/util/general/string_test.go new file mode 100644 index 000000000..ceb48102b --- /dev/null +++ b/pkg/util/general/string_test.go @@ -0,0 +1,55 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package general + +import ( + "testing" +) + +type struct1 struct { + Name string + Age int +} + +func TestStructToString(t *testing.T) { + type args struct { + val interface{} + } + tests := []struct { + name string + args args + want string + }{ + { + name: "case1", + args: args{ + val: struct1{ + Name: "ct1", + Age: 18, + }, + }, + want: `{"Name":"ct1","Age":18}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := StructToString(tt.args.val); got != tt.want { + t.Errorf("StructToString() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/util/resource-recommend/log/logger.go b/pkg/util/resource-recommend/log/logger.go new file mode 100644 index 000000000..d09c5e5f5 --- /dev/null +++ b/pkg/util/resource-recommend/log/logger.go @@ -0,0 +1,107 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package log + +import ( + "context" + + "github.com/google/uuid" + "k8s.io/klog/v2" +) + +// This log is only used in resource-recommend. +// TODO: How to do the common log needs to be discussed again + +var ( + fieldsKey = &[1]int{1} +) + +const LinkIDKey = "linkId" + +// InitContext set context info such as accountId for logger +func InitContext(ctx context.Context) context.Context { + fields := []interface{}{ + LinkIDKey, uuid.New(), + } + return context.WithValue(ctx, fieldsKey, fields) +} + +// SetKeysAndValues set KeysAndValues into ctx +func SetKeysAndValues(ctx context.Context, keysAndValues ...interface{}) context.Context { + ctxFields := ctx.Value(fieldsKey) + if ctxFields == nil { + return context.WithValue(ctx, fieldsKey, keysAndValues) + } + if fields, ok := ctxFields.([]interface{}); !ok { + return context.WithValue(ctx, fieldsKey, keysAndValues) + } else { + fields = append(fields, keysAndValues...) + return context.WithValue(ctx, fieldsKey, fields) + } +} + +func GetCtxFields(ctx context.Context) []interface{} { + ctxFields := ctx.Value(fieldsKey) + if ctxFields == nil { + return []interface{}{} + } + if fields, ok := ctxFields.([]interface{}); ok { + return fields + } + return []interface{}{} +} + +// ErrorS print error log +func ErrorS(ctx context.Context, err error, msg string, keysAndValues ...interface{}) { + fields := GetCtxFields(ctx) + fields = append(fields, keysAndValues...) + if len(fields) == 0 { + klog.ErrorSDepth(1, err, msg) + } else { + klog.ErrorSDepth(1, err, msg, fields...) + } +} + +// InfoS print info log +func InfoS(ctx context.Context, msg string, keysAndValues ...interface{}) { + fields := GetCtxFields(ctx) + fields = append(fields, keysAndValues...) + if len(fields) == 0 { + klog.InfoSDepth(1, msg) + } else { + klog.InfoSDepth(1, msg, fields...) + } +} + +type Verbose struct { + klog.Verbose +} + +func V(level klog.Level) Verbose { + return Verbose{Verbose: klog.V(level)} +} + +// InfoS print info log +func (v Verbose) InfoS(ctx context.Context, msg string, keysAndValues ...interface{}) { + fields := GetCtxFields(ctx) + fields = append(fields, keysAndValues...) + if len(fields) == 0 { + v.Verbose.InfoSDepth(1, msg) + } else { + v.Verbose.InfoSDepth(1, msg, fields...) + } +} diff --git a/pkg/util/resource-recommend/log/logger_test.go b/pkg/util/resource-recommend/log/logger_test.go new file mode 100644 index 000000000..f22c3c224 --- /dev/null +++ b/pkg/util/resource-recommend/log/logger_test.go @@ -0,0 +1,343 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package log + +import ( + "context" + "flag" + "fmt" + "os" + "regexp" + "runtime" + "testing" + + "k8s.io/klog/v2" +) + +func TestInitContextAndGetCtxFields(t *testing.T) { + type args struct { + ctx context.Context + } + tests := []struct { + name string + args args + }{ + { + name: "case1", + args: args{ + ctx: context.Background(), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := InitContext(tt.args.ctx) + fields := GetCtxFields(ctx) + if len(fields) != 2 { + t.Errorf("The fields length must be 2") + } + if fields[0] != LinkIDKey { + t.Errorf("fields must be included: %s", LinkIDKey) + } + }) + } +} + +func TestSetKeysAndValues(t *testing.T) { + type tempStruct struct { + a int + b string + } + tempStruct1 := &tempStruct{111, "aaa"} + type args struct { + ctx context.Context + keysAndValues []interface{} + } + type want struct { + index int + wantValue interface{} + } + tests := []struct { + name string + args args + want []want + }{ + { + name: "case1", + args: args{ + ctx: InitContext(context.Background()), + keysAndValues: []interface{}{"k1", "v1", "k2", "v2"}, + }, + want: []want{ + { + index: 2, wantValue: "k1", + }, + { + index: 3, wantValue: "v1", + }, + { + index: 4, wantValue: "k2", + }, + { + index: 5, wantValue: "v2", + }, + }, + }, + { + name: "case2", + args: args{ + ctx: InitContext(context.Background()), + keysAndValues: []interface{}{"k1", tempStruct{111, "aaa"}}, + }, + want: []want{ + { + index: 2, wantValue: "k1", + }, + { + index: 3, wantValue: tempStruct{111, "aaa"}, + }, + }, + }, + { + name: "case3", + args: args{ + ctx: InitContext(context.Background()), + keysAndValues: []interface{}{"k1", tempStruct1}, + }, + want: []want{ + { + index: 2, wantValue: "k1", + }, + { + index: 3, wantValue: tempStruct1, + }, + }, + }, + { + name: "case4", + args: args{ + ctx: context.Background(), + keysAndValues: []interface{}{"k1", tempStruct1}, + }, + want: []want{ + { + index: 0, wantValue: "k1", + }, + { + index: 1, wantValue: tempStruct1, + }, + }, + }, + { + name: "case4", + args: args{ + ctx: InitContext(context.Background()), + keysAndValues: []interface{}{}, + }, + want: []want{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := SetKeysAndValues(tt.args.ctx, tt.args.keysAndValues...) + fields := GetCtxFields(ctx) + for _, w := range tt.want { + if fields[w.index] != w.wantValue { + t.Errorf("set KeysAndValues error, index(%d) should be %v, got: %v", w.index, w.wantValue, fields[w.index]) + } else { + t.Logf("set KeysAndValues successful, index(%d) should be %v, got: %v", w.index, w.wantValue, fields[w.index]) + } + } + }) + } +} + +// 自定义的测试日志记录器，实现 io.Writer 接口 +type TestLogger struct { + RegularxEpression string + t *testing.T +} + +func (tl *TestLogger) Write(p []byte) (n int, err error) { + str := string(p) + + regex, err := regexp.Compile(tl.RegularxEpression) + if err != nil { + fmt.Println("Error compiling regex:", err) + return + } + isMatch := regex.MatchString(str) + if !isMatch { + tl.t.Errorf("match the log content error, regex is %s, got: %s", tl.RegularxEpression, str) + } + + return len(p), nil +} + +func TestErrorS(t *testing.T) { + type args struct { + ctx context.Context + err error + msg string + keysAndValues []interface{} + } + tests := []struct { + name string + args args + wantLogStr string + }{ + { + name: "case1", + args: args{ + ctx: SetKeysAndValues(context.Background(), "baseK1", "baseV1"), + err: fmt.Errorf("err1"), + msg: "errMsg1", + keysAndValues: []interface{}{"k1", "v1", "k2", "v2"}, + }, + wantLogStr: "E\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6} %d logger_test.go:%d] \"errMsg1\" err=\"err1\" baseK1=\"baseV1\" k1=\"v1\" k2=\"v2\"\n", + }, + { + name: "case2", + args: args{ + ctx: context.Background(), + err: fmt.Errorf("err1"), + msg: "errMsg1", + keysAndValues: []interface{}{}, + }, + wantLogStr: "E\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6} %d logger_test.go:%d] \"errMsg1\" err=\"err1\"\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, _, line, ok := runtime.Caller(0) + if !ok { + panic("Unable to retrieve caller information") + } + pid := os.Getpid() + klog.LogToStderr(false) + klog.SetOutput(&TestLogger{ + RegularxEpression: fmt.Sprintf(tt.wantLogStr, pid, line+10), + t: t, + }) + ErrorS(tt.args.ctx, tt.args.err, tt.args.msg, tt.args.keysAndValues...) + }) + } +} + +func TestInfoS(t *testing.T) { + type args struct { + ctx context.Context + msg string + keysAndValues []interface{} + } + tests := []struct { + name string + args args + wantLogStr string + }{ + { + name: "case1", + args: args{ + ctx: SetKeysAndValues(context.Background(), "baseK1", "baseV1"), + msg: "InfoMsg1", + keysAndValues: []interface{}{"k1", "v1", "k2", "v2"}, + }, + wantLogStr: "I\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6} %d logger_test.go:%d] \"InfoMsg1\" baseK1=\"baseV1\" k1=\"v1\" k2=\"v2\"\n", + }, + { + name: "case2", + args: args{ + ctx: context.Background(), + msg: "InfoMsg1", + keysAndValues: []interface{}{}, + }, + wantLogStr: "I\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6} %d logger_test.go:%d] \"InfoMsg1\"\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, _, line, ok := runtime.Caller(0) + if !ok { + panic("Unable to retrieve caller information") + } + pid := os.Getpid() + klog.LogToStderr(false) + klog.SetOutput(&TestLogger{ + RegularxEpression: fmt.Sprintf(tt.wantLogStr, pid, line+10), + t: t, + }) + InfoS(tt.args.ctx, tt.args.msg, tt.args.keysAndValues...) + }) + } +} + +func TestVerbose_InfoS(t *testing.T) { + logLevel := klog.Level(1) + klog.InitFlags(nil) + flag.Set("v", logLevel.String()) + flag.Parse() + + type fields struct { + Verbose klog.Verbose + } + type args struct { + ctx context.Context + msg string + keysAndValues []interface{} + } + tests := []struct { + name string + fields fields + args args + wantLogStr string + }{ + { + name: "case1", + args: args{ + ctx: SetKeysAndValues(context.Background(), "baseK1", "baseV1"), + msg: "InfoMsg1", + keysAndValues: []interface{}{"k1", "v1", "k2", "v2"}, + }, + wantLogStr: "I\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6} %d logger_test.go:%d] \"InfoMsg1\" baseK1=\"baseV1\" k1=\"v1\" k2=\"v2\"\n", + }, + { + name: "case2", + args: args{ + ctx: context.Background(), + msg: "InfoMsg1", + keysAndValues: []interface{}{}, + }, + wantLogStr: "I\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6} %d logger_test.go:%d] \"InfoMsg1\"\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, _, line, ok := runtime.Caller(0) + if !ok { + panic("Unable to retrieve caller information") + } + pid := os.Getpid() + klog.LogToStderr(false) + klog.SetOutput(&TestLogger{ + RegularxEpression: fmt.Sprintf(tt.wantLogStr, pid, line+10), + t: t, + }) + V(logLevel).InfoS(tt.args.ctx, tt.args.msg, tt.args.keysAndValues...) + klog.Flush() + }) + } +} diff --git a/pkg/util/resource-recommend/resource/k8s_resource.go b/pkg/util/resource-recommend/resource/k8s_resource.go new file mode 100644 index 000000000..79fd5502b --- /dev/null +++ b/pkg/util/resource-recommend/resource/k8s_resource.go @@ -0,0 +1,108 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resource + +import ( + "context" + "encoding/json" + + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" +) + +func ConvertAndGetResource(ctx context.Context, client k8sclient.Client, namespace string, targetRef v1alpha1.CrossVersionObjectReference) (*unstructured.Unstructured, error) { + klog.V(5).Infof("Get resource in", "targetRef", targetRef, "namespace", namespace) + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(targetRef.APIVersion) + obj.SetKind(targetRef.Kind) + if err := client.Get(ctx, k8stypes.NamespacedName{Namespace: namespace, Name: targetRef.Name}, obj); err != nil { + return nil, err + } + return obj, nil +} + +func GetAllClaimedContainers(controller *unstructured.Unstructured) ([]string, error) { + klog.V(5).InfoS("Get all controller claimed containers", "controller", controller) + templateSpec, found, err := unstructured.NestedMap(controller.Object, "spec", "template", "spec") + if err != nil { + return nil, errors.Wrapf(err, "unstructured.NestedMap err") + } + if !found { + return nil, errors.Errorf("spec.template.spec not found in the controller") + } + containersList, found, err := unstructured.NestedSlice(templateSpec, "containers") + if err != nil { + return nil, errors.Wrapf(err, "unstructured.NestedSlice err") + } + if !found { + return nil, errors.Errorf("failure to find containers in the controller") + } + + containerNames := make([]string, 0, len(containersList)) + for _, container := range containersList { + containerMap, ok := container.(map[string]interface{}) + if !ok { + klog.Errorf("Unable to convert container:%v to map[string]interface{}", container) + continue + } + name, found, err := unstructured.NestedString(containerMap, "name") + if err != nil || !found { + klog.Errorf("Container name not found or get container name err:%v in the containerMap %v", err, containerMap) + continue + } + containerNames = append(containerNames, name) + } + return containerNames, nil +} + +type patchRecord struct { + Op string `json:"op,inline"` + Path string `json:"path,inline"` + Value interface{} `json:"value"` +} + +func PatchUpdateResourceRecommend(client k8sclient.Client, namespaceName k8stypes.NamespacedName, + resourceRecommend *v1alpha1.ResourceRecommend) error { + obj := &v1alpha1.ResourceRecommend{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName.Name, + Namespace: namespaceName.Namespace, + }, + } + patches := []patchRecord{{ + Op: "replace", + Path: "/status", + Value: resourceRecommend.Status, + }} + + patch, err := json.Marshal(patches) + if err != nil { + return errors.Wrapf(err, "failed to Marshal resourceRecommend: %+v", resourceRecommend) + } + patchDate := k8sclient.RawPatch(k8stypes.JSONPatchType, patch) + err = client.Status().Patch(context.TODO(), obj, patchDate) + if err != nil { + return errors.Wrapf(err, "failed to patch resource") + } + return nil +} diff --git a/pkg/util/resource-recommend/resource/k8s_resource_test.go b/pkg/util/resource-recommend/resource/k8s_resource_test.go new file mode 100644 index 000000000..2fb50b55b --- /dev/null +++ b/pkg/util/resource-recommend/resource/k8s_resource_test.go @@ -0,0 +1,441 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resource + +import ( + "context" + "reflect" + "testing" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" +) + +func TestConvertAndGetResource(t *testing.T) { + type args struct { + ctx context.Context + client client.Client + namespace string + targetRef v1alpha1.CrossVersionObjectReference + } + type env struct { + namespace string + kind string + name string + apiVersion string + } + tests := []struct { + name string + args args + env env + want *unstructured.Unstructured + wantErr bool + }{ + { + name: "Get resource failed", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "fakeNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "mockResourceName", + APIVersion: "apps/v1", + }, + }, + env: env{ + namespace: "fakeNamespace", + kind: "Deployment", + name: "mockResourceName1", + apiVersion: "apps/v1", + }, + wantErr: true, + }, + { + name: "Get resource failed", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "fakeNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "mockResourceName", + APIVersion: "apps/v1", + }, + }, + env: env{ + namespace: "fakeNamespace", + kind: "Deployment", + name: "mockResourceName", + apiVersion: "apps/v1", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + CreateMockUnstructured(nil, nil, tt.env.name, tt.env.namespace, tt.env.apiVersion, tt.env.kind, tt.args.client) + _, err := ConvertAndGetResource(tt.args.ctx, tt.args.client, tt.args.namespace, tt.args.targetRef) + if (err != nil) != tt.wantErr { + t.Errorf("ConvertAndGetResource() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestGetAllClaimedContainers(t *testing.T) { + type args struct { + controller *unstructured.Unstructured + } + type env struct { + needCreate bool + unstructuredTemplateSpec map[string]interface{} + } + tests := []struct { + name string + args args + env env + want []string + wantErr bool + }{ + { + name: "spec.template.spec not found", + args: args{ + controller: &unstructured.Unstructured{}, + }, + env: env{ + needCreate: false, + unstructuredTemplateSpec: map[string]interface{}{}, + }, + want: nil, + wantErr: true, + }, + { + name: "nestedMap failed", + args: args{ + controller: &unstructured.Unstructured{}, + }, + env: env{ + needCreate: true, + unstructuredTemplateSpec: map[string]interface{}{ + "container": []interface{}{ + map[string]interface{}{ + "name": "container-1", + "image": "image:latest", + }, + map[string]interface{}{ + "name": "container-2", + "image": "image:latest", + }, + }, + }, + }, + want: nil, + wantErr: true, + }, + { + name: "container name not found", + args: args{ + controller: &unstructured.Unstructured{}, + }, + env: env{ + needCreate: true, + unstructuredTemplateSpec: map[string]interface{}{ + "containers": []interface{}{ + map[string]interface{}{ + "image": "image:latest", + }, + map[string]interface{}{ + "image": "image:latest", + }, + }, + }, + }, + want: []string{}, + wantErr: false, + }, + { + name: "all right", + args: args{ + controller: &unstructured.Unstructured{}, + }, + env: env{ + needCreate: true, + unstructuredTemplateSpec: map[string]interface{}{ + "containers": []interface{}{ + map[string]interface{}{ + "name": "container-1", + "image": "image:latest", + }, + map[string]interface{}{ + "name": "container-2", + "image": "image:latest", + }, + }, + }, + }, + want: []string{"container-1", "container-2"}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.env.needCreate { + tt.args.controller.SetName("mockName") + unstructured.SetNestedMap(tt.args.controller.Object, tt.env.unstructuredTemplateSpec, "spec", "template", "spec") + } + got, err := GetAllClaimedContainers(tt.args.controller) + if (err != nil) != tt.wantErr { + t.Errorf("GetAllClaimedContainers() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetAllClaimedContainers() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPatchUpdateResourceRecommend(t *testing.T) { + type args struct { + client client.Client + namespaceName types.NamespacedName + resourceRecommend *v1alpha1.ResourceRecommend + } + tests := []struct { + name string + args args + wantErr bool + wantResourceRecommend *v1alpha1.ResourceRecommend + }{ + { + name: "all right 1", + args: args{ + client: fake.NewClientBuilder().Build(), + namespaceName: types.NamespacedName{ + Name: "mockName", + Namespace: "mockNamespace", + }, + resourceRecommend: &v1alpha1.ResourceRecommend{ + TypeMeta: metav1.TypeMeta{ + Kind: "ResourceRecommend", + APIVersion: "autoscaling.katalyst.kubewharf.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mockName", + Namespace: "mockNamespace", + }, + Status: v1alpha1.ResourceRecommendStatus{}, + }, + }, + wantResourceRecommend: &v1alpha1.ResourceRecommend{ + Status: v1alpha1.ResourceRecommendStatus{ + RecommendResources: &v1alpha1.RecommendResources{ + ContainerRecommendations: []v1alpha1.ContainerResources{ + { + ContainerName: "ContainerName1", + Requests: &v1alpha1.ContainerResourceList{}, + }, + { + ContainerName: "ContainerName2", + Requests: &v1alpha1.ContainerResourceList{}, + }, + { + ContainerName: "ContainerName3", + Requests: &v1alpha1.ContainerResourceList{}, + }, + { + ContainerName: "ContainerName4", + Requests: &v1alpha1.ContainerResourceList{}, + }, + }, + }, + Conditions: []v1alpha1.ResourceRecommendCondition{ + { + Type: v1alpha1.Validated, + Reason: "reason1", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason2", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason3", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason4", + Status: v1.ConditionTrue, + Message: "Message", + }, + }, + }, + }, + wantErr: false, + }, + { + name: "all right 2", + args: args{ + client: fake.NewClientBuilder().Build(), + namespaceName: types.NamespacedName{ + Name: "mockName", + Namespace: "mockNamespace", + }, + resourceRecommend: &v1alpha1.ResourceRecommend{ + TypeMeta: metav1.TypeMeta{ + Kind: "ResourceRecommend", + APIVersion: "autoscaling.katalyst.kubewharf.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mockName", + Namespace: "mockNamespace", + }, + Status: v1alpha1.ResourceRecommendStatus{ + RecommendResources: &v1alpha1.RecommendResources{ + ContainerRecommendations: []v1alpha1.ContainerResources{ + { + ContainerName: "ContainerName0", + }, + { + ContainerName: "ContainerName1", + }, + { + ContainerName: "ContainerName2", + }, + { + ContainerName: "ContainerName3", + }, + }, + }, + Conditions: []v1alpha1.ResourceRecommendCondition{ + { + Type: v1alpha1.Validated, + Reason: "reason0", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason1", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason2", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason3", + Status: v1.ConditionTrue, + Message: "Message", + }, + }, + }, + }, + }, + wantResourceRecommend: &v1alpha1.ResourceRecommend{ + Status: v1alpha1.ResourceRecommendStatus{ + RecommendResources: &v1alpha1.RecommendResources{ + ContainerRecommendations: []v1alpha1.ContainerResources{ + { + ContainerName: "ContainerName1", + Requests: &v1alpha1.ContainerResourceList{}, + }, + { + ContainerName: "ContainerName2", + Requests: &v1alpha1.ContainerResourceList{}, + }, + { + ContainerName: "ContainerName3", + Requests: &v1alpha1.ContainerResourceList{}, + }, + { + ContainerName: "ContainerName4", + Requests: &v1alpha1.ContainerResourceList{}, + }, + }, + }, + Conditions: []v1alpha1.ResourceRecommendCondition{ + { + Type: v1alpha1.Validated, + Reason: "reason1", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason2", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason3", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason4", + Status: v1.ConditionTrue, + Message: "Message", + }, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v1alpha1.AddToScheme(tt.args.client.Scheme()) + tt.args.client.Create(context.TODO(), tt.args.resourceRecommend) + if err := PatchUpdateResourceRecommend(tt.args.client, tt.args.namespaceName, tt.args.resourceRecommend); (err != nil) != tt.wantErr { + t.Errorf("PatchUpdateResourceRecommend() error = %v, wantErr %v", err, tt.wantErr) + } + if err := PatchUpdateResourceRecommend(tt.args.client, tt.args.namespaceName, tt.wantResourceRecommend); (err != nil) != tt.wantErr { + t.Errorf("PatchUpdateResourceRecommend() error = %v, wantErr %v", err, tt.wantErr) + } + gotResourceRecommend := &v1alpha1.ResourceRecommend{} + tt.args.client.Get(context.TODO(), client.ObjectKey{ + Name: tt.args.namespaceName.Name, + Namespace: tt.args.namespaceName.Namespace, + }, gotResourceRecommend) + if !reflect.DeepEqual(gotResourceRecommend.Status, tt.wantResourceRecommend.Status) { + t.Errorf("PatchUpdateResourceRecommend() gotResourceRecommend.Status = %v, wantResourceRecommend.Status = %v", + gotResourceRecommend.Status, tt.wantResourceRecommend.Status) + } + }) + } +} diff --git a/pkg/util/resource-recommend/resource/k8s_resource_test_util.go b/pkg/util/resource-recommend/resource/k8s_resource_test_util.go new file mode 100644 index 000000000..2b02a5bf1 --- /dev/null +++ b/pkg/util/resource-recommend/resource/k8s_resource_test_util.go @@ -0,0 +1,52 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resource + +import ( + "context" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func CreateMockPod(labels, annotations map[string]string, name, namespace, nodeName string, containers []v1.Container, client client.Client) { + pod := &v1.Pod{ + Spec: v1.PodSpec{ + NodeName: nodeName, + Containers: containers, + }, + } + pod.SetLabels(labels) + pod.SetAnnotations(annotations) + pod.SetName(name) + pod.SetNamespace(namespace) + + client.Create(context.TODO(), pod) +} + +func CreateMockUnstructured(matchLabels, unstructuredTemplateSpec map[string]interface{}, name, namespace, apiVersion, kind string, client client.Client) { + collectorObject := &unstructured.Unstructured{} + collectorObject.SetName(name) + collectorObject.SetNamespace(namespace) + collectorObject.SetKind(kind) + collectorObject.SetAPIVersion(apiVersion) + unstructured.SetNestedMap(collectorObject.Object, matchLabels, "spec", "selector", "matchLabels") + unstructured.SetNestedMap(collectorObject.Object, unstructuredTemplateSpec, "spec", "template", "spec") + client.Create(context.TODO(), collectorObject) + +} diff --git a/pkg/util/resource-recommend/types/conditions/conditions.go b/pkg/util/resource-recommend/types/conditions/conditions.go new file mode 100644 index 000000000..fe82244b0 --- /dev/null +++ b/pkg/util/resource-recommend/types/conditions/conditions.go @@ -0,0 +1,109 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conditions + +import ( + "sort" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + errortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" +) + +// ResourceRecommendConditionsMap is map from recommend condition type to condition. +type ResourceRecommendConditionsMap map[v1alpha1.ResourceRecommendConditionType]v1alpha1.ResourceRecommendCondition + +func NewResourceRecommendConditionsMap() *ResourceRecommendConditionsMap { + convertedConditionsMap := make(ResourceRecommendConditionsMap) + return &convertedConditionsMap +} + +func (conditionsMap *ResourceRecommendConditionsMap) Set(condition v1alpha1.ResourceRecommendCondition) { + oldCondition, alreadyPresent := (*conditionsMap)[condition.Type] + if alreadyPresent && oldCondition.Type == condition.Type && + oldCondition.Status == condition.Status && + oldCondition.Reason == condition.Reason && + oldCondition.Message == condition.Message { + return + } else { + condition.LastTransitionTime = metav1.Now() + } + (*conditionsMap)[condition.Type] = condition +} + +func (conditionsMap *ResourceRecommendConditionsMap) AsList() []v1alpha1.ResourceRecommendCondition { + conditions := make([]v1alpha1.ResourceRecommendCondition, 0, len(*conditionsMap)) + for _, condition := range *conditionsMap { + conditions = append(conditions, condition) + } + + // Sort conditions by type to avoid elements floating on the list + sort.Slice(conditions, func(i, j int) bool { + return conditions[i].Type < conditions[j].Type + }) + + return conditions +} + +func (conditionsMap *ResourceRecommendConditionsMap) ConditionActive(conditionType v1alpha1.ResourceRecommendConditionType) bool { + condition, found := (*conditionsMap)[conditionType] + return found && condition.Status == v1.ConditionTrue +} + +func ValidationSucceededCondition() *v1alpha1.ResourceRecommendCondition { + return &v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Validated, + Status: v1.ConditionTrue, + } +} + +func InitializationSucceededCondition() *v1alpha1.ResourceRecommendCondition { + return &v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Initialized, + Status: v1.ConditionTrue, + } +} + +func RecommendationReadyCondition() *v1alpha1.ResourceRecommendCondition { + return &v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.RecommendationProvided, + Status: v1.ConditionTrue, + } +} + +func ConvertCustomErrorToCondition(err errortypes.CustomError) *v1alpha1.ResourceRecommendCondition { + var conditionType v1alpha1.ResourceRecommendConditionType + switch err.Phase { + case errortypes.Validated: + conditionType = v1alpha1.Validated + case errortypes.ProcessRegister: + conditionType = v1alpha1.Initialized + case errortypes.RecommendationProvided: + conditionType = v1alpha1.RecommendationProvided + default: + conditionType = v1alpha1.ResourceRecommendConditionType(err.Phase) + } + + return &v1alpha1.ResourceRecommendCondition{ + Type: conditionType, + Status: v1.ConditionFalse, + Reason: string(err.Code), + Message: err.Message, + } +} diff --git a/pkg/util/resource-recommend/types/conditions/conditions_test.go b/pkg/util/resource-recommend/types/conditions/conditions_test.go new file mode 100644 index 000000000..89fff411a --- /dev/null +++ b/pkg/util/resource-recommend/types/conditions/conditions_test.go @@ -0,0 +1,442 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conditions + +import ( + "reflect" + "testing" + "time" + + "bou.ke/monkey" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + errortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" +) + +func TestResourceRecommendConditionsMap_Set(t *testing.T) { + case1ConditionsMap := NewResourceRecommendConditionsMap() + var fakeTime1 = time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + case1WantConditionsMap := ResourceRecommendConditionsMap{ + v1alpha1.Validated: v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Validated, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.NewTime(fakeTime1), + Reason: "reason1", + Message: "test msg1", + }, + } + + case2ConditionsMap := ResourceRecommendConditionsMap{ + v1alpha1.Validated: v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Validated, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC)), + Reason: "reason1", + Message: "test msg1", + }, + } + var fakeTime2 = time.Date(2023, 2, 2, 2, 0, 0, 0, time.UTC) + case2WantConditionsMap := ResourceRecommendConditionsMap{ + v1alpha1.Validated: v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Validated, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Date(2023, 2, 2, 2, 0, 0, 0, time.UTC)), + Reason: "reason2", + Message: "test msg2", + }, + } + + case3ConditionsMap := ResourceRecommendConditionsMap{ + v1alpha1.Validated: v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Validated, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Date(2023, 3, 3, 3, 0, 0, 0, time.UTC)), + Reason: "reason3", + Message: "test msg3", + }, + } + var fakeTime3 = time.Date(2023, 3, 3, 3, 0, 0, 0, time.UTC) + case3WantConditionsMap := ResourceRecommendConditionsMap{ + v1alpha1.Validated: v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Validated, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Date(2023, 3, 3, 3, 0, 0, 0, time.UTC)), + Reason: "reason3", + Message: "test msg3", + }, + } + + case4ConditionsMap := ResourceRecommendConditionsMap{ + v1alpha1.Validated: v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Validated, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Date(2023, 3, 3, 3, 0, 0, 0, time.UTC)), + }, + } + var fakeTime4 = time.Date(2023, 4, 4, 4, 0, 0, 0, time.UTC) + case4WantConditionsMap := ResourceRecommendConditionsMap{ + v1alpha1.Validated: { + Type: v1alpha1.Validated, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Date(2023, 3, 3, 3, 0, 0, 0, time.UTC)), + }, + v1alpha1.Initialized: { + Type: v1alpha1.Initialized, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.NewTime(fakeTime4), + Reason: "reason4", + Message: "test msg4", + }, + } + + type args struct { + condition v1alpha1.ResourceRecommendCondition + fakeTime time.Time + } + tests := []struct { + name string + conditionsMap *ResourceRecommendConditionsMap + args args + want *ResourceRecommendConditionsMap + }{ + { + name: "notExist", + conditionsMap: case1ConditionsMap, + args: args{ + condition: v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Validated, + Status: v1.ConditionFalse, + Reason: "reason1", + Message: "test msg1", + }, + fakeTime: fakeTime1, + }, + want: &case1WantConditionsMap, + }, + { + name: "update", + conditionsMap: &case2ConditionsMap, + args: args{ + condition: v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Validated, + Status: v1.ConditionFalse, + Reason: "reason2", + Message: "test msg2", + }, + fakeTime: fakeTime2, + }, + want: &case2WantConditionsMap, + }, + { + name: "same", + conditionsMap: &case3ConditionsMap, + args: args{ + condition: v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Validated, + Status: v1.ConditionFalse, + Reason: "reason3", + Message: "test msg3", + }, + fakeTime: fakeTime3, + }, + want: &case3WantConditionsMap, + }, + { + name: "add", + conditionsMap: &case4ConditionsMap, + args: args{ + condition: v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Initialized, + Status: v1.ConditionFalse, + Reason: "reason4", + Message: "test msg4", + }, + fakeTime: fakeTime4, + }, + want: &case4WantConditionsMap, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer monkey.UnpatchAll() + + monkey.Patch(time.Now, func() time.Time { return tt.args.fakeTime }) + + tt.conditionsMap.Set(tt.args.condition) + + if !reflect.DeepEqual(tt.conditionsMap, tt.want) { + t.Errorf("conditionsMap set failed, got: %v, want: %v", tt.conditionsMap, tt.want) + } + }) + } +} + +func TestResourceRecommendConditionsMap_AsList(t *testing.T) { + tests := []struct { + name string + conditionsMap ResourceRecommendConditionsMap + want []v1alpha1.ResourceRecommendCondition + }{ + { + name: "case", + conditionsMap: ResourceRecommendConditionsMap{ + v1alpha1.Validated: { + Type: v1alpha1.Validated, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Date(2023, 3, 3, 3, 0, 0, 0, time.UTC)), + }, + v1alpha1.Initialized: { + Type: v1alpha1.Initialized, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Date(2023, 4, 4, 4, 0, 0, 0, time.UTC)), + Reason: "reason4", + Message: "test msg4", + }, + }, + want: []v1alpha1.ResourceRecommendCondition{ + { + Type: v1alpha1.Initialized, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Date(2023, 4, 4, 4, 0, 0, 0, time.UTC)), + Reason: "reason4", + Message: "test msg4", + }, + { + Type: v1alpha1.Validated, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Date(2023, 3, 3, 3, 0, 0, 0, time.UTC)), + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.conditionsMap.AsList(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("AsList() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestResourceRecommendConditionsMap_ConditionActive(t *testing.T) { + conditionsMap := ResourceRecommendConditionsMap{ + v1alpha1.Validated: { + Type: v1alpha1.Validated, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Date(2023, 3, 3, 3, 0, 0, 0, time.UTC)), + }, + v1alpha1.Initialized: { + Type: v1alpha1.Initialized, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Date(2023, 4, 4, 4, 0, 0, 0, time.UTC)), + Reason: "reason4", + Message: "test msg4", + }, + } + type args struct { + conditionType v1alpha1.ResourceRecommendConditionType + } + tests := []struct { + name string + conditionsMap ResourceRecommendConditionsMap + args args + want bool + }{ + { + name: "notFound", + conditionsMap: conditionsMap, + args: args{ + conditionType: v1alpha1.RecommendationProvided, + }, + want: false, + }, + { + name: "true", + conditionsMap: conditionsMap, + args: args{ + conditionType: v1alpha1.Validated, + }, + want: true, + }, + { + name: "false", + conditionsMap: conditionsMap, + args: args{ + conditionType: v1alpha1.Initialized, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.conditionsMap.ConditionActive(tt.args.conditionType); got != tt.want { + t.Errorf("ConditionActive() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestValidationSucceededCondition(t *testing.T) { + tests := []struct { + name string + want *v1alpha1.ResourceRecommendCondition + }{ + { + name: "case", + want: &v1alpha1.ResourceRecommendCondition{ + Type: "Validated", + Status: "True", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ValidationSucceededCondition(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ValidationSucceededCondition() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestInitializationSucceededCondition(t *testing.T) { + tests := []struct { + name string + want *v1alpha1.ResourceRecommendCondition + }{ + { + name: "case", + want: &v1alpha1.ResourceRecommendCondition{ + Type: "Initialized", + Status: "True", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := InitializationSucceededCondition(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("InitializationSucceededCondition() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRecommendationReadyCondition(t *testing.T) { + tests := []struct { + name string + want *v1alpha1.ResourceRecommendCondition + }{ + { + name: "case", + want: &v1alpha1.ResourceRecommendCondition{ + Type: "RecommendationProvided", + Status: "True", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := RecommendationReadyCondition(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("RecommendationReadyCondition() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestConvertCustomErrorToCondition(t *testing.T) { + type args struct { + err errortypes.CustomError + } + tests := []struct { + name string + args args + want *v1alpha1.ResourceRecommendCondition + }{ + { + name: "Validated_err", + args: args{ + err: errortypes.CustomError{ + Phase: errortypes.Validated, + Code: "code1", + Message: "err_msg1", + }, + }, + want: &v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Validated, + Status: v1.ConditionFalse, + Reason: "code1", + Message: "err_msg1", + }, + }, + { + name: "ProcessRegister_err", + args: args{ + err: errortypes.CustomError{ + Phase: errortypes.ProcessRegister, + Code: "code2", + Message: "err_msg2", + }, + }, + want: &v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Initialized, + Status: v1.ConditionFalse, + Reason: "code2", + Message: "err_msg2", + }, + }, + { + name: "RecommendationProvided_err", + args: args{ + err: errortypes.CustomError{ + Phase: errortypes.RecommendationProvided, + Code: "code3", + Message: "err_msg3", + }, + }, + want: &v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.RecommendationProvided, + Status: v1.ConditionFalse, + Reason: "code3", + Message: "err_msg3", + }, + }, + { + name: "Unknown_err", + args: args{ + err: errortypes.CustomError{ + Phase: "testPhase", + Code: "code4", + Message: "err_msg4", + }, + }, + want: &v1alpha1.ResourceRecommendCondition{ + Type: "testPhase", + Status: v1.ConditionFalse, + Reason: "code4", + Message: "err_msg4", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ConvertCustomErrorToCondition(tt.args.err); !reflect.DeepEqual(got, tt.want) { + t.Errorf("ConvertCustomErrorToCondition() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/util/resource-recommend/types/datasource/operator.go b/pkg/util/resource-recommend/types/datasource/operator.go new file mode 100644 index 000000000..93e59e255 --- /dev/null +++ b/pkg/util/resource-recommend/types/datasource/operator.go @@ -0,0 +1,84 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package datasource + +import ( + "sort" +) + +type SamplesOverview struct { + AvgValue float64 + MinValue float64 + MaxValue float64 + Percentile50thValue float64 + Percentile90thValue float64 + LastTimestamp int64 + FirstTimestamp int64 + Count int +} + +func GetSamplesOverview(timeSeries *TimeSeries) *SamplesOverview { + if timeSeries == nil || len(timeSeries.Samples) == 0 { + return nil + } + samples := timeSeries.Samples + sort.Slice(samples, func(i, j int) bool { + return samples[i].Value < samples[j].Value + }) + aggregationSample := &SamplesOverview{ + Count: len(samples), + MinValue: samples[0].Value, + MaxValue: samples[len(samples)-1].Value, + Percentile50thValue: calculateSamplesPercentile(samples, 0.5), + Percentile90thValue: calculateSamplesPercentile(samples, 0.9), + } + + sumValue := 0.0 + for _, sample := range samples { + if aggregationSample.LastTimestamp < sample.Timestamp { + aggregationSample.LastTimestamp = sample.Timestamp + } + if aggregationSample.FirstTimestamp == 0 || aggregationSample.FirstTimestamp > sample.Timestamp { + aggregationSample.FirstTimestamp = sample.Timestamp + } + sumValue += sample.Value + } + aggregationSample.AvgValue = sumValue / float64(len(timeSeries.Samples)) + return aggregationSample +} + +func CalculateSamplesPercentile(samples []Sample, percentile float64) float64 { + sort.Slice(samples, func(i, j int) bool { + return samples[i].Value < samples[j].Value + }) + + return calculateSamplesPercentile(samples, percentile) +} + +func calculateSamplesPercentile(samples []Sample, percentile float64) float64 { + length := float64(len(samples)) + index := (length - 1) * percentile + + // 判断 index 是否为整数 + if index == float64(int(index)) { + return samples[int(index)].Value + } + + lower := samples[int(index)].Value + upper := samples[int(index)+1].Value + return (lower + upper) / 2 +} diff --git a/pkg/util/resource-recommend/types/datasource/operator_test.go b/pkg/util/resource-recommend/types/datasource/operator_test.go new file mode 100644 index 000000000..7d6a01cc7 --- /dev/null +++ b/pkg/util/resource-recommend/types/datasource/operator_test.go @@ -0,0 +1,70 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package datasource + +import ( + "reflect" + "testing" +) + +func TestGetSamplesOverview(t *testing.T) { + type args struct { + timeSeries *TimeSeries + } + tests := []struct { + name string + args args + want *SamplesOverview + }{ + { + name: "Empty time series", + args: args{ + timeSeries: &TimeSeries{Samples: []Sample{}}, + }, + want: nil, + }, + { + name: "Non-empty time series", + args: args{ + timeSeries: &TimeSeries{ + Samples: []Sample{ + {Value: 10, Timestamp: 1630838400}, + {Value: 20, Timestamp: 1630842000}, + {Value: 15, Timestamp: 1630845600}, + }, + }, + }, + want: &SamplesOverview{ + AvgValue: 15, + MinValue: 10, + MaxValue: 20, + Percentile50thValue: 15, + Percentile90thValue: 17.5, + FirstTimestamp: 1630838400, + LastTimestamp: 1630845600, + Count: 3, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetSamplesOverview(tt.args.timeSeries); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetSamplesOverview() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/util/resource-recommend/types/datasource/types.go b/pkg/util/resource-recommend/types/datasource/types.go new file mode 100644 index 000000000..f499cab9e --- /dev/null +++ b/pkg/util/resource-recommend/types/datasource/types.go @@ -0,0 +1,99 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package datasource + +import ( + "fmt" + "sort" + "strings" + + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" +) + +// Sample is a single timestamped value of the metric. +type Sample struct { + Value float64 + Timestamp int64 +} + +// TimeSeries represents a metric with given labels, with its values possibly changing in time. +type TimeSeries struct { + Labels map[string]string + Samples []Sample +} + +// WorkloadKind is k8s resource kind +type WorkloadKind string + +// Resource Name +const ( + // workload kind is deployment + WorkloadDeployment WorkloadKind = "Deployment" +) + +type Metric struct { + //to be extended when new datasource is added + Namespace string + Kind string + APIVersion string + WorkloadName string + PodName string + ContainerName string + Resource v1.ResourceName + Selectors string +} + +func (m *Metric) SetSelector(selectors map[string]string) { + keys := make([]string, 0, len(selectors)) + for key := range selectors { + keys = append(keys, key) + } + sort.Strings(keys) + + var formattedParams []string + for _, key := range keys { + value := selectors[key] + formattedParam := fmt.Sprintf("%s=\"%s\"", key, value) + formattedParams = append(formattedParams, formattedParam) + } + m.Selectors = strings.Join(formattedParams, ",") + klog.V(4).InfoS("Setting selectors", "selectors", m.Selectors) +} + +type Query struct { + //to be extended when new datasource is added + Prometheus *PrometheusQuery +} +type PrometheusQuery struct { + Query string +} + +func NewTimeSeries() *TimeSeries { + return &TimeSeries{ + Labels: make(map[string]string), + Samples: make([]Sample, 0), + } +} + +func (ts *TimeSeries) AppendLabel(key, val string) { + ts.Labels[key] = val +} + +func (ts *TimeSeries) AppendSample(timestamp int64, val float64) { + ts.Samples = append(ts.Samples, Sample{Timestamp: timestamp, Value: val}) +} diff --git a/pkg/util/resource-recommend/types/error/errors.go b/pkg/util/resource-recommend/types/error/errors.go new file mode 100644 index 000000000..bf8ecd8c1 --- /dev/null +++ b/pkg/util/resource-recommend/types/error/errors.go @@ -0,0 +1,34 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package error + +type Phase string + +type Code string + +type CustomError struct { + // Phase in which error occurs + Phase Phase + // Code of the current error + Code Code + // Message is a human-readable explanation containing details about the error + Message string +} + +func (err *CustomError) Error() string { + return err.Message +} diff --git a/pkg/util/resource-recommend/types/error/process.go b/pkg/util/resource-recommend/types/error/process.go new file mode 100644 index 000000000..80f1a0dd1 --- /dev/null +++ b/pkg/util/resource-recommend/types/error/process.go @@ -0,0 +1,93 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package error + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/types" +) + +const ( + ProcessRegister Phase = "ProcessRegister" + ProcessCancel Phase = "ProcessCancel" +) + +const ( + DataProcessRegisterFailed Code = "DataProcessRegisterFailed" + ValidateProcessConfigFailed Code = "ValidateProcessConfigFailed" + RegisterTaskPanic Code = "RegisterTaskPanic" + CancelTaskPanic Code = "CancelTaskPanic" + NewProcessTaskFailed Code = "NewProcessTaskFailed" + NotFoundTasks Code = "NotFoundTask" +) + +const ( + RegisterProcessTaskPanicMessage = "register data process task panic" + CancelProcessTaskPanicMessage = "cancel data process task panic" + ProcessTaskValidateErrorMessageTemplate = "register data process task failed, get error when validate, err: %s" + NewProcessTaskErrorMessageTemplate = "register data process task failed, fail to get new task, err: %s" + TasksNotFoundErrorMessage = "process tasks not found " +) + +func DataProcessRegisteredFailedError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: ProcessRegister, + Code: DataProcessRegisterFailed, + Message: fmt.Sprintf(msg, arg...), + } +} + +func RegisterProcessTaskValidateError(err error) *CustomError { + return &CustomError{ + Phase: ProcessRegister, + Code: ValidateProcessConfigFailed, + Message: fmt.Sprintf(ProcessTaskValidateErrorMessageTemplate, err), + } +} + +func RegisterProcessTaskPanic() *CustomError { + return &CustomError{ + Phase: ProcessRegister, + Code: RegisterTaskPanic, + Message: RegisterProcessTaskPanicMessage, + } +} + +func NewProcessTaskError(err error) *CustomError { + return &CustomError{ + Phase: ProcessRegister, + Code: NewProcessTaskFailed, + Message: fmt.Sprintf(NewProcessTaskErrorMessageTemplate, err), + } +} + +func CancelProcessTaskPanic() *CustomError { + return &CustomError{ + Phase: ProcessCancel, + Code: CancelTaskPanic, + Message: CancelProcessTaskPanicMessage, + } +} + +func NotFoundTasksError(namespacedName types.NamespacedName) *CustomError { + return &CustomError{ + Phase: ProcessCancel, + Code: NotFoundTasks, + Message: fmt.Sprintf(TasksNotFoundErrorMessage+"for ResourceRecommend(%s)", namespacedName), + } +} diff --git a/pkg/util/resource-recommend/types/error/recommend.go b/pkg/util/resource-recommend/types/error/recommend.go new file mode 100644 index 000000000..8b02a8fba --- /dev/null +++ b/pkg/util/resource-recommend/types/error/recommend.go @@ -0,0 +1,35 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package error + +import "fmt" + +const ( + RecommendationProvided Phase = "RecommendationProvided" +) + +const ( + RecommendationNotReady Code = "RecommendationNotReady" +) + +func RecommendationNotReadyError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: RecommendationProvided, + Code: RecommendationNotReady, + Message: fmt.Sprintf(msg, arg...), + } +} diff --git a/pkg/util/resource-recommend/types/error/validate.go b/pkg/util/resource-recommend/types/error/validate.go new file mode 100644 index 000000000..f6b39009f --- /dev/null +++ b/pkg/util/resource-recommend/types/error/validate.go @@ -0,0 +1,169 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package error + +import "fmt" + +const ( + Validated Phase = "Validated" +) + +const ( + WorkloadNameIsEmpty Code = "WorkloadNameIsEmpty" + WorkloadsUnsupported Code = "WorkloadsUnsupported" + AlgorithmUnsupported Code = "AlgorithmUnsupported" + WorkloadMatchError Code = "WorkloadMatchError" + WorkloadNotFound Code = "WorkloadNotFound" + ContainerPoliciesNotFound Code = "ContainerPoliciesNotFound" + ContainersNotFound Code = "ContainersNotFound" + ContainersMatchErrorCode Code = "ContainersMatchErrorCode" + ContainerDuplicate Code = "ContainerDuplicate" + ContainerNameEmpty Code = "ContainerNameEmpty" + ControlledResourcesPoliciesEmpty Code = "ControlledResourcesPoliciesEmpty" + ResourceBuffersUnsupported Code = "ResourceBuffersUnsupported" + ResourceNameUnsupported Code = "ResourceNameUnsupported" + ControlledValuesUnsupported Code = "ControlledValuesUnsupported" +) + +const ( + WorkloadNameIsEmptyMessage = "spec.targetRef.name cannot be empty" + WorkloadsUnsupportedMessage = "spec.targetRef.kind %s is currently unsupported, only support in %s" + AlgorithmUnsupportedMessage = "spec.resourcePolicy.algorithmPolicy.algorithm %s is currently unsupported, only support in %s" + WorkloadNotFoundMessage = "workload not found" + WorkloadMatchedErrorMessage = "workload matched err" + ContainerPoliciesNotFoundMessage = "spec.containerPolicies cannot be empty" + ContainersMatchedErrorMessage = "containers matched err" + ContainerDuplicateMessage = "container name %s is duplicate" + ContainerNameEmptyMessage = "empty container name" + ControlledResourcesPoliciesEmptyMessage = "container(%s) Resources Policies is empty" + ContainerNotFoundMessage = "container %s is not found" + ResourceNameUnsupportedMessage = "unsupported ResourceName, current supported values: %s" + ControlledValuesUnsupportedMessage = "unsupported ControlledValues, current supported values: %s" + ResourceBuffersUnsupportedMessage = "resource buffers should be in the range from 0 to 100" +) + +func WorkloadNameIsEmptyError() *CustomError { + return &CustomError{ + Phase: Validated, + Code: WorkloadNameIsEmpty, + Message: WorkloadNameIsEmptyMessage, + } +} + +func WorkloadsUnsupportedError(kind string, supportedKinds []string) *CustomError { + return &CustomError{ + Phase: Validated, + Code: WorkloadsUnsupported, + Message: fmt.Sprintf(WorkloadsUnsupportedMessage, kind, supportedKinds), + } +} + +func AlgorithmUnsupportedError(algorithm string, supportedAlgorithm []string) *CustomError { + return &CustomError{ + Phase: Validated, + Code: AlgorithmUnsupported, + Message: fmt.Sprintf(AlgorithmUnsupportedMessage, algorithm, supportedAlgorithm), + } +} + +func ResourceNameUnsupportedError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ResourceNameUnsupported, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ControlledValuesUnsupportedError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ControlledValuesUnsupported, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ResourceBuffersUnsupportedError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ResourceBuffersUnsupported, + Message: fmt.Sprintf(msg, arg...), + } +} + +func WorkloadMatchedError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: WorkloadMatchError, + Message: fmt.Sprintf(msg, arg...), + } +} + +func WorkloadNotFoundError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: WorkloadNotFound, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ContainerPoliciesNotFoundError() *CustomError { + return &CustomError{ + Phase: Validated, + Code: ContainerPoliciesNotFound, + Message: ContainerPoliciesNotFoundMessage, + } +} + +func ContainersMatchedError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ContainersMatchErrorCode, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ContainerNameEmptyError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ContainerNameEmpty, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ControlledResourcesPoliciesEmptyError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ControlledResourcesPoliciesEmpty, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ContainersNotFoundError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ContainersNotFound, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ContainerDuplicateError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ContainerDuplicate, + Message: fmt.Sprintf(msg, arg...), + } +} diff --git a/pkg/util/resource-recommend/types/processor/task_key.go b/pkg/util/resource-recommend/types/processor/task_key.go new file mode 100644 index 000000000..c375d3da7 --- /dev/null +++ b/pkg/util/resource-recommend/types/processor/task_key.go @@ -0,0 +1,93 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package processor + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + datasourcetypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" +) + +type TaskConfigStr string + +type TaskID string + +type ProcessKey struct { + ResourceRecommendNamespacedName types.NamespacedName + *datasourcetypes.Metric +} + +type ProcessConfig struct { + ProcessKey + Config TaskConfigStr +} + +func NewProcessConfig(NamespacedName types.NamespacedName, targetRef v1alpha1.CrossVersionObjectReference, containerName string, controlledResource v1.ResourceName, taskConfig TaskConfigStr) *ProcessConfig { + return &ProcessConfig{ + ProcessKey: GetProcessKey(NamespacedName, targetRef, containerName, controlledResource), + Config: taskConfig, + } +} + +func GetProcessKey(NamespacedName types.NamespacedName, targetRef v1alpha1.CrossVersionObjectReference, containerName string, controlledResource v1.ResourceName) ProcessKey { + return ProcessKey{ + ResourceRecommendNamespacedName: NamespacedName, + Metric: &datasourcetypes.Metric{ + Namespace: NamespacedName.Namespace, + Kind: targetRef.Kind, + APIVersion: targetRef.APIVersion, + WorkloadName: targetRef.Name, + ContainerName: containerName, + Resource: controlledResource, + }, + } +} + +func (pc *ProcessConfig) Validate() error { + if pc.Metric == nil { + return fmt.Errorf("metric is empty") + } + if pc.ContainerName == "" { + return fmt.Errorf("containerName is empty") + } + + if pc.Kind == "" { + return fmt.Errorf("kind is empty") + } + + if pc.WorkloadName == "" { + return fmt.Errorf("workloadName is empty") + } + if !(pc.Resource == v1.ResourceCPU || pc.Resource == v1.ResourceMemory) { + return fmt.Errorf("controlledResource only can be [%s, %s]", v1.ResourceCPU, v1.ResourceMemory) + } + return nil +} + +func (pc *ProcessConfig) GenerateTaskID() TaskID { + return TaskID(pc.ResourceRecommendNamespacedName.String() + "-" + + pc.Kind + "-" + + pc.APIVersion + "-" + + pc.WorkloadName + "-" + + pc.ContainerName + "-" + + string(pc.Resource) + "-" + + string(pc.Config)) +} diff --git a/pkg/util/resource-recommend/types/processor/task_key_test.go b/pkg/util/resource-recommend/types/processor/task_key_test.go new file mode 100644 index 000000000..4af2f45e9 --- /dev/null +++ b/pkg/util/resource-recommend/types/processor/task_key_test.go @@ -0,0 +1,137 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package processor + +import ( + "fmt" + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + datasourcetype "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/datasource" +) + +func TestProcessConfig_GenerateTaskID(t *testing.T) { + type fields struct { + namespacedName types.NamespacedName + targetRef v1alpha1.CrossVersionObjectReference + containerName string + controlledResource v1.ResourceName + taskConfig TaskConfigStr + } + tests := []struct { + name string + fields fields + want TaskID + }{ + { + name: "case1", + fields: fields{ + namespacedName: types.NamespacedName{ + Name: "recommendation1", + Namespace: "default", + }, + targetRef: v1alpha1.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "demo", + APIVersion: "app/v1", + }, + containerName: "c1", + controlledResource: "cpu", + taskConfig: "", + }, + want: "default/recommendation1-Deployment-app/v1-demo-c1-cpu-", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pc := NewProcessConfig(tt.fields.namespacedName, tt.fields.targetRef, tt.fields.containerName, tt.fields.controlledResource, tt.fields.taskConfig) + if got := pc.GenerateTaskID(); got != tt.want { + t.Errorf("Validate() error, got: %s, want: %s", got, tt.want) + } + }) + } +} + +func TestProcessConfig_Validate(t *testing.T) { + tests := []struct { + name string + processConfig ProcessConfig + wantErr error + }{ + { + name: "metric is empty", + processConfig: ProcessConfig{}, + wantErr: fmt.Errorf("metric is empty"), + }, + { + name: "containerName is empty", + processConfig: ProcessConfig{ + ProcessKey: ProcessKey{ + Metric: &datasourcetype.Metric{}, + }, + }, + wantErr: fmt.Errorf("containerName is empty"), + }, + { + name: "kind is empty", + processConfig: ProcessConfig{ + ProcessKey: ProcessKey{ + Metric: &datasourcetype.Metric{ + ContainerName: "c1", + }, + }, + }, + wantErr: fmt.Errorf("kind is empty"), + }, + { + name: "workloadName is empty", + processConfig: ProcessConfig{ + ProcessKey: ProcessKey{ + Metric: &datasourcetype.Metric{ + ContainerName: "c1", + Kind: "Deployment", + }, + }, + }, + wantErr: fmt.Errorf("workloadName is empty"), + }, + { + name: "controlledResource not support", + processConfig: ProcessConfig{ + ProcessKey: ProcessKey{ + Metric: &datasourcetype.Metric{ + ContainerName: "c1", + Kind: "Deployment", + WorkloadName: "w1", + Resource: "Disk", + }, + }, + }, + wantErr: fmt.Errorf("controlledResource only can be [%s, %s]", v1.ResourceCPU, v1.ResourceMemory), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.processConfig.Validate(); err.Error() != tt.wantErr.Error() { + t.Errorf("Validate() error, gotErr: %v, wantErr: %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/util/resource-recommend/types/recommendation/recommendation.go b/pkg/util/resource-recommend/types/recommendation/recommendation.go new file mode 100644 index 000000000..9f6f8414c --- /dev/null +++ b/pkg/util/resource-recommend/types/recommendation/recommendation.go @@ -0,0 +1,144 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommendation + +import ( + "context" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + conditionstypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/conditions" + errortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" +) + +const ( + // TargetRefKindDeployment is Deployment + TargetRefKindDeployment string = "Deployment" +) + +var TargetRefKinds = []string{TargetRefKindDeployment} + +const ( + DefaultRecommenderType = "default" +) + +const ( + PercentileAlgorithmType = "percentile" + //DefaultAlgorithmType use percentile as the default algorithm + DefaultAlgorithmType = PercentileAlgorithmType +) + +var AlgorithmTypes = []string{PercentileAlgorithmType} + +var ResourceNames = []v1.ResourceName{v1.ResourceCPU, v1.ResourceMemory} + +var SupportControlledValues = []v1alpha1.ContainerControlledValues{v1alpha1.ContainerControlledValuesRequestsOnly} + +const ( + DefaultUsageBuffer = 30 + MaxUsageBuffer = 100 + MinUsageBuffer = 0 +) + +const ( + ContainerPolicySelectAllFlag = "*" +) + +type Recommendation struct { + types.NamespacedName + ObservedGeneration int64 + Config + // Map of the status conditions (keys are condition types). + Conditions *conditionstypes.ResourceRecommendConditionsMap + Recommendations []v1alpha1.ContainerResources +} + +type Config struct { + TargetRef v1alpha1.CrossVersionObjectReference + Containers []Container + AlgorithmPolicy v1alpha1.AlgorithmPolicy +} + +type Container struct { + ContainerName string + ContainerConfigs []ContainerConfig +} + +type ContainerConfig struct { + ControlledResource v1.ResourceName + ResourceBufferPercent int32 +} + +func NewRecommendation(resourceRecommend *v1alpha1.ResourceRecommend) *Recommendation { + return &Recommendation{ + NamespacedName: types.NamespacedName{Name: resourceRecommend.Name, Namespace: resourceRecommend.Namespace}, + ObservedGeneration: resourceRecommend.Generation, + Conditions: conditionstypes.NewResourceRecommendConditionsMap(), + } +} + +func (r *Recommendation) SetConfig(ctx context.Context, client k8sclient.Client, + resourceRecommend *v1alpha1.ResourceRecommend) *errortypes.CustomError { + + targetRef, customErr := ValidateAndExtractTargetRef(resourceRecommend.Spec.TargetRef) + if customErr != nil { + klog.Errorf("spec.targetRef validate error, "+ + "reason: %s, msg: %s", customErr.Code, customErr.Message) + return customErr + } + + algorithmPolicy, customErr := ValidateAndExtractAlgorithmPolicy(resourceRecommend.Spec.ResourcePolicy.AlgorithmPolicy) + if customErr != nil { + klog.Errorf("spec.resourcePolicy.algorithmPolicy validate error,"+ + " reason: %s, msg: %s", customErr.Code, customErr.Message) + return customErr + } + + containers, customErr := ValidateAndExtractContainers(ctx, client, resourceRecommend.Namespace, targetRef, resourceRecommend.Spec.ResourcePolicy.ContainerPolicies) + if customErr != nil { + klog.Errorf("spec.resourcePolicy.containerPolicies validate error, "+ + "reason: %s, msg: %s", customErr.Code, customErr.Message) + return customErr + } + + r.Config = Config{ + TargetRef: targetRef, + AlgorithmPolicy: algorithmPolicy, + Containers: containers, + } + return nil +} + +// AsStatus returns this objects equivalent of VPA Status. +func (r *Recommendation) AsStatus() v1alpha1.ResourceRecommendStatus { + status := v1alpha1.ResourceRecommendStatus{ + Conditions: r.Conditions.AsList(), + } + if r.Recommendations != nil { + now := metav1.Now() + status.LastRecommendationTime = &now + status.RecommendResources = &v1alpha1.RecommendResources{ + ContainerRecommendations: r.Recommendations, + } + } + return status +} diff --git a/pkg/util/resource-recommend/types/recommendation/recommendation_test.go b/pkg/util/resource-recommend/types/recommendation/recommendation_test.go new file mode 100644 index 000000000..f414dfa65 --- /dev/null +++ b/pkg/util/resource-recommend/types/recommendation/recommendation_test.go @@ -0,0 +1,254 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommendation + +import ( + "context" + "reflect" + "testing" + "time" + + "bou.ke/monkey" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + conditionstypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/conditions" + errortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" +) + +func TestRecommendation_AsStatus(t *testing.T) { + var fakeTime1 = time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + var fakeMetaTime1 = metav1.NewTime(fakeTime1) + tests := []struct { + name string + recommendation *Recommendation + want v1alpha1.ResourceRecommendStatus + }{ + { + name: "notRecommend", + recommendation: &Recommendation{ + Conditions: &conditionstypes.ResourceRecommendConditionsMap{ + v1alpha1.Validated: { + Type: v1alpha1.Validated, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Date(2023, 3, 3, 3, 0, 0, 0, time.UTC)), + }, + v1alpha1.Initialized: { + Type: v1alpha1.Initialized, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Date(2023, 4, 4, 4, 0, 0, 0, time.UTC)), + Reason: "reason4", + Message: "test msg4", + }, + }, + }, + want: v1alpha1.ResourceRecommendStatus{ + Conditions: []v1alpha1.ResourceRecommendCondition{ + { + Type: v1alpha1.Initialized, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Date(2023, 4, 4, 4, 0, 0, 0, time.UTC)), + Reason: "reason4", + Message: "test msg4", + }, + { + Type: v1alpha1.Validated, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Date(2023, 3, 3, 3, 0, 0, 0, time.UTC)), + }, + }, + }, + }, + { + name: "recommended", + recommendation: &Recommendation{ + Conditions: &conditionstypes.ResourceRecommendConditionsMap{ + v1alpha1.Validated: { + Type: v1alpha1.Validated, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Date(2023, 3, 3, 3, 0, 0, 0, time.UTC)), + }, + v1alpha1.Initialized: { + Type: v1alpha1.Initialized, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Date(2023, 4, 4, 4, 0, 0, 0, time.UTC)), + Reason: "reason4", + Message: "test msg4", + }, + }, + Recommendations: []v1alpha1.ContainerResources{ + { + ContainerName: "c1", + }, + }, + }, + want: v1alpha1.ResourceRecommendStatus{ + Conditions: []v1alpha1.ResourceRecommendCondition{ + { + Type: v1alpha1.Initialized, + Status: v1.ConditionFalse, + LastTransitionTime: metav1.NewTime(time.Date(2023, 4, 4, 4, 0, 0, 0, time.UTC)), + Reason: "reason4", + Message: "test msg4", + }, + { + Type: v1alpha1.Validated, + Status: v1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Date(2023, 3, 3, 3, 0, 0, 0, time.UTC)), + }, + }, + LastRecommendationTime: &fakeMetaTime1, + RecommendResources: &v1alpha1.RecommendResources{ + ContainerRecommendations: []v1alpha1.ContainerResources{ + { + ContainerName: "c1", + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + + defer monkey.UnpatchAll() + + monkey.Patch(time.Now, func() time.Time { return fakeTime1 }) + + if got := tt.recommendation.AsStatus(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("AsStatus() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestRecommendation_SetConfig(t *testing.T) { + type args struct { + targetRef v1alpha1.CrossVersionObjectReference + customErr1 *errortypes.CustomError + algorithmPolicy v1alpha1.AlgorithmPolicy + customErr2 *errortypes.CustomError + containers []Container + customErr3 *errortypes.CustomError + } + tests := []struct { + name string + args args + wantErr *errortypes.CustomError + }{ + { + name: "targetRef_Validate_err", + args: args{ + customErr1: &errortypes.CustomError{ + Phase: errortypes.Validated, + Code: errortypes.WorkloadNameIsEmpty, + Message: "err_msg1", + }, + }, + wantErr: &errortypes.CustomError{ + Phase: errortypes.Validated, + Code: errortypes.WorkloadNameIsEmpty, + Message: "err_msg1", + }, + }, + { + name: "targetRef_Validate_err", + args: args{ + customErr2: &errortypes.CustomError{ + Phase: errortypes.Validated, + Code: errortypes.AlgorithmUnsupported, + Message: "err_msg1", + }, + }, + wantErr: &errortypes.CustomError{ + Phase: errortypes.Validated, + Code: errortypes.AlgorithmUnsupported, + Message: "err_msg1", + }, + }, + { + name: "targetRef_Validate_err", + args: args{ + customErr3: &errortypes.CustomError{ + Phase: errortypes.Validated, + Code: errortypes.WorkloadNotFound, + Message: "err_msg1", + }, + }, + wantErr: &errortypes.CustomError{ + Phase: errortypes.Validated, + Code: errortypes.WorkloadNotFound, + Message: "err_msg1", + }, + }, + { + name: "paas", + args: args{ + targetRef: v1alpha1.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "demo", + }, + algorithmPolicy: v1alpha1.AlgorithmPolicy{ + Recommender: "default", + }, + containers: []Container{ + { + ContainerName: "c1", + }, + }, + }, + wantErr: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer monkey.UnpatchAll() + + monkey.Patch(ValidateAndExtractTargetRef, func(targetRefReq v1alpha1.CrossVersionObjectReference) ( + v1alpha1.CrossVersionObjectReference, *errortypes.CustomError) { + return tt.args.targetRef, tt.args.customErr1 + }) + monkey.Patch(ValidateAndExtractAlgorithmPolicy, func(algorithmPolicyReq v1alpha1.AlgorithmPolicy) ( + v1alpha1.AlgorithmPolicy, *errortypes.CustomError) { + return tt.args.algorithmPolicy, tt.args.customErr2 + }) + monkey.Patch(ValidateAndExtractContainers, func(ctx context.Context, client k8sclient.Client, namespace string, + targetRef v1alpha1.CrossVersionObjectReference, + containerPolicies []v1alpha1.ContainerResourcePolicy) ([]Container, *errortypes.CustomError) { + return tt.args.containers, tt.args.customErr3 + }) + + r := NewRecommendation(&v1alpha1.ResourceRecommend{}) + if gotErr := r.SetConfig(context.Background(), fake.NewClientBuilder().Build(), &v1alpha1.ResourceRecommend{}); !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("SetConfig() = %v, want %v", gotErr, tt.wantErr) + } + if tt.wantErr == nil { + config := Config{ + TargetRef: tt.args.targetRef, + AlgorithmPolicy: tt.args.algorithmPolicy, + Containers: tt.args.containers, + } + if !reflect.DeepEqual(config, r.Config) { + t.Errorf("SetConfig() failed, want config: %v, got: %v", config, r.Config) + } + } + }) + } +} diff --git a/pkg/util/resource-recommend/types/recommendation/validate.go b/pkg/util/resource-recommend/types/recommendation/validate.go new file mode 100644 index 000000000..ed4fcbd77 --- /dev/null +++ b/pkg/util/resource-recommend/types/recommendation/validate.go @@ -0,0 +1,157 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommendation + +import ( + "context" + + "k8s.io/klog/v2" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/util/general" + resourceutils "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/resource" + errortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" +) + +func ValidateAndExtractTargetRef(targetRefReq v1alpha1.CrossVersionObjectReference) ( + v1alpha1.CrossVersionObjectReference, *errortypes.CustomError) { + convertedTargetRef := v1alpha1.CrossVersionObjectReference{} + if targetRefReq.Name == "" { + return convertedTargetRef, errortypes.WorkloadNameIsEmptyError() + } + if ok := general.SliceContains(TargetRefKinds, targetRefReq.Kind); !ok { + return convertedTargetRef, errortypes.WorkloadsUnsupportedError(targetRefReq.Kind, TargetRefKinds) + } + convertedTargetRef.Kind = targetRefReq.Kind + convertedTargetRef.Name = targetRefReq.Name + convertedTargetRef.APIVersion = targetRefReq.APIVersion + return convertedTargetRef, nil +} + +func ValidateAndExtractAlgorithmPolicy(algorithmPolicyReq v1alpha1.AlgorithmPolicy) ( + v1alpha1.AlgorithmPolicy, *errortypes.CustomError) { + algorithmPolicy := v1alpha1.AlgorithmPolicy{ + Recommender: DefaultRecommenderType, + } + + if algorithmPolicyReq.Algorithm == "" { + algorithmPolicy.Algorithm = DefaultAlgorithmType + } else { + if ok := general.SliceContains(AlgorithmTypes, algorithmPolicyReq.Algorithm); !ok { + return algorithmPolicy, errortypes.AlgorithmUnsupportedError(string(algorithmPolicyReq.Algorithm), AlgorithmTypes) + } + algorithmPolicy.Algorithm = algorithmPolicyReq.Algorithm + } + + algorithmPolicy.Extensions = algorithmPolicyReq.Extensions + return algorithmPolicy, nil +} + +func ValidateAndExtractContainers(ctx context.Context, client k8sclient.Client, namespace string, + targetRef v1alpha1.CrossVersionObjectReference, + containerPolicies []v1alpha1.ContainerResourcePolicy) ( + []Container, *errortypes.CustomError) { + if len(containerPolicies) == 0 { + return nil, errortypes.ContainerPoliciesNotFoundError() + } + + resource, err := resourceutils.ConvertAndGetResource(ctx, client, namespace, targetRef) + if err != nil { + klog.ErrorS(err, "ConvertAndGetResource err") + if k8sclient.IgnoreNotFound(err) == nil { + return nil, errortypes.WorkloadNotFoundError(errortypes.WorkloadNotFoundMessage) + } + return nil, errortypes.WorkloadMatchedError(errortypes.WorkloadMatchedErrorMessage) + } + + existContainerList, err := resourceutils.GetAllClaimedContainers(resource) + if err != nil { + klog.ErrorS(err, "get all claimed containers err") + return nil, errortypes.ContainersMatchedError(errortypes.ContainersMatchedErrorMessage) + } + + containers, validateErr := validateAndExtractContainers(containerPolicies, existContainerList) + if validateErr != nil { + return nil, validateErr + } + + return containers, nil +} + +func validateAndExtractContainers(containerPolicies []v1alpha1.ContainerResourcePolicy, + existContainerList []string) ([]Container, *errortypes.CustomError) { + resourcePoliciesMap := make(map[string]v1alpha1.ContainerResourcePolicy) + + for _, resourcePolicy := range containerPolicies { + containerName := resourcePolicy.ContainerName + if _, ok := resourcePoliciesMap[containerName]; ok { + return nil, errortypes.ContainerDuplicateError(errortypes.ContainerDuplicateMessage, containerName) + } else if containerName == "" { + return nil, errortypes.ContainerNameEmptyError(errortypes.ContainerNameEmptyMessage) + } else if !(containerName == ContainerPolicySelectAllFlag) && !general.SliceContains(existContainerList, containerName) { + return nil, errortypes.ContainersNotFoundError(errortypes.ContainerNotFoundMessage, containerName) + } else if len(resourcePolicy.ControlledResourcesPolicies) == 0 { + return nil, errortypes.ControlledResourcesPoliciesEmptyError(errortypes.ControlledResourcesPoliciesEmptyMessage, containerName) + } + resourcePoliciesMap[containerName] = resourcePolicy + } + + if defaultPolicy, exist := resourcePoliciesMap[ContainerPolicySelectAllFlag]; exist { + for _, container := range existContainerList { + if _, ok := resourcePoliciesMap[container]; !ok { + defaultPolicy.ContainerName = container + resourcePoliciesMap[container] = defaultPolicy + } + } + delete(resourcePoliciesMap, "*") + } + + containers := make([]Container, 0, len(resourcePoliciesMap)) + for _, containerResourcePolicy := range resourcePoliciesMap { + container := Container{ + ContainerName: containerResourcePolicy.ContainerName, + ContainerConfigs: []ContainerConfig{}, + } + for _, resourcesPolicy := range containerResourcePolicy.ControlledResourcesPolicies { + if ok := general.SliceContains(ResourceNames, resourcesPolicy.ResourceName); !ok { + return containers, errortypes.ResourceNameUnsupportedError(errortypes.ResourceNameUnsupportedMessage, ResourceNames) + } + if resourcesPolicy.ControlledValues != nil { + if ok := general.SliceContains(SupportControlledValues, *resourcesPolicy.ControlledValues); !ok { + return containers, errortypes.ControlledValuesUnsupportedError(errortypes.ResourceNameUnsupportedMessage, SupportControlledValues) + } + } + + containerConfig := ContainerConfig{ + ControlledResource: resourcesPolicy.ResourceName, + } + resourceBufferPercent := resourcesPolicy.BufferPercent + if resourceBufferPercent == nil { + containerConfig.ResourceBufferPercent = DefaultUsageBuffer + } else if *resourceBufferPercent > MaxUsageBuffer || *resourceBufferPercent < MinUsageBuffer { + return containers, errortypes.ResourceBuffersUnsupportedError(errortypes.ResourceBuffersUnsupportedMessage) + } else { + containerConfig.ResourceBufferPercent = *resourceBufferPercent + } + + container.ContainerConfigs = append(container.ContainerConfigs, containerConfig) + } + containers = append(containers, container) + } + return containers, nil +} diff --git a/pkg/util/resource-recommend/types/recommendation/validate_test.go b/pkg/util/resource-recommend/types/recommendation/validate_test.go new file mode 100644 index 000000000..c5a0ad39d --- /dev/null +++ b/pkg/util/resource-recommend/types/recommendation/validate_test.go @@ -0,0 +1,837 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommendation + +import ( + "context" + "reflect" + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + resourceutils "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/resource" + errortypes "github.com/kubewharf/katalyst-core/pkg/util/resource-recommend/types/error" +) + +func TestValidateAndExtractAlgorithmPolicy(t *testing.T) { + type args struct { + algorithmPolicyReq v1alpha1.AlgorithmPolicy + } + tests := []struct { + name string + args args + want v1alpha1.AlgorithmPolicy + wantErr *errortypes.CustomError + }{ + { + name: "algorithmPolicyReq.Algorithm empty", + args: args{ + algorithmPolicyReq: v1alpha1.AlgorithmPolicy{ + Recommender: "Recommender", + Algorithm: "", + Extensions: &runtime.RawExtension{ + Raw: []byte("Data"), + }, + }, + }, + want: v1alpha1.AlgorithmPolicy{ + Recommender: "default", + Algorithm: DefaultAlgorithmType, + Extensions: &runtime.RawExtension{ + Raw: []byte("Data"), + }, + }, + wantErr: nil, + }, + { + name: "algorithmPolicyReq.Algorithm unsupported", + args: args{ + algorithmPolicyReq: v1alpha1.AlgorithmPolicy{ + Recommender: "Recommender", + Algorithm: "mockAlgorithm", + Extensions: &runtime.RawExtension{ + Raw: []byte("Data"), + }, + }, + }, + want: v1alpha1.AlgorithmPolicy{ + Recommender: "default", + }, + wantErr: errortypes.AlgorithmUnsupportedError("mockAlgorithm", AlgorithmTypes), + }, + { + name: "algorithmPolicyReq.Algorithm supported", + args: args{ + algorithmPolicyReq: v1alpha1.AlgorithmPolicy{ + Recommender: "default", + Algorithm: PercentileAlgorithmType, + Extensions: &runtime.RawExtension{ + Raw: []byte("Data"), + }, + }, + }, + want: v1alpha1.AlgorithmPolicy{ + Recommender: "default", + Algorithm: PercentileAlgorithmType, + Extensions: &runtime.RawExtension{ + Raw: []byte("Data"), + }, + }, + wantErr: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotErr := ValidateAndExtractAlgorithmPolicy(tt.args.algorithmPolicyReq) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ValidateAndExtractAlgorithmPolicy() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("ValidateAndExtractAlgorithmPolicy() gotErr = %v, want %v", gotErr, tt.wantErr) + } + }) + } +} + +func TestValidateAndExtractContainers(t *testing.T) { + type args struct { + ctx context.Context + client client.Client + namespace string + targetRef v1alpha1.CrossVersionObjectReference + containerPolicies []v1alpha1.ContainerResourcePolicy + } + type env struct { + podLabels map[string]string + podAnnotations map[string]string + matchLabelKey string + matchLabelValue string + podName string + podNodeName string + unstructuredName string + unstructuredTemplateSpec map[string]interface{} + namespace string + kind string + apiVersion string + } + tests := []struct { + name string + args args + env env + want []Container + wantErr *errortypes.CustomError + }{ + { + name: errortypes.WorkloadNotFoundMessage, + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Kind: "kind", + Name: "Name", + APIVersion: "version", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + }, + }, + }, + want: nil, + wantErr: errortypes.WorkloadNotFoundError(errortypes.WorkloadNotFoundMessage), + }, + { + name: errortypes.WorkloadMatchedErrorMessage, + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Kind: "", + Name: "", + APIVersion: "", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + }, + }, + }, + }, + }, + want: nil, + wantErr: errortypes.WorkloadMatchedError(errortypes.WorkloadMatchedErrorMessage), + }, + { + name: "all right", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Name: "mockName5", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "container-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + }, + }, + }, + { + ContainerName: "container-2", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + }, + }, + }, + }, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels5", + }, + matchLabelKey: "app", + matchLabelValue: "mockPodLabels5", + podName: "mockPodName5", + podNodeName: "mockPodNodeName5", + unstructuredName: "mockName5", + unstructuredTemplateSpec: map[string]interface{}{ + "containers": []interface{}{ + map[string]interface{}{ + "name": "container-1", + "image": "image:latest", + }, + map[string]interface{}{ + "name": "container-2", + "image": "image:latest", + }, + }, + }, + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: []Container{ + { + ContainerName: "container-1", + ContainerConfigs: []ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: DefaultUsageBuffer, + }, + }, + }, + { + ContainerName: "container-2", + ContainerConfigs: []ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: DefaultUsageBuffer, + }, + }, + }, + }, + wantErr: nil, + }, + { + name: "controlled resources is empty", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Name: "mockName5", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + }, + }, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels5", + }, + matchLabelKey: "app", + matchLabelValue: "mockPodLabels5", + podName: "mockPodName5", + podNodeName: "mockPodNodeName5", + unstructuredName: "mockName5", + unstructuredTemplateSpec: map[string]interface{}{ + "containers": []interface{}{ + map[string]interface{}{ + "name": "container-1", + "image": "image:latest", + }, + map[string]interface{}{ + "name": "container-2", + "image": "image:latest", + }, + }, + }, + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: nil, + wantErr: errortypes.ControlledResourcesPoliciesEmptyError(errortypes.ControlledResourcesPoliciesEmptyMessage, "*"), + }, + { + name: errortypes.ContainersMatchedErrorMessage, + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Name: "mockName5", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "container-1", + }, + { + ContainerName: "container-2", + }, + }, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels5", + }, + podAnnotations: map[string]string{}, + matchLabelKey: "app", + matchLabelValue: "mockPodLabels5", + podName: "mockPodName5", + podNodeName: "mockPodNodeName5", + unstructuredName: "mockName5", + unstructuredTemplateSpec: map[string]interface{}{ + "container": []interface{}{ + map[string]interface{}{ + "name": "container-1", + "image": "image:latest", + }, + map[string]interface{}{ + "name": "container-2", + "image": "image:latest", + }, + }, + }, + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: nil, + wantErr: errortypes.ContainersMatchedError(errortypes.ContainersMatchedErrorMessage), + }, + { + name: "validate containers err", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Name: "mockName5", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "container-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + }, + }, + }, + { + ContainerName: "container-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + }, + }, + }, + }, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels5", + }, + podAnnotations: map[string]string{}, + matchLabelKey: "app", + matchLabelValue: "mockPodLabels5", + podName: "mockPodName5", + podNodeName: "mockPodNodeName5", + unstructuredName: "mockName5", + unstructuredTemplateSpec: map[string]interface{}{ + "containers": []interface{}{ + map[string]interface{}{ + "name": "container-1", + "image": "image:latest", + }, + map[string]interface{}{ + "name": "container-2", + "image": "image:latest", + }, + }, + }, + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: nil, + wantErr: errortypes.ContainerDuplicateError(errortypes.ContainerDuplicateMessage, "container-1"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matchLabels := map[string]interface{}{ + tt.env.matchLabelKey: tt.env.matchLabelValue, + } + resourceutils.CreateMockUnstructured(matchLabels, tt.env.unstructuredTemplateSpec, tt.env.unstructuredName, tt.env.namespace, tt.env.apiVersion, tt.env.kind, tt.args.client) + resourceutils.CreateMockPod(tt.env.podLabels, tt.env.podAnnotations, tt.env.podName, tt.env.namespace, tt.env.podNodeName, nil, tt.args.client) + got, gotErr := ValidateAndExtractContainers(tt.args.ctx, tt.args.client, tt.args.namespace, tt.args.targetRef, tt.args.containerPolicies) + SortContainersByContainerName(got) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ValidateAndExtractContainers() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("ValidateAndExtractContainers() gotErr = %v, wantErr = %v", gotErr, tt.wantErr) + } + }) + } +} + +func TestValidateAndExtractTargetRef(t *testing.T) { + type args struct { + targetRefReq v1alpha1.CrossVersionObjectReference + } + tests := []struct { + name string + args args + want v1alpha1.CrossVersionObjectReference + wantErr *errortypes.CustomError + }{ + { + name: "targetRefReq.Name empty", + args: args{ + targetRefReq: v1alpha1.CrossVersionObjectReference{ + Name: "", + Kind: "Deployment", + }, + }, + want: v1alpha1.CrossVersionObjectReference{}, + wantErr: errortypes.WorkloadNameIsEmptyError(), + }, + { + name: "targetRefReq.Kind unsupported", + args: args{ + targetRefReq: v1alpha1.CrossVersionObjectReference{ + Name: "test", + Kind: "ReplicaSet", + }, + }, + want: v1alpha1.CrossVersionObjectReference{}, + wantErr: errortypes.WorkloadsUnsupportedError("ReplicaSet", TargetRefKinds), + }, + { + name: "all right", + args: args{ + targetRefReq: v1alpha1.CrossVersionObjectReference{ + Name: "test1", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + }, + want: v1alpha1.CrossVersionObjectReference{ + Name: "test1", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + wantErr: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotErr := ValidateAndExtractTargetRef(tt.args.targetRefReq) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ValidateAndExtractTargetRef() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("ValidateAndExtractTargetRef() gotErr = %v, want %v", gotErr, tt.wantErr) + } + }) + } +} + +func ptrInt32(i int32) *int32 { + return &i +} + +func Test_validateAndExtractContainers(t *testing.T) { + errControlledValues := v1alpha1.ContainerControlledValues("errControlledValues") + type args struct { + containerPolicies []v1alpha1.ContainerResourcePolicy + existContainerList []string + } + tests := []struct { + name string + args args + want []Container + wantErr *errortypes.CustomError + }{ + { + name: errortypes.ContainerDuplicateMessage, + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "mockContainerName-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: ptrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: ptrInt32(10), + }, + }, + }, + { + ContainerName: "mockContainerName-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: ptrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: ptrInt32(10), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1"}, + }, + want: nil, + wantErr: errortypes.ContainerDuplicateError(errortypes.ContainerDuplicateMessage, "mockContainerName-1"), + }, + { + name: errortypes.ContainerNameEmptyMessage, + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "mockContainerName-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: ptrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: ptrInt32(10), + }, + }, + }, + { + ContainerName: "", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: ptrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: ptrInt32(10), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1"}, + }, + want: nil, + wantErr: errortypes.ContainerNameEmptyError(errortypes.ContainerNameEmptyMessage), + }, + { + name: errortypes.ContainerDuplicateMessage, + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "mockContainerName-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: ptrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: ptrInt32(10), + }, + }, + }, + { + ContainerName: "mockContainerName-2", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: ptrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: ptrInt32(10), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1"}, + }, + want: nil, + wantErr: errortypes.ContainersNotFoundError(errortypes.ContainerNotFoundMessage, "mockContainerName-2"), + }, + { + name: "Container name is only *", + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: ptrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: ptrInt32(20), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1", "mockContainerName-2"}, + }, + want: []Container{ + { + ContainerName: "mockContainerName-1", + ContainerConfigs: []ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: 10, + }, + { + ControlledResource: v1.ResourceMemory, + ResourceBufferPercent: 20, + }, + }, + }, + { + ContainerName: "mockContainerName-2", + ContainerConfigs: []ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: 10, + }, + { + ControlledResource: v1.ResourceMemory, + ResourceBufferPercent: 20, + }, + }, + }, + }, + wantErr: nil, + }, + { + name: "Container name list includes *", + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: ptrInt32(20), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: ptrInt32(20), + }, + }, + }, + { + ContainerName: "mockContainerName-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: ptrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: ptrInt32(10), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1", "mockContainerName-2", "mockContainerName-3"}, + }, + want: []Container{ + { + ContainerName: "mockContainerName-1", + ContainerConfigs: []ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: 10, + }, + { + ControlledResource: v1.ResourceMemory, + ResourceBufferPercent: 10, + }, + }, + }, + { + ContainerName: "mockContainerName-2", + ContainerConfigs: []ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: 20, + }, + { + ControlledResource: v1.ResourceMemory, + ResourceBufferPercent: 20, + }, + }, + }, + { + ContainerName: "mockContainerName-3", + ContainerConfigs: []ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: 20, + }, + { + ControlledResource: v1.ResourceMemory, + ResourceBufferPercent: 20, + }, + }, + }, + }, + wantErr: nil, + }, + { + name: errortypes.ResourceNameUnsupportedMessage, + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: ptrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: ptrInt32(20), + }, + { + ResourceName: "errResource", + BufferPercent: ptrInt32(20), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1", "mockContainerName-2"}, + }, + want: []Container{}, + wantErr: errortypes.ResourceNameUnsupportedError(errortypes.ResourceNameUnsupportedMessage, ResourceNames), + }, + { + name: errortypes.ResourceBuffersUnsupportedMessage, + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: ptrInt32(101), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: ptrInt32(20), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1", "mockContainerName-2"}, + }, + want: []Container{}, + wantErr: errortypes.ResourceBuffersUnsupportedError(errortypes.ResourceBuffersUnsupportedMessage), + }, + { + name: errortypes.ResourceBuffersUnsupportedMessage, + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: ptrInt32(101), + ControlledValues: &errControlledValues, + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: ptrInt32(20), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1", "mockContainerName-2"}, + }, + want: []Container{}, + wantErr: errortypes.ControlledValuesUnsupportedError(errortypes.ResourceNameUnsupportedMessage, SupportControlledValues), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotErr := validateAndExtractContainers(tt.args.containerPolicies, tt.args.existContainerList) + SortContainersByContainerName(got) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("validateAndExtractContainers() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("validateAndExtractContainers() gotErr = %v, want %v", gotErr, tt.wantErr) + } + }) + } +} diff --git a/pkg/util/resource-recommend/types/recommendation/validate_test_util.go b/pkg/util/resource-recommend/types/recommendation/validate_test_util.go new file mode 100644 index 000000000..a1e9218aa --- /dev/null +++ b/pkg/util/resource-recommend/types/recommendation/validate_test_util.go @@ -0,0 +1,31 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommendation + +import ( + "sort" +) + +type ByContainerName []Container + +func (c ByContainerName) Len() int { return len(c) } +func (c ByContainerName) Less(i, j int) bool { return c[i].ContainerName < c[j].ContainerName } +func (c ByContainerName) Swap(i, j int) { c[i], c[j] = c[j], c[i] } + +func SortContainersByContainerName(containers []Container) { + sort.Sort(ByContainerName(containers)) +}