From ca0cb72ddfcd23093689640de81f0b0b1f0c7e7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E5=BC=A0=E5=B4=A7?= Date: Sun, 18 Feb 2024 15:52:31 +0800 Subject: [PATCH] feat(resource-recommend): add resource recommend controller --- .../app/controller/resourcerecommender.go | 50 + .../app/enablecontrollers.go | 1 + .../app/options/controller.go | 16 +- .../app/options/resourcerecommender.go | 100 ++ go.mod | 57 +- go.sum | 200 +++- pkg/config/controller/controller_base.go | 16 +- pkg/config/controller/resourcerecommender.go | 39 + .../controller/controller.go | 205 ++++ .../controller/oom_recorder_controller.go | 97 ++ .../resourcerecommend_controller.go | 240 ++++ .../datasource/datasource_proxy.go | 66 ++ .../datasource/datasource_proxy_test.go | 107 ++ .../prometheus/auth/volc-engine/client.go | 73 ++ .../prometheus/auth/volc-engine/config.go | 55 + .../auth/volc-engine/creds/credentials.go | 111 ++ .../ecs_role_provider/ecs_role_provider.go | 90 ++ .../prometheus/auth/volc-engine/env/env.go | 36 + .../prometheus/mock_prometheus_client.go | 114 ++ .../prometheus/prometheus_auth_provider.go | 65 ++ .../prometheus/prometheus_client.go | 114 ++ .../prometheus/prometheus_provider.go | 148 +++ .../prometheus/prometheus_provider_test.go | 453 ++++++++ .../datasource/prometheus/prometheus_query.go | 58 + .../prometheus/prometheus_query_test.go | 136 +++ .../resource-recommend/datasource/types.go | 99 ++ .../datasource/utils/operator.go | 86 ++ .../datasource/utils/operator_test.go | 72 ++ .../resource-recommend/oom/oom_recorder.go | 222 ++++ .../oom/oom_recorder_test.go | 321 ++++++ .../processor/common/task_key.go | 71 ++ .../processor/manager/processor_manager.go | 75 ++ .../manager/processor_manager_test.go | 118 ++ .../processor/percentile/process_gc.go | 128 +++ .../processor/percentile/process_gc_test.go | 120 ++ .../processor/percentile/process_tasks.go | 99 ++ .../percentile/process_tasks_test.go | 289 +++++ .../processor/percentile/process_util.go | 62 + .../processor/percentile/process_util_test.go | 91 ++ .../processor/percentile/processor.go | 219 ++++ .../processor/percentile/processor_mock.go | 131 +++ .../processor/percentile/processor_test.go | 370 ++++++ .../processor/percentile/task/config.go | 133 +++ .../processor/percentile/task/config_test.go | 190 ++++ .../percentile/task/histogram_task.go | 191 ++++ .../percentile/task/histogram_task_test.go | 489 ++++++++ .../resource-recommend/processor/processor.go | 35 + .../manager/recommender_manager.go | 47 + .../recommender/recommender.go | 26 + .../recommenders/percentile_recommender.go | 190 ++++ .../percentile_recommender_test.go | 242 ++++ .../types/conditions/conditions.go | 109 ++ .../resource-recommend/types/error/errors.go | 34 + .../resource-recommend/types/error/process.go | 93 ++ .../types/error/recommend.go | 35 + .../types/error/validate.go | 189 ++++ .../types/recommendation/recommendation.go | 147 +++ .../types/recommendation/validate.go | 165 +++ .../types/recommendation/validate_test.go | 1004 +++++++++++++++++ .../recommendation/validate_test_util.go | 31 + .../resource-recommend/utils/k8s_resource.go | 160 +++ .../utils/k8s_resource_test.go | 656 +++++++++++ .../utils/k8s_resource_test_util.go | 52 + .../resource-recommend/utils/list.go | 39 + .../resource-recommend/utils/list_test.go | 147 +++ .../resource-recommend/utils/log/logger.go | 104 ++ .../utils/log/logger_test.go | 343 ++++++ .../resource-recommend/utils/ptr.go | 25 + .../resource-recommend/utils/string.go | 37 + 69 files changed, 10068 insertions(+), 65 deletions(-) create mode 100644 cmd/katalyst-controller/app/controller/resourcerecommender.go create mode 100644 cmd/katalyst-controller/app/options/resourcerecommender.go create mode 100644 pkg/config/controller/resourcerecommender.go create mode 100644 pkg/controller/resource-recommend/controller/controller.go create mode 100644 pkg/controller/resource-recommend/controller/oom_recorder_controller.go create mode 100644 pkg/controller/resource-recommend/controller/resourcerecommend_controller.go create mode 100644 pkg/controller/resource-recommend/datasource/datasource_proxy.go create mode 100644 pkg/controller/resource-recommend/datasource/datasource_proxy_test.go create mode 100644 pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/client.go create mode 100644 pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/config.go create mode 100644 pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/creds/credentials.go create mode 100644 pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/creds/ecs_role_provider/ecs_role_provider.go create mode 100644 pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/env/env.go create mode 100644 pkg/controller/resource-recommend/datasource/prometheus/mock_prometheus_client.go create mode 100644 pkg/controller/resource-recommend/datasource/prometheus/prometheus_auth_provider.go create mode 100644 pkg/controller/resource-recommend/datasource/prometheus/prometheus_client.go create mode 100644 pkg/controller/resource-recommend/datasource/prometheus/prometheus_provider.go create mode 100644 pkg/controller/resource-recommend/datasource/prometheus/prometheus_provider_test.go create mode 100644 pkg/controller/resource-recommend/datasource/prometheus/prometheus_query.go create mode 100644 pkg/controller/resource-recommend/datasource/prometheus/prometheus_query_test.go create mode 100644 pkg/controller/resource-recommend/datasource/types.go create mode 100644 pkg/controller/resource-recommend/datasource/utils/operator.go create mode 100644 pkg/controller/resource-recommend/datasource/utils/operator_test.go create mode 100644 pkg/controller/resource-recommend/oom/oom_recorder.go create mode 100644 pkg/controller/resource-recommend/oom/oom_recorder_test.go create mode 100644 pkg/controller/resource-recommend/processor/common/task_key.go create mode 100644 pkg/controller/resource-recommend/processor/manager/processor_manager.go create mode 100644 pkg/controller/resource-recommend/processor/manager/processor_manager_test.go create mode 100644 pkg/controller/resource-recommend/processor/percentile/process_gc.go create mode 100644 pkg/controller/resource-recommend/processor/percentile/process_gc_test.go create mode 100644 pkg/controller/resource-recommend/processor/percentile/process_tasks.go create mode 100644 pkg/controller/resource-recommend/processor/percentile/process_tasks_test.go create mode 100644 pkg/controller/resource-recommend/processor/percentile/process_util.go create mode 100644 pkg/controller/resource-recommend/processor/percentile/process_util_test.go create mode 100644 pkg/controller/resource-recommend/processor/percentile/processor.go create mode 100644 pkg/controller/resource-recommend/processor/percentile/processor_mock.go create mode 100644 pkg/controller/resource-recommend/processor/percentile/processor_test.go create mode 100644 pkg/controller/resource-recommend/processor/percentile/task/config.go create mode 100644 pkg/controller/resource-recommend/processor/percentile/task/config_test.go create mode 100644 pkg/controller/resource-recommend/processor/percentile/task/histogram_task.go create mode 100644 pkg/controller/resource-recommend/processor/percentile/task/histogram_task_test.go create mode 100644 pkg/controller/resource-recommend/processor/processor.go create mode 100644 pkg/controller/resource-recommend/recommender/manager/recommender_manager.go create mode 100644 pkg/controller/resource-recommend/recommender/recommender.go create mode 100644 pkg/controller/resource-recommend/recommender/recommenders/percentile_recommender.go create mode 100644 pkg/controller/resource-recommend/recommender/recommenders/percentile_recommender_test.go create mode 100644 pkg/controller/resource-recommend/types/conditions/conditions.go create mode 100644 pkg/controller/resource-recommend/types/error/errors.go create mode 100644 pkg/controller/resource-recommend/types/error/process.go create mode 100644 pkg/controller/resource-recommend/types/error/recommend.go create mode 100644 pkg/controller/resource-recommend/types/error/validate.go create mode 100644 pkg/controller/resource-recommend/types/recommendation/recommendation.go create mode 100644 pkg/controller/resource-recommend/types/recommendation/validate.go create mode 100644 pkg/controller/resource-recommend/types/recommendation/validate_test.go create mode 100644 pkg/controller/resource-recommend/types/recommendation/validate_test_util.go create mode 100644 pkg/controller/resource-recommend/utils/k8s_resource.go create mode 100644 pkg/controller/resource-recommend/utils/k8s_resource_test.go create mode 100644 pkg/controller/resource-recommend/utils/k8s_resource_test_util.go create mode 100644 pkg/controller/resource-recommend/utils/list.go create mode 100644 pkg/controller/resource-recommend/utils/list_test.go create mode 100644 pkg/controller/resource-recommend/utils/log/logger.go create mode 100644 pkg/controller/resource-recommend/utils/log/logger_test.go create mode 100644 pkg/controller/resource-recommend/utils/ptr.go create mode 100644 pkg/controller/resource-recommend/utils/string.go diff --git a/cmd/katalyst-controller/app/controller/resourcerecommender.go b/cmd/katalyst-controller/app/controller/resourcerecommender.go new file mode 100644 index 0000000000..cc68557365 --- /dev/null +++ b/cmd/katalyst-controller/app/controller/resourcerecommender.go @@ -0,0 +1,50 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + "k8s.io/klog/v2" + + katalyst "github.com/kubewharf/katalyst-core/cmd/base" + "github.com/kubewharf/katalyst-core/pkg/config" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/controller" +) + +const ( + ResourceRecommenderControllerName = "resourcerecommender" +) + +func StartResourceRecommenderController( + ctx context.Context, + _ *katalyst.GenericContext, + conf *config.Configuration, + _ interface{}, + _ string, +) (bool, error) { + resourceRecommenderController, err := controller.NewResourceRecommenderController(ctx, + conf.GenericConfiguration, + conf.ControllersConfiguration.ResourceRecommenderConfig) + if err != nil { + klog.Errorf("failed to new ResourceRecommender controller") + return false, err + } + + go resourceRecommenderController.Run() + return true, nil +} diff --git a/cmd/katalyst-controller/app/enablecontrollers.go b/cmd/katalyst-controller/app/enablecontrollers.go index f827e17337..0d74453fc8 100644 --- a/cmd/katalyst-controller/app/enablecontrollers.go +++ b/cmd/katalyst-controller/app/enablecontrollers.go @@ -54,6 +54,7 @@ func init() { controllerInitializers.Store(controller.MonitorControllerName, ControllerStarter{Starter: controller.StartMonitorController}) controllerInitializers.Store(controller.OvercommitControllerName, ControllerStarter{Starter: controller.StartOvercommitController}) controllerInitializers.Store(controller.TideControllerName, ControllerStarter{Starter: controller.StartTideController}) + controllerInitializers.Store(controller.ResourceRecommenderControllerName, ControllerStarter{Starter: controller.StartResourceRecommenderController}) } // RegisterControllerInitializer is used to register user-defined controllers diff --git a/cmd/katalyst-controller/app/options/controller.go b/cmd/katalyst-controller/app/options/controller.go index e662e99d0e..d915dddb93 100644 --- a/cmd/katalyst-controller/app/options/controller.go +++ b/cmd/katalyst-controller/app/options/controller.go @@ -30,16 +30,18 @@ type ControllersOptions struct { *LifeCycleOptions *MonitorOptions *OvercommitOptions + *ResourceRecommenderOptions } func NewControllersOptions() *ControllersOptions { return &ControllersOptions{ - VPAOptions: NewVPAOptions(), - KCCOptions: NewKCCOptions(), - SPDOptions: NewSPDOptions(), - LifeCycleOptions: NewLifeCycleOptions(), - MonitorOptions: NewMonitorOptions(), - OvercommitOptions: NewOvercommitOptions(), + VPAOptions: NewVPAOptions(), + KCCOptions: NewKCCOptions(), + SPDOptions: NewSPDOptions(), + LifeCycleOptions: NewLifeCycleOptions(), + MonitorOptions: NewMonitorOptions(), + OvercommitOptions: NewOvercommitOptions(), + ResourceRecommenderOptions: NewResourceRecommenderOptions(), } } @@ -50,6 +52,7 @@ func (o *ControllersOptions) AddFlags(fss *cliflag.NamedFlagSets) { o.LifeCycleOptions.AddFlags(fss) o.MonitorOptions.AddFlags(fss) o.OvercommitOptions.AddFlags(fss) + o.ResourceRecommenderOptions.AddFlags(fss) } // ApplyTo fills up config with options @@ -62,6 +65,7 @@ func (o *ControllersOptions) ApplyTo(c *controllerconfig.ControllersConfiguratio errList = append(errList, o.LifeCycleOptions.ApplyTo(c.LifeCycleConfig)) errList = append(errList, o.MonitorOptions.ApplyTo(c.MonitorConfig)) errList = append(errList, o.OvercommitOptions.ApplyTo(c.OvercommitConfig)) + errList = append(errList, o.ResourceRecommenderOptions.ApplyTo(c.ResourceRecommenderConfig)) return errors.NewAggregate(errList) } diff --git a/cmd/katalyst-controller/app/options/resourcerecommender.go b/cmd/katalyst-controller/app/options/resourcerecommender.go new file mode 100644 index 0000000000..e1a0eeb87b --- /dev/null +++ b/cmd/katalyst-controller/app/options/resourcerecommender.go @@ -0,0 +1,100 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "time" + + cliflag "k8s.io/component-base/cli/flag" + + "github.com/kubewharf/katalyst-core/pkg/config/controller" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus" +) + +type ResourceRecommenderOptions struct { + OOMRecordMaxNumber int `desc:"max number for oom record"` + + HealthProbeBindPort string `desc:"The port the health probe binds to."` + MetricsBindPort string `desc:"The port the metric endpoint binds to."` + + // available datasource: prom + DataSource []string + // DataSourcePromConfig is the prometheus datasource config + DataSourcePromConfig prometheus.PromConfig + + // LogVerbosityLevel to specify log verbosity level. (The default level is 4) + // Set it to something larger than 4 if more detailed logs are needed. + LogVerbosityLevel string +} + +// NewResourceRecommenderOptions creates a new Options with a default config. +func NewResourceRecommenderOptions() *ResourceRecommenderOptions { + return &ResourceRecommenderOptions{ + OOMRecordMaxNumber: 5000, + HealthProbeBindPort: "8080", + MetricsBindPort: "8081", + DataSource: []string{"prom"}, + DataSourcePromConfig: prometheus.PromConfig{ + KeepAlive: 60 * time.Second, + Timeout: 3 * time.Minute, + BRateLimit: false, + MaxPointsLimitPerTimeSeries: 11000, + }, + LogVerbosityLevel: "4", + } +} + +// AddFlags adds flags to the specified FlagSet +func (o *ResourceRecommenderOptions) AddFlags(fss *cliflag.NamedFlagSets) { + fs := fss.FlagSet("resource-recommend") + + fs.StringVar(&o.LogVerbosityLevel, "v", "4", "log verbosity level. The default level is 4. Set it to something larger than 4 if more detailed logs are needed") + fs.IntVar(&o.OOMRecordMaxNumber, "oom-record-max-number", 5000, "Max number for oom records to store in configmap") + + fs.StringVar(&o.HealthProbeBindPort, "health-probe-bind-port", "8080", "The port the health probe binds to.") + fs.StringVar(&o.MetricsBindPort, "metrics-bind-port", "8081", "The port the metric endpoint binds to.") + + fs.StringSliceVar(&o.DataSource, "datasource", []string{"prom"}, "available datasource: prom") + fs.StringVar(&o.DataSourcePromConfig.Auth.Type, "prometheus-auth-type", "", "prometheus auth type") + fs.StringVar(&o.DataSourcePromConfig.Address, "prometheus-address", "", "prometheus address") + fs.StringVar(&o.DataSourcePromConfig.Auth.Username, "prometheus-auth-username", "", "prometheus auth username") + fs.StringVar(&o.DataSourcePromConfig.Auth.Password, "prometheus-auth-password", "", "prometheus auth password") + fs.StringVar(&o.DataSourcePromConfig.Auth.BearerToken, "prometheus-auth-bearertoken", "", "prometheus auth bearertoken") + fs.DurationVar(&o.DataSourcePromConfig.KeepAlive, "prometheus-keepalive", 60*time.Second, "prometheus keep alive") + fs.DurationVar(&o.DataSourcePromConfig.Timeout, "prometheus-timeout", 3*time.Minute, "prometheus timeout") + fs.BoolVar(&o.DataSourcePromConfig.BRateLimit, "prometheus-bratelimit", false, "prometheus bratelimit") + fs.IntVar(&o.DataSourcePromConfig.MaxPointsLimitPerTimeSeries, "prometheus-maxpoints", 11000, "prometheus max points limit per time series") + +} + +func (o *ResourceRecommenderOptions) ApplyTo(c *controller.ResourceRecommenderConfig) error { + c.OOMRecordMaxNumber = o.OOMRecordMaxNumber + c.HealthProbeBindPort = o.HealthProbeBindPort + c.MetricsBindPort = o.MetricsBindPort + c.DataSource = o.DataSource + c.DataSourcePromConfig = o.DataSourcePromConfig + c.LogVerbosityLevel = o.LogVerbosityLevel + return nil +} + +func (o *ResourceRecommenderOptions) Config() (*controller.ResourceRecommenderConfig, error) { + c := &controller.ResourceRecommenderConfig{} + if err := o.ApplyTo(c); err != nil { + return nil, err + } + return c, nil +} diff --git a/go.mod b/go.mod index 3a413e666e..d4780a55ba 100644 --- a/go.mod +++ b/go.mod @@ -12,43 +12,47 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/golang/protobuf v1.5.2 github.com/google/cadvisor v0.44.2 + github.com/google/uuid v1.3.0 github.com/kubewharf/katalyst-api v0.4.1-0.20240222122824-be538f641f58 github.com/montanaflynn/stats v0.7.1 github.com/opencontainers/runc v1.1.6 github.com/opencontainers/selinux v1.10.0 github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.12.1 - github.com/prometheus/client_model v0.2.0 - github.com/prometheus/common v0.32.1 + github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_model v0.3.0 + github.com/prometheus/common v0.37.0 github.com/slok/kubewebhook v0.11.0 - github.com/spf13/cobra v1.4.0 + github.com/spf13/cobra v1.6.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.1 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae + github.com/volcengine/volc-sdk-golang v1.0.71 go.opentelemetry.io/otel v0.20.0 go.opentelemetry.io/otel/exporters/metric/prometheus v0.20.0 go.opentelemetry.io/otel/metric v0.20.0 go.opentelemetry.io/otel/sdk v0.20.0 go.opentelemetry.io/otel/sdk/export/metric v0.20.0 go.opentelemetry.io/otel/sdk/metric v0.20.0 - go.uber.org/atomic v1.7.0 + go.uber.org/atomic v1.9.0 golang.org/x/sys v0.7.0 golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 - gonum.org/v1/gonum v0.6.2 + gonum.org/v1/gonum v0.8.2 google.golang.org/grpc v1.51.0 - k8s.io/api v0.24.16 - k8s.io/apimachinery v0.24.16 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.26.1 + k8s.io/apimachinery v0.26.1 k8s.io/apiserver v0.24.16 k8s.io/autoscaler/cluster-autoscaler v0.0.0-20231010095923-8a61add71154 - k8s.io/client-go v0.24.16 - k8s.io/component-base v0.24.16 + k8s.io/autoscaler/vertical-pod-autoscaler v1.0.0 + k8s.io/client-go v0.26.1 + k8s.io/component-base v0.25.0 k8s.io/component-helpers v0.24.16 k8s.io/cri-api v0.24.6 k8s.io/klog/v2 v2.80.1 k8s.io/kube-aggregator v0.24.6 k8s.io/kubelet v0.24.6 k8s.io/kubernetes v1.24.16 - k8s.io/metrics v0.24.6 + k8s.io/metrics v0.25.0 k8s.io/utils v0.0.0-20221108210102-8e77b1f39fe2 sigs.k8s.io/controller-runtime v0.11.2 sigs.k8s.io/custom-metrics-apiserver v1.24.0 @@ -60,10 +64,9 @@ require ( github.com/Microsoft/go-winio v0.4.17 // indirect github.com/NYTimes/gziphandler v1.1.1 // indirect github.com/OneOfOne/xxhash v1.2.5 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.1.2 // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect @@ -76,23 +79,22 @@ require ( github.com/felixge/httpsnoop v1.0.1 // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.19.15 // indirect github.com/godbus/dbus/v5 v5.0.6 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/gnostic v0.6.9 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/imdario/mergo v0.3.12 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/mailru/easyjson v0.7.6 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect github.com/moby/spdystream v0.2.0 // indirect github.com/moby/sys/mountinfo v0.6.0 // indirect @@ -105,8 +107,9 @@ require ( github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect - github.com/sirupsen/logrus v1.8.1 // indirect + github.com/prometheus/procfs v0.8.0 // indirect + github.com/sirupsen/logrus v1.9.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect go.etcd.io/etcd/api/v3 v3.5.4 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect go.etcd.io/etcd/client/v3 v3.5.4 // indirect @@ -116,27 +119,27 @@ require ( go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect go.opentelemetry.io/otel/trace v0.20.0 // indirect go.opentelemetry.io/proto/otlp v0.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect + go.uber.org/multierr v1.7.0 // indirect go.uber.org/zap v1.19.1 // indirect golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be // indirect golang.org/x/net v0.9.0 // indirect - golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect + golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect golang.org/x/sync v0.1.0 // indirect golang.org/x/term v0.7.0 // indirect golang.org/x/text v0.9.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect gomodules.xyz/jsonpatch/v3 v3.0.1 // indirect gomodules.xyz/orderedmap v0.1.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect - google.golang.org/protobuf v1.28.0 // indirect - gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/apiextensions-apiserver v0.24.2 // indirect k8s.io/cloud-provider v0.24.16 // indirect k8s.io/csi-translation-lib v0.24.16 // indirect - k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect + k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect k8s.io/kube-scheduler v0.24.6 // indirect k8s.io/mount-utils v0.24.16 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37 // indirect diff --git a/go.sum b/go.sum index 92088a55e6..d8c907df94 100644 --- a/go.sum +++ b/go.sum @@ -56,9 +56,12 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Djarvur/go-err113 v0.0.0-20200511133814-5174e21577d5/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= github.com/GoogleCloudPlatform/k8s-cloud-provider v1.16.1-0.20210702024009-ea6160c1d0e3/go.mod h1:8XasY4ymP2V/tn2OOV9ZadmiTE1FIB/h3W+yNlPttKw= github.com/HdrHistogram/hdrhistogram-go v1.0.0/go.mod h1:YzE1EgsuAz8q9lfGdlxBZo2Ma655+PfKp2mlzcAqIFw= +github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= +github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd/go.mod h1:64YHyfSL2R96J44Nlwm39UHepQbyR5q10x7iYa1ks2E= @@ -74,12 +77,12 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/sarama v1.30.1/go.mod h1:hGgx05L/DiW8XYBXeJdKIN6V2QUy2H6JqME5VT1NLRw= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/Shopify/toxiproxy/v2 v2.1.6-0.20210914104332-15ea381dcdae/go.mod h1:/cvHQkZ1fst0EmZnA5dFtiQdWCNCFYzb+uE2vqVgvx0= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= @@ -98,17 +101,24 @@ github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/auth0/go-jwt-middleware v1.0.1/go.mod h1:YSeUX3z6+TF2H+7padiEqNJ73Zy9vXW72U//IgN0BIM= +github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/aws/aws-sdk-go v1.38.49/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aws/aws-sdk-go-v2 v1.9.1/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= +github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.8.1/go.mod h1:CM+19rL1+4dFWnOQKwDc7H1KwXTz+h61oUSHyhV0b3o= +github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -126,7 +136,11 @@ github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx2 github.com/bombsimon/wsl/v3 v3.1.0/go.mod h1:st10JtZYLE4D5sC7b8xV4zTKZwAQjCH/Hy2Pm1FNZIc= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/casbin/casbin/v2 v2.37.0/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo= +github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= @@ -143,6 +157,9 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.7.0 h1:1k/q3ATgxSXRdrmPfH8d7YK0GfqVsEKZAX9dQZvs56k= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1wt9Z3DP8O6W3rvwCt0REIlshg1InHImaLW0t3ObY0= @@ -191,6 +208,7 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= @@ -218,6 +236,7 @@ github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:Htrtb github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= @@ -240,6 +259,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/euank/go-kmsg-parser v2.0.0+incompatible/go.mod h1:MhmAMZ8V4CYH4ybgdRwPr2TU5ThnS43puaKEMpja1uw= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -249,6 +269,7 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZM github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= @@ -257,7 +278,9 @@ github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= +github.com/franela/goblin v0.0.0-20210519012713-85d372ac71e2/go.mod h1:VzmDKDJVZI3aJmnRI9VjAn9nJ8qPPsN1fqzr9dqInIo= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= @@ -277,10 +300,13 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/kit v0.12.0/go.mod h1:lHd+EkCZPIwYItmGDDRdhinkzX2A1sj+M9biaEaizzs= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -295,8 +321,8 @@ github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUe github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= -github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= @@ -304,6 +330,7 @@ github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/ github.com/go-ozzo/ozzo-validation v3.5.0+incompatible/go.mod h1:gsEKFIVnabGBt6mXmxK0MoFy+cZoTJY6mu5Ll3LVLBU= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4= github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ= github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY= @@ -315,6 +342,7 @@ github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslW github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= github.com/go-toolsmith/typep v1.0.2/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU= github.com/go-xmlfmt/xmlfmt v0.0.0-20191208150333-d5b6f63a941b/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= +github.com/go-zookeeper/zk v1.0.2/go.mod h1:nOB03cncLtlp4t+UAkGSV+9beXP/akpekBwL+UX1Qcw= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -329,6 +357,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -368,6 +397,8 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4= github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk= github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0= @@ -408,8 +439,9 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= @@ -446,6 +478,8 @@ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= @@ -465,40 +499,65 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= +github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= github.com/heketi/heketi v10.3.0+incompatible/go.mod h1:bB9ly3RchcQqsQ9CpyaQwvva7RS5ytVoSoholZQON6o= github.com/heketi/tests v0.0.0-20151005000721-f3775cbcefd6/go.mod h1:xGMAM8JLi7UkZt1i4FQeQy0R2T8GLUwQhOP5M1gBhy4= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/hudl/fargo v1.4.0/go.mod h1:9Ai6uvFy5fQNq6VPKtg+Ceq1+eTY4nKUlR2JElEOcDo= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/influxdata/influxdb1-client v0.0.0-20200827194710-b269163b24ab/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/ishidawataru/sctp v0.0.0-20190723014705-7c296d48a2b5/go.mod h1:DM4VvS+hD/kDi1U1QsX2fnZowwBhqD0Dk3bRPKF/Oc8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jingyugao/rowserrcheck v0.0.0-20191204022205-72ab7603b68a/go.mod h1:xRskid8CManxVta/ALEhJha/pweKBaVG6fWgc0yH25s= github.com/jirfag/go-printf-func-name v0.0.0-20191110105641-45db9963cdd3/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -516,6 +575,7 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -533,6 +593,12 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -566,30 +632,44 @@ github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPK github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU= github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.2 h1:hAHbPm5IJGijwng3PWk09JkG9WeqChjprR5s9bBZ+OM= +github.com/matttproud/golang_protobuf_extensions v1.0.2/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY= +github.com/minio/highwayhash v1.0.1/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.34/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= @@ -600,6 +680,7 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/ipvs v1.0.1/go.mod h1:2pngiyseZbIKXNv7hsKj3O9UEz30c53MT9005gt2hxQ= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= @@ -633,17 +714,23 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+ github.com/nakabonne/nestif v0.3.0/go.mod h1:dI314BppzXjJ4HsCnbo7XzrJHPszZsjnk5wEBSYHI2c= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/jwt v1.2.2/go.mod h1:/xX356yQA6LuXI9xWW7mZNpxgF2mBmGecH+Fj34sP5Q= +github.com/nats-io/jwt/v2 v2.0.3/go.mod h1:VRP+deawSXyhNjXmxPCHskrR6Mq50BqpEI5SEcNiGlY= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= +github.com/nats-io/nats-server/v2 v2.5.0/go.mod h1:Kj86UtrXAL6LwYRA6H4RqzkHhK0Vcv2ZnKD5WbQ1t3g= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nats.go v1.12.1/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.2.0/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= +github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nishanths/exhaustive v0.0.0-20200811152831-6cf413ae40e0/go.mod h1:wBEpHwM2OdmeNpdCvRPUlkEbBuaFmcK4Wv8Q7FuGW3c= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= @@ -655,11 +742,13 @@ github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvwxz1E= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -685,16 +774,20 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/openzipkin/zipkin-go v0.2.5/go.mod h1:KpXfKdgRDnnhsxw4pNIH9Md5lyFqKUa4YDFlwRYAMyE= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/performancecopilot/speed/v4 v4.0.0/go.mod h1:qxrSyuDGrTOWfV+uKRFhfxw6h/4HXRGUiZiufxo49BM= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -704,37 +797,44 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.8.0/go.mod h1:O9VU6huf47PktckDQfMTX0Y8tY0/7TSWwj+ITvv0TnM= github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.14.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= +github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -744,20 +844,23 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= github.com/quasilyte/go-ruleguard v0.2.0/go.mod h1:2RT/tf0Ce0UDj5y243iWKosQogJd8+1G3Rs2fxmlYnw= github.com/quasilyte/regex/syntax v0.0.0-20200407221936-30656e2c4a95/go.mod h1:rlzQ04UMyJXu/aOvhd8qT+hvDrFpiwqp8MRXDY9szc0= github.com/quobyte/api v0.1.8/go.mod h1:jL7lIHrmqQ7yh05OJ+eEEdHr0u/kmT1Ff9iHd+4H6VI= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021/go.mod h1:DM5xW0nvfNNm2uytzsvhI3OnX8uzaRAg8UX/CnDqbto= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -782,8 +885,9 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/slok/kubewebhook v0.11.0 h1:mRUOHXpMNxROTcqGVq06BQX2r13cT1Kjw0ylcJhTg0g= github.com/slok/kubewebhook v0.11.0/go.mod h1:HWkaQH3ZbQpLeP3ylW/NPhOaYByxCIRU36HPmUEoqyo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -806,8 +910,9 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= -github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI= +github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -823,7 +928,9 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag github.com/storageos/go-api v2.2.0+incompatible/go.mod h1:ZrLn+e0ZuF3Y65PNF6dIwbJPZqfmtCXxFm9ckv0agOY= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= +github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= +github.com/streadway/handy v0.0.0-20200128134331-0f66f006fb2e/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -850,6 +957,7 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= @@ -858,6 +966,7 @@ github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= @@ -869,6 +978,11 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= +github.com/volcengine/volc-sdk-golang v1.0.71 h1:KBHp8WbmHcvHDKwt0OvPihhjc6jhQef/64QMplXgj5Y= +github.com/volcengine/volc-sdk-golang v1.0.71/go.mod h1:mp3ZNJPp+9upTRgfMIS7+YeaAVuxn60VQfvlOoc2TOM= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= @@ -948,15 +1062,17 @@ go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0H go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= @@ -972,11 +1088,19 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210915214749-c084706c2272/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A= golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1049,6 +1173,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1075,14 +1200,17 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= @@ -1101,8 +1229,9 @@ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 h1:RerP+noqYHUQ8CMRcPlC2nvTa4dcBIjegkuWdcUDuqg= golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1125,6 +1254,7 @@ golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1140,9 +1270,12 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1155,6 +1288,7 @@ golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1179,9 +1313,11 @@ golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1189,15 +1325,16 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210917161153-d61c044b1678/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1206,6 +1343,8 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= @@ -1232,7 +1371,9 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1260,6 +1401,7 @@ golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1311,6 +1453,7 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -1324,14 +1467,16 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= +gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= gomodules.xyz/jsonpatch/v3 v3.0.1 h1:Te7hKxV52TKCbNYq3t84tzKav3xhThdvSsSp/W89IyI= gomodules.xyz/jsonpatch/v3 v3.0.1/go.mod h1:CBhndykehEwTOlEfnsfJwvkFQbSN8YZFr9M+cIHAJto= gomodules.xyz/orderedmap v0.1.0 h1:fM/+TGh/O1KkqGR5xjTKg6bU8OKBkg7p0Y+x/J9m8Os= gomodules.xyz/orderedmap v0.1.0/go.mod h1:g9/TPUCm1t2gwD3j3zfV8uylyYhVdCNSi+xCEIu7yTU= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= -gonum.org/v1/gonum v0.6.2 h1:4r+yNT0+8SWcOkXP+63H2zQbN+USnC73cjGUxnDF94Q= gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= +gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e h1:jRyg0XfpwWlhEV8mDfdNGBeSJM2fuyh9Yjrnd8kF2Ts= gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= @@ -1417,6 +1562,7 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 h1:hrbNEivu7Zn1pxvHk6MBrq9iE22woVILTHqexqBxe6I= google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= @@ -1463,15 +1609,16 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= -gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -1481,6 +1628,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -1526,6 +1674,8 @@ k8s.io/apiserver v0.24.6 h1:LEVuJb3bwjbZSIA8Ltm6iNBJiFawJtEGmAP7bgMljIE= k8s.io/apiserver v0.24.6/go.mod h1:ZmMXlYsNUhWzEOMJv01JqjL/psUzmvcIF70yDnzen/A= k8s.io/autoscaler/cluster-autoscaler v0.0.0-20231010095923-8a61add71154 h1:UxhksRGQqiuHhuQsY935KuUAaRMG7yAFLz1BhrSbtWE= k8s.io/autoscaler/cluster-autoscaler v0.0.0-20231010095923-8a61add71154/go.mod h1:Yipa7m+SwYJwkNvtcxeI1SboxM5b8yecXvUw/pezXfk= +k8s.io/autoscaler/vertical-pod-autoscaler v1.0.0 h1:y0TgWoHaeYEv3L1MfLC+D2WVxyN1fGr6axURHXq+wHE= +k8s.io/autoscaler/vertical-pod-autoscaler v1.0.0/go.mod h1:w6/LjLR3DPQd57vlgvgbpzpuJKsCiily0+OzQI+nyfI= k8s.io/cli-runtime v0.24.6/go.mod h1:dZghAkzYOsDUDqz1pjFi1KDo+1ly1gKGrIPaX4HMHPs= k8s.io/client-go v0.24.6 h1:q7gZYyGL0Iv9zynYOFi5DHc3NFZ2aA0P56QpWFXbEyE= k8s.io/client-go v0.24.6/go.mod h1:qaJRTFlI24ONWGplf+j8IgTyb6ztpwS6SGfjcNCRpQ8= diff --git a/pkg/config/controller/controller_base.go b/pkg/config/controller/controller_base.go index 54f4b2c509..0d434bebf0 100644 --- a/pkg/config/controller/controller_base.go +++ b/pkg/config/controller/controller_base.go @@ -49,6 +49,7 @@ type ControllersConfiguration struct { *MonitorConfig *OvercommitConfig *TideConfig + *ResourceRecommenderConfig } func NewGenericControllerConfiguration() *GenericControllerConfiguration { @@ -57,12 +58,13 @@ func NewGenericControllerConfiguration() *GenericControllerConfiguration { func NewControllersConfiguration() *ControllersConfiguration { return &ControllersConfiguration{ - VPAConfig: NewVPAConfig(), - KCCConfig: NewKCCConfig(), - SPDConfig: NewSPDConfig(), - LifeCycleConfig: NewLifeCycleConfig(), - MonitorConfig: NewMonitorConfig(), - OvercommitConfig: NewOvercommitConfig(), - TideConfig: NewTideConfig(), + VPAConfig: NewVPAConfig(), + KCCConfig: NewKCCConfig(), + SPDConfig: NewSPDConfig(), + LifeCycleConfig: NewLifeCycleConfig(), + MonitorConfig: NewMonitorConfig(), + OvercommitConfig: NewOvercommitConfig(), + TideConfig: NewTideConfig(), + ResourceRecommenderConfig: NewResourceRecommenderConfig(), } } diff --git a/pkg/config/controller/resourcerecommender.go b/pkg/config/controller/resourcerecommender.go new file mode 100644 index 0000000000..8cc88d9534 --- /dev/null +++ b/pkg/config/controller/resourcerecommender.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus" + +type ResourceRecommenderConfig struct { + OOMRecordMaxNumber int + + HealthProbeBindPort string + MetricsBindPort string + + // available datasource: prom + DataSource []string + // DataSourcePromConfig is the prometheus datasource config + DataSourcePromConfig prometheus.PromConfig + + // LogVerbosityLevel to specify log verbosity level. (The default level is 4) + // Set it to something larger than 4 if more detailed logs are needed. + LogVerbosityLevel string +} + +func NewResourceRecommenderConfig() *ResourceRecommenderConfig { + return &ResourceRecommenderConfig{} +} diff --git a/pkg/controller/resource-recommend/controller/controller.go b/pkg/controller/resource-recommend/controller/controller.go new file mode 100644 index 0000000000..fe95dade7d --- /dev/null +++ b/pkg/controller/resource-recommend/controller/controller.go @@ -0,0 +1,205 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/healthz" + + spendsmartv1alpha1 "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/config/controller" + "github.com/kubewharf/katalyst-core/pkg/config/generic" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/oom" + processormanager "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/manager" + recommendermanager "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/recommender/manager" +) + +var ( + scheme = runtime.NewScheme() + setupLog = ctrl.Log.WithName("setup") + + preCacheObjects = []client.Object{ + &spendsmartv1alpha1.ResourceRecommend{}, + } +) + +func init() { + utilruntime.Must(clientgoscheme.AddToScheme(scheme)) + + utilruntime.Must(spendsmartv1alpha1.AddToScheme(scheme)) + //+kubebuilder:scaffold:scheme +} + +type ResourceRecommender struct { + ctx context.Context + genericConf *generic.GenericConfiguration + opts *controller.ResourceRecommenderConfig +} + +func NewResourceRecommenderController(ctx context.Context, genericConf *generic.GenericConfiguration, opts *controller.ResourceRecommenderConfig) (*ResourceRecommender, error) { + return &ResourceRecommender{ + ctx: ctx, + genericConf: genericConf, + opts: opts, + }, nil +} + +func (r ResourceRecommender) Run() { + //klog.InitFlags(nil) + //// A flag to specify log verbosity level. + //err := flag.Set("v", r.opts.LogVerbosityLevel) + //if err != nil { + // panic("set log level err") + //} + //flag.Parse() + + config := ctrl.GetConfigOrDie() + config.QPS = r.genericConf.ClientConnection.QPS + config.Burst = int(r.genericConf.ClientConnection.Burst) + + ctrlOptions := ctrl.Options{ + Scheme: scheme, + MetricsBindAddress: ":" + r.opts.MetricsBindPort, + HealthProbeBindAddress: ":" + r.opts.HealthProbeBindPort, + Port: 9443, + LeaderElection: true, + LeaderElectionID: "controller-leader-election-recommend-controller", + LeaderElectionReleaseOnCancel: true, + LeaderElectionNamespace: "kube-system", + } + + mgr, err := ctrl.NewManager(config, ctrlOptions) + if err != nil { + klog.Fatal(fmt.Sprintf("unable to start manager: %s", err)) + os.Exit(1) + } + + if err = mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + klog.Fatal(fmt.Sprintf("unable to set up health check: %s", err)) + os.Exit(1) + } + if err = mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + klog.Fatal(fmt.Sprintf("unable to set up ready check: %s", err)) + os.Exit(1) + } + + dataProxy := initDataSources(r.opts) + klog.Infof("successfully init data proxy %v", *dataProxy) + + // Processor Manager + processorManager := processormanager.NewManager(dataProxy, mgr.GetClient()) + go func() { + defer func() { + if r := recover(); r != nil { + err = errors.Errorf("start processor panic: %v", r.(error)) + klog.Error(err) + panic(err) + } + }() + processorManager.StartProcess(r.ctx) + }() + + // OOM Recorder + podOOMRecorder := &PodOOMRecorderController{ + PodOOMRecorder: &oom.PodOOMRecorder{ + Client: mgr.GetClient(), + OOMRecordMaxNumber: r.opts.OOMRecordMaxNumber, + }, + } + if err = podOOMRecorder.SetupWithManager(mgr); err != nil { + klog.Fatal(fmt.Sprintf("Unable to create controller: %s", err)) + os.Exit(1) + } + go func() { + defer func() { + if r := recover(); r != nil { + err = errors.Errorf("Run oom recorder panic: %v", r.(error)) + klog.Error(err) + panic(err) + } + }() + for count := 0; count < 100; count++ { + cacheReady := mgr.GetCache().WaitForCacheSync(r.ctx) + if cacheReady { + break + } + time.Sleep(100 * time.Millisecond) + } + if err = podOOMRecorder.Run(r.ctx.Done()); err != nil { + klog.Warningf("Run oom recorder failed: %v", err) + } + }() + + recommenderManager := recommendermanager.NewManager(*processorManager, podOOMRecorder) + + for _, obj := range preCacheObjects { + _, _ = mgr.GetCache().GetInformer(context.TODO(), obj) + } + + // Resource Recommend Controller + resourceRecommendController := &ResourceRecommendController{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + ProcessorManager: processorManager, + RecommenderManager: recommenderManager, + } + + if err = resourceRecommendController.SetupWithManager(mgr); err != nil { + klog.Fatal(fmt.Sprintf("ResourceRecommend Controller unable to SetupWithManager, err: %s", err)) + os.Exit(1) + } + + //+kubebuilder:scaffold:builder + + setupLog.Info("starting manager") + if err = mgr.Start(r.ctx); err != nil { + klog.Fatal(fmt.Sprintf("problem running manager: %s", err)) + os.Exit(1) + } +} + +func initDataSources(opts *controller.ResourceRecommenderConfig) *datasource.Proxy { + dataProxy := datasource.NewProxy() + for _, datasourceProvider := range opts.DataSource { + switch datasourceProvider { + case string(datasource.PrometheusDatasource): + fallthrough + default: + // default is prom + prometheusProvider, err := prometheus.NewPrometheus(&opts.DataSourcePromConfig) + if err != nil { + klog.Exitf("unable to create datasource provider %v, err: %v", prometheusProvider, err) + panic(err) + } + dataProxy.RegisterDatasource(datasource.PrometheusDatasource, prometheusProvider) + } + } + return dataProxy +} diff --git a/pkg/controller/resource-recommend/controller/oom_recorder_controller.go b/pkg/controller/resource-recommend/controller/oom_recorder_controller.go new file mode 100644 index 0000000000..400cf9e7f2 --- /dev/null +++ b/pkg/controller/resource-recommend/controller/oom_recorder_controller.go @@ -0,0 +1,97 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/oom" +) + +// PodOOMRecorderController reconciles a PodOOMRecorder object +type PodOOMRecorderController struct { + *oom.PodOOMRecorder +} + +//+kubebuilder:rbac:groups=recommendation.katalyst.kubewharf.io,resources=podOOMRecorder,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=recommendation.katalyst.kubewharf.io,resources=podOOMRecorder/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=recommendation.katalyst.kubewharf.io,resources=podOOMRecorder/finalizers,verbs=update + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// reconcile takes an pod resource.It records information when a Pod experiences an OOM (Out of Memory) state. +// the ResourceRecommend object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by +// the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.2/pkg/reconcile +func (r *PodOOMRecorderController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + klog.V(5).InfoS("Get pods", "NamespacedName", req.NamespacedName) + pod := &v1.Pod{} + err := r.Client.Get(ctx, req.NamespacedName, pod) + if err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + for _, containerStatus := range pod.Status.ContainerStatuses { + if containerStatus.RestartCount > 0 && + containerStatus.LastTerminationState.Terminated != nil && + containerStatus.LastTerminationState.Terminated.Reason == "OOMKilled" { + if container := GetContainer(pod, containerStatus.Name); container != nil { + if memory, ok := container.Resources.Requests[v1.ResourceMemory]; ok { + r.Queue.Add(oom.OOMRecord{ + Namespace: pod.Namespace, + Pod: pod.Name, + Container: containerStatus.Name, + Memory: memory, + OOMAt: containerStatus.LastTerminationState.Terminated.FinishedAt.Time, + }) + klog.V(2).InfoS("Last termination state of the pod is oom", "namespace", pod.Namespace, + "pod", pod.Name, "container", containerStatus.Name, "MemoryRequest", memory) + } + } + } + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *PodOOMRecorderController) SetupWithManager(mgr ctrl.Manager) error { + r.Queue = workqueue.New() + return ctrl.NewControllerManagedBy(mgr). + WithOptions(controller.Options{ + RecoverPanic: true, + }). + For(&v1.Pod{}). + Complete(r) +} + +func GetContainer(pod *v1.Pod, containerName string) *v1.Container { + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == containerName { + return &pod.Spec.Containers[i] + } + } + return nil +} diff --git a/pkg/controller/resource-recommend/controller/resourcerecommend_controller.go b/pkg/controller/resource-recommend/controller/resourcerecommend_controller.go new file mode 100644 index 0000000000..0bbf01ef43 --- /dev/null +++ b/pkg/controller/resource-recommend/controller/resourcerecommend_controller.go @@ -0,0 +1,240 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "runtime/debug" + "time" + + "golang.org/x/time/rate" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/predicate" + + spendsmartv1alpha1 "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/env" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + processormanager "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/manager" + recommendermanager "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/recommender/manager" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/conditions" + customError "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/error" + recommendationtypes "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/recommendation" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/utils" +) + +const ( + ExponentialFailureRateLimiterBaseDelay = time.Minute + ExponentialFailureRateLimiterMaxDelay = 30 * time.Minute + DefaultRecommendInterval = 24 * time.Hour +) + +// ResourceRecommendController reconciles a ResourceRecommend object +type ResourceRecommendController struct { + client.Client + Scheme *runtime.Scheme + ProcessorManager *processormanager.Manager + RecommenderManager *recommendermanager.Manager +} + +//+kubebuilder:rbac:groups=recommendation.katalyst.kubewharf.io,resources=resourcerecommends,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=recommendation.katalyst.kubewharf.io,resources=resourcerecommends/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=recommendation.katalyst.kubewharf.io,resources=resourcerecommends/finalizers,verbs=update + +// SetupWithManager sets up the controller with the Manager. +func (r *ResourceRecommendController) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&spendsmartv1alpha1.ResourceRecommend{}). + // We will only focus on the event of the Spec update, filter update events for status and meta + WithEventFilter(predicate.GenerationChangedPredicate{}). + WithOptions(controller.Options{ + MaxConcurrentReconciles: 10, + RecoverPanic: true, + RateLimiter: workqueue.NewMaxOfRateLimiter( + // For reconcile failures(i.e. reconcile return err), the retry time is (2*minutes)*2^ + // The maximum retry time is 24 hours + workqueue.NewItemExponentialFailureRateLimiter(ExponentialFailureRateLimiterBaseDelay, ExponentialFailureRateLimiterMaxDelay), + // 10 qps, 100 bucket size. This is only for retry speed and its only the overall factor (not per item) + &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)}, + ), + }). + Complete(r) +} + +// Reconcile is part of the main kubernetes reconciliation loop which aims to +// move the current state of the cluster closer to the desired state. +// compare the state specified by the ResourceRecommend object against the actual cluster state, and then +// perform operations to make the cluster state reflect the state specified by the user. +// +// For more details, check Reconcile and its Result here: +// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.2/pkg/reconcile +func (r *ResourceRecommendController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + klog.InfoS("Get resourceRecommend to reconcile", "req", req) + resourceRecommend := &spendsmartv1alpha1.ResourceRecommend{} + err := r.Get(ctx, req.NamespacedName, resourceRecommend) + if err != nil { + if k8serrors.IsNotFound(err) { + // Object not found, return + klog.V(2).InfoS("ResourceRecommend has been deleted.", "req", req) + // CancelTasks err dno‘t need to be processed, because the Processor side has gc logic + _ = r.CancelTasks(req.NamespacedName) + return ctrl.Result{}, nil + } + return ctrl.Result{}, err + } + + if recommender := resourceRecommend.Spec.ResourcePolicy.AlgorithmPolicy.Recommender; recommender != "" && + recommender != recommendationtypes.DefaultRecommenderType { + klog.InfoS("ResourceRecommend is not controlled by the default controller", "req", req) + return ctrl.Result{}, nil + } + + if lastRecommendationTime := resourceRecommend.Status.LastRecommendationTime; lastRecommendationTime != nil { + requeueAfter := time.Until(lastRecommendationTime.Add(DefaultRecommendInterval)) + observedGeneration := resourceRecommend.Status.ObservedGeneration + if requeueAfter > time.Duration(0) && observedGeneration == resourceRecommend.GetGeneration() { + klog.InfoS("no spec change and not time to reconcile, skipping this reconcile", "requeueAfter", requeueAfter, "observedGeneration", observedGeneration, "generation", resourceRecommend.GetGeneration(), "resourceRecommendName", resourceRecommend.GetName()) + return ctrl.Result{RequeueAfter: requeueAfter}, nil + } + } + + err = r.doReconcile(ctx, req.NamespacedName, resourceRecommend) + if err != nil { + klog.ErrorS(err, "Failed to reconcile", "req", req) + return ctrl.Result{}, err + } + + klog.InfoS("reconcile succeeded, requeue after 24hours", "req", req) + //reconcile succeeded, requeue after 24hours + return ctrl.Result{RequeueAfter: DefaultRecommendInterval}, nil +} + +func (r *ResourceRecommendController) doReconcile(ctx context.Context, namespacedName k8stypes.NamespacedName, + resourceRecommend *spendsmartv1alpha1.ResourceRecommend) (err error) { + recommendation := recommendationtypes.NewRecommendation(resourceRecommend) + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("%v", r) + klog.ErrorS(err, "doReconcile panic", "resourceRecommend", namespacedName, "stack", string(debug.Stack())) + return + } + updateStatusErr := r.UpdateRecommendationStatus(namespacedName, recommendation) + if err == nil { + err = updateStatusErr + } + }() + + //conduct validation + validationError := recommendation.SetConfig(ctx, r.Client, resourceRecommend) + if validationError != nil { + klog.ErrorS(validationError, "Failed to get Recommendation", "resourceRecommend", namespacedName) + recommendation.Conditions.Set(*conditions.ConvertCustomErrorToCondition(*validationError)) + return validationError + } + //set the condition of the validation step to be true + recommendation.Conditions.Set(*conditions.ValidationSucceededCondition()) + + //Initialization + if registerTaskErr := r.RegisterTasks(*recommendation); registerTaskErr != nil { + klog.ErrorS(registerTaskErr, "Failed to register process task", "resourceRecommend", namespacedName) + recommendation.Conditions.Set(*conditions.ConvertCustomErrorToCondition(*registerTaskErr)) + return registerTaskErr + } + //set the condition of the initialization step to be true + recommendation.Conditions.Set(*conditions.InitializationSucceededCondition()) + + //recommendation logic + defaultRecommender := r.RecommenderManager.NewRecommender(recommendation.AlgorithmPolicy.Algorithm) + if recommendationError := defaultRecommender.Recommend(recommendation); recommendationError != nil { + klog.ErrorS(recommendationError, "error when getting recommendation for resource", "resourceRecommend", namespacedName) + recommendation.Conditions.Set(*conditions.ConvertCustomErrorToCondition(*recommendationError)) + return recommendationError + } + //set the condition of the recommendation step to be true + recommendation.Conditions.Set(*conditions.RecommendationReadyCondition()) + return nil +} + +// RegisterTasks Register all process task +func (r *ResourceRecommendController) RegisterTasks(recommendation recommendationtypes.Recommendation) *customError.CustomError { + processor := r.ProcessorManager.GetProcessor(recommendation.AlgorithmPolicy.Algorithm) + cluster := env.GetEnvWithDefault("VOLC_CLUSTER_ID", "") + klog.V(4).InfoS("Got cluster id from env", "cluster", cluster) + for _, container := range recommendation.Containers { + for _, containerConfig := range container.ContainerConfigs { + processConfig := &common.ProcessConfig{ + ProcessKey: common.ProcessKey{ + ResourceRecommendNamespacedName: k8stypes.NamespacedName{ + Name: recommendation.Name, + Namespace: recommendation.Namespace, + }, + Metric: &datasource.Metric{ + Namespace: recommendation.Namespace, + Kind: recommendation.TargetRef.Kind, + APIVersion: recommendation.TargetRef.APIVersion, + WorkloadName: recommendation.TargetRef.Name, + ContainerName: container.ContainerName, + Resource: containerConfig.ControlledResource, + }, + }, + } + if len(cluster) != 0 { + selectors := make(map[string]string) + selectors["cluster"] = cluster + processConfig.SetSelector(selectors) + } + if err := processor.Register(processConfig); err != nil { + return customError.DataProcessRegisteredFailedError(err.Error()) + } + } + } + + return nil +} + +// CancelTasks Cancel all process task +func (r *ResourceRecommendController) CancelTasks(namespacedName k8stypes.NamespacedName) *customError.CustomError { + processor := r.ProcessorManager.GetProcessor(spendsmartv1alpha1.AlgorithmPercentile) + err := processor.Cancel(&common.ProcessKey{ResourceRecommendNamespacedName: namespacedName}) + if err != nil { + klog.ErrorS(err, "cancel processor task failed", "namespacedName", namespacedName) + } + return err +} + +func (r *ResourceRecommendController) UpdateRecommendationStatus(namespaceName k8stypes.NamespacedName, + recommendation *recommendationtypes.Recommendation) error { + updateStatus := &spendsmartv1alpha1.ResourceRecommend{ + Status: recommendation.AsStatus(), + } + //record generation + updateStatus.Status.ObservedGeneration = recommendation.ObservedGeneration + + err := utils.PatchUpdateResourceRecommend(r.Client, namespaceName, updateStatus) + if err != nil { + klog.ErrorS(err, "Update resourceRecommend status error") + } + return err +} diff --git a/pkg/controller/resource-recommend/datasource/datasource_proxy.go b/pkg/controller/resource-recommend/datasource/datasource_proxy.go new file mode 100644 index 0000000000..9d35e22aff --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/datasource_proxy.go @@ -0,0 +1,66 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package datasource + +import ( + "errors" + "time" +) + +type DatasourceType string + +const ( + PrometheusDatasource DatasourceType = "Prometheus" +) + +type Datasource interface { + QueryTimeSeries(query *Query, start time.Time, end time.Time, step time.Duration) (*TimeSeries, error) + ConvertMetricToQuery(metric Metric) (*Query, error) +} + +type Proxy struct { + datasourceMap map[DatasourceType]Datasource +} + +func NewProxy() *Proxy { + return &Proxy{ + datasourceMap: make(map[DatasourceType]Datasource), + } +} + +func (p *Proxy) RegisterDatasource(name DatasourceType, datasource Datasource) { + p.datasourceMap[name] = datasource +} + +func (p *Proxy) getDatasource(name DatasourceType) (Datasource, error) { + if datasource, ok := p.datasourceMap[name]; ok { + return datasource, nil + } + return nil, errors.New("datasource not found") +} + +func (p *Proxy) QueryTimeSeries(DatasourceName DatasourceType, metric Metric, start time.Time, end time.Time, step time.Duration) (*TimeSeries, error) { + datasource, err := p.getDatasource(DatasourceName) + if err != nil { + return nil, err + } + query, err := datasource.ConvertMetricToQuery(metric) + if err != nil { + return nil, err + } + return datasource.QueryTimeSeries(query, start, end, step) +} diff --git a/pkg/controller/resource-recommend/datasource/datasource_proxy_test.go b/pkg/controller/resource-recommend/datasource/datasource_proxy_test.go new file mode 100644 index 0000000000..cf54204be8 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/datasource_proxy_test.go @@ -0,0 +1,107 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package datasource + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +type MockDatasource struct { + mock.Mock +} + +func (m *MockDatasource) ConvertMetricToQuery(metric Metric) (*Query, error) { + args := m.Called(metric) + return args.Get(0).(*Query), args.Error(1) +} + +func (m *MockDatasource) QueryTimeSeries(query *Query, start, end time.Time, step time.Duration) (*TimeSeries, error) { + args := m.Called(query, start, end, step) + return args.Get(0).(*TimeSeries), args.Error(1) +} + +func TestProxy_QueryTimeSeries(t *testing.T) { + // Create mock objects + mockDatasource := &MockDatasource{} + mockTimeSeries := &TimeSeries{} + + // Create Proxy with mock datasource + proxy := &Proxy{ + datasourceMap: map[DatasourceType]Datasource{ + DatasourceType("mock"): mockDatasource, + }, + } + + // Define test cases + testCases := []struct { + name string + datasource DatasourceType + metric Metric + start time.Time + end time.Time + step time.Duration + expectedTS *TimeSeries + expectedErr error + }{ + { + name: "Valid query", + datasource: "mock", + metric: Metric{}, + start: time.Now(), + end: time.Now(), + step: time.Second, + expectedTS: mockTimeSeries, + }, + { + name: "Error getting datasource", + datasource: "invalid", + metric: Metric{}, + start: time.Now(), + end: time.Now(), + step: time.Second, + expectedErr: errors.New("datasource not found"), + }, + } + + // Set up mock behaviors and perform tests + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Set up mock behaviors + mockDatasource.On("ConvertMetricToQuery", tc.metric).Return(&Query{}, nil) + if tc.expectedErr == nil { + mockDatasource.On("QueryTimeSeries", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(mockTimeSeries, nil) + } else { + mockDatasource.On("QueryTimeSeries", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, tc.expectedErr) + } + + // Call the QueryTimeSeries method + result, err := proxy.QueryTimeSeries(tc.datasource, tc.metric, tc.start, tc.end, tc.step) + + // Verify the results + assert.Equal(t, tc.expectedTS, result) + assert.Equal(t, tc.expectedErr, err) + + // Verify mock invocations + mockDatasource.AssertExpectations(t) + }) + } +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/client.go b/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/client.go new file mode 100644 index 0000000000..11ea214448 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/client.go @@ -0,0 +1,73 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volcengine + +import ( + "net/http" + + "github.com/volcengine/volc-sdk-golang/base" + + credential "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/creds" + ecsroleprovider "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/creds/ecs_role_provider" +) + +type volcRoundTripper struct { + next http.RoundTripper + + volcCred credential.SignerProvider +} + +// NewRoundTripper returns a new http.RoundTripper that will sign requests +// using Volcengine's Signature Verification signing procedure. The request will +// then be handed off to the next RoundTripper provided by next. If next is nil, +// http.DefaultTransport will be used. +// +// Credentials for signing are retrieved using the the default volcengine credential +// chain. If credentials cannot be found, an error will be returned. +func NewRoundTripper(cfg *Config, next http.RoundTripper) (http.RoundTripper, error) { + if next == nil { + next = http.DefaultTransport + } + + var cred credential.SignerProvider + var err error + if len(cfg.AccessKey) != 0 && len(cfg.SecretKey) != 0 { + cred = &credential.StaticCred{ + Credentials: base.Credentials{ + AccessKeyID: cfg.AccessKey, + SecretAccessKey: string(cfg.SecretKey), + Service: cfg.Service, + Region: cfg.Region, + }, + } + } else { + cred, err = credential.New(ecsroleprovider.New(cfg.VolcECSRoleName), cfg.Service, cfg.Region) + if err != nil { + return next, err + } + } + rt := &volcRoundTripper{ + next: next, + volcCred: cred, + } + + return rt, nil +} + +func (rt *volcRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return rt.next.RoundTrip(rt.volcCred.Get().Sign(req)) +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/config.go b/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/config.go new file mode 100644 index 0000000000..52c1f0702b --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/config.go @@ -0,0 +1,55 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volcengine + +import ( + "fmt" + + "github.com/prometheus/common/config" +) + +// Config is the configuration for signing remote write requests with +// volcengine verification process. Empty values will be retrieved using the +// volcengine default credentials chain. +type Config struct { + Region string `yaml:"region,omitempty"` + AccessKey string `yaml:"access_key,omitempty"` + SecretKey config.Secret `yaml:"secret_key,omitempty"` + Service string `yaml:"service,omitempty"` + VolcECSRoleName string `yaml:"volc_ecs_role_name,omitempty"` +} + +func (c *Config) Validate() error { + if (c.AccessKey == "") != (c.SecretKey == "") { + return fmt.Errorf("must provide a volcengine Access key and Secret Key if credentials are specified in the volcengine config") + } + + if len(c.Service) == 0 { + c.Service = "vmp" + } + + return nil +} + +func (c *Config) UnmarshalYAML(unmarshal func(interface{}) error) error { + type plain Config + *c = Config{} + if err := unmarshal((*plain)(c)); err != nil { + return err + } + return c.Validate() +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/creds/credentials.go b/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/creds/credentials.go new file mode 100644 index 0000000000..b0558eb0f2 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/creds/credentials.go @@ -0,0 +1,111 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package credential + +import ( + "net/http" + "sync" + "time" + + "github.com/volcengine/volc-sdk-golang/base" +) + +const ( + expiredWindow = time.Minute * 10 // refresh token if it is to be expired within expiredWindow +) + +type Credential struct { + base.Credentials + expiredTime time.Time +} + +type SignerProvider interface { + Get() Signer +} + +type Signer interface { + Sign(request *http.Request) *http.Request +} + +type CredProvider interface { + Retrieve() (Value, error) +} + +type Value struct { + AccessKeyID string + SecretAccessKey string + SessionToken string + ExpiredTime time.Time +} + +type credentialCache struct { + sync.RWMutex + credential Credential + provider CredProvider +} + +func New(provider CredProvider, service, region string) (SignerProvider, error) { + v, err := provider.Retrieve() + if err != nil { + return nil, err + } + return &credentialCache{ + provider: provider, + credential: Credential{ + Credentials: base.Credentials{ + AccessKeyID: v.AccessKeyID, + SecretAccessKey: v.SecretAccessKey, + Service: service, + Region: region, + SessionToken: v.SessionToken, + }, + expiredTime: v.ExpiredTime, + }, + }, nil +} + +func (c *credentialCache) Get() Signer { + c.RLock() + refresh := time.Until(c.credential.expiredTime) < expiredWindow + c.RUnlock() + if refresh { + v, err := c.provider.Retrieve() + if err == nil { + c.setCredential(v) + } + } + c.RLock() + defer c.RUnlock() + return &c.credential +} + +func (c *credentialCache) setCredential(v Value) { + c.Lock() + c.credential.Credentials.AccessKeyID = v.AccessKeyID + c.credential.Credentials.SecretAccessKey = v.SecretAccessKey + c.credential.Credentials.SessionToken = v.SessionToken + c.credential.expiredTime = v.ExpiredTime + c.Unlock() +} + +type StaticCred struct { + base.Credentials +} + +func (c *StaticCred) Get() Signer { + return c +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/creds/ecs_role_provider/ecs_role_provider.go b/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/creds/ecs_role_provider/ecs_role_provider.go new file mode 100644 index 0000000000..1c7ea27929 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/creds/ecs_role_provider/ecs_role_provider.go @@ -0,0 +1,90 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package ecsroleprovider + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "time" + + "k8s.io/klog/v2" + + credential "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/creds" +) + +const ( + DefalutTimeout = time.Second * 10 +) + +type ecsRoleProvider struct { + roleName string +} + +func New(roleName string) *ecsRoleProvider { + return &ecsRoleProvider{roleName: roleName} +} + +func (provider *ecsRoleProvider) Retrieve() (credential.Value, error) { + ctx, cancel := context.WithTimeout(context.Background(), DefalutTimeout) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf( + "http://100.96.0.96/volcstack/latest/iam/security_credentials/%s", provider.roleName, + ), nil) + if err != nil { + return credential.Value{}, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + klog.Errorf("retrieve ecs role %s failed: %w", provider.roleName, err) + return credential.Value{}, err + } + defer resp.Body.Close() + + data, err := io.ReadAll(resp.Body) + if err != nil { + return credential.Value{}, err + } + if resp.StatusCode != http.StatusOK { + err = errors.New(string(data)) + klog.Error("retrieve ecs role %s failed with status code %d: %w", provider.roleName, resp.Status, err) + return credential.Value{}, err + } + var respCred credentials + if err = json.Unmarshal(data, &respCred); err != nil { + err = fmt.Errorf("cannot unmarshal data: %s into credentials; error: %s", string(data), err) + klog.Error("retrieve ecs role %s failed: %w", err) + return credential.Value{}, err + } + return credential.Value{ + AccessKeyID: respCred.AccessKeyId, + SecretAccessKey: respCred.SecretAccessKey, + SessionToken: respCred.SessionToken, + ExpiredTime: respCred.ExpiredTime, + }, nil +} + +type credentials struct { + ExpiredTime time.Time + CurrentTime time.Time + AccessKeyId string + SecretAccessKey string + SessionToken string +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/env/env.go b/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/env/env.go new file mode 100644 index 0000000000..e874020f67 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/env/env.go @@ -0,0 +1,36 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package env + +import "os" + +// adapter start env +var ( + Region = GetEnvWithDefault("VOLC_VMP_REGION", "") + VolcAccessKey = GetEnvWithDefault("VOLC_VMP_ACCESS_KEY", "") // service ak + VolcSecretKey = GetEnvWithDefault("VOLC_VMP_SECRET_KEY", "") // service sk + VolcService = GetEnvWithDefault("VOLC_VMP_SERVICE", "vmp") + VolcRoleName = GetEnvWithDefault("VOLC_VMP_ASSUME_ROLE", "") +) + +// GetEnvWithDefault ... +func GetEnvWithDefault(key, fallback string) string { + if value, ok := os.LookupEnv(key); ok { + return value + } + return fallback +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/mock_prometheus_client.go b/pkg/controller/resource-recommend/datasource/prometheus/mock_prometheus_client.go new file mode 100644 index 0000000000..1387f111f6 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/mock_prometheus_client.go @@ -0,0 +1,114 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "context" + "time" + + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" +) + +// mockPromAPIClient is a mock implementation of v1.API for testing purposes. +type mockPromAPIClient struct { + QueryRangeFunc func(ctx context.Context, query string, r v1.Range) (model.Value, v1.Warnings, error) +} + +func (m *mockPromAPIClient) Alerts(ctx context.Context) (v1.AlertsResult, error) { + return v1.AlertsResult{}, nil +} + +func (m *mockPromAPIClient) AlertManagers(ctx context.Context) (v1.AlertManagersResult, error) { + return v1.AlertManagersResult{}, nil +} + +func (m *mockPromAPIClient) CleanTombstones(ctx context.Context) error { + return nil +} + +func (m *mockPromAPIClient) Config(ctx context.Context) (v1.ConfigResult, error) { + return v1.ConfigResult{}, nil +} + +func (m *mockPromAPIClient) DeleteSeries(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) error { + return nil +} + +func (m *mockPromAPIClient) Flags(ctx context.Context) (v1.FlagsResult, error) { + return nil, nil +} + +func (m *mockPromAPIClient) LabelNames(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]string, v1.Warnings, error) { + return nil, nil, nil +} + +func (m *mockPromAPIClient) LabelValues(ctx context.Context, label string, matches []string, startTime time.Time, endTime time.Time) (model.LabelValues, v1.Warnings, error) { + return nil, nil, nil +} + +func (m *mockPromAPIClient) Query(ctx context.Context, query string, ts time.Time, opts ...v1.Option) (model.Value, v1.Warnings, error) { + return nil, nil, nil +} + +func (m *mockPromAPIClient) QueryExemplars(ctx context.Context, query string, startTime time.Time, endTime time.Time) ([]v1.ExemplarQueryResult, error) { + return nil, nil +} + +func (m *mockPromAPIClient) Buildinfo(ctx context.Context) (v1.BuildinfoResult, error) { + return v1.BuildinfoResult{}, nil +} + +func (m *mockPromAPIClient) Runtimeinfo(ctx context.Context) (v1.RuntimeinfoResult, error) { + return v1.RuntimeinfoResult{}, nil +} + +func (m *mockPromAPIClient) Series(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]model.LabelSet, v1.Warnings, error) { + return nil, nil, nil +} + +func (m *mockPromAPIClient) Snapshot(ctx context.Context, skipHead bool) (v1.SnapshotResult, error) { + return v1.SnapshotResult{}, nil +} + +func (m *mockPromAPIClient) Rules(ctx context.Context) (v1.RulesResult, error) { + return v1.RulesResult{}, nil +} + +func (m *mockPromAPIClient) Targets(ctx context.Context) (v1.TargetsResult, error) { + return v1.TargetsResult{}, nil +} + +func (m *mockPromAPIClient) TargetsMetadata(ctx context.Context, matchTarget string, metric string, limit string) ([]v1.MetricMetadata, error) { + return nil, nil +} + +func (m *mockPromAPIClient) Metadata(ctx context.Context, metric string, limit string) (map[string][]v1.Metadata, error) { + return nil, nil +} + +func (m *mockPromAPIClient) TSDB(ctx context.Context) (v1.TSDBResult, error) { + return v1.TSDBResult{}, nil +} + +func (m *mockPromAPIClient) WalReplay(ctx context.Context) (v1.WalReplayStatus, error) { + return v1.WalReplayStatus{}, nil +} + +func (m *mockPromAPIClient) QueryRange(ctx context.Context, query string, r v1.Range, opts ...v1.Option) (model.Value, v1.Warnings, error) { + return m.QueryRangeFunc(ctx, query, r) +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/prometheus_auth_provider.go b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_auth_provider.go new file mode 100644 index 0000000000..94d2c2bfd0 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_auth_provider.go @@ -0,0 +1,65 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "fmt" + "net/http" + + "github.com/prometheus/common/config" + "k8s.io/klog/v2" + + volcengine "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/env" +) + +type authRoundTripperProvider struct { + next http.RoundTripper +} + +func NewAuthRoundTripperProvider(next http.RoundTripper) *authRoundTripperProvider { + return &authRoundTripperProvider{ + next: next, + } +} + +func (p *authRoundTripperProvider) GetAuthRoundTripper(c *ClientAuth) (http.RoundTripper, error) { + switch AuthType(c.Type) { + case PrometheusBasicAuth: + klog.Info("prom auth using basic auth") + if len(c.Username) == 0 || len(c.Password) == 0 { + return nil, fmt.Errorf("invalid user name or password for basic auth") + } + return config.NewBasicAuthRoundTripper(c.Username, config.Secret(c.Password), "", p.next), nil + case PrometheusBearerTokenAuth: + klog.Info("prom auth using bearer token") + if len(c.BearerToken) == 0 { + return nil, fmt.Errorf("invalid bearer token for bearer token auth") + } + return config.NewAuthorizationCredentialsRoundTripper("Bearer", config.Secret(c.BearerToken), p.next), nil + case PrometheusVolcEngineAuth: + klog.Info("prom auth using volcEngine") + return volcengine.NewRoundTripper(&volcengine.Config{ + Region: env.Region, + AccessKey: env.VolcAccessKey, + SecretKey: config.Secret(env.VolcSecretKey), + Service: env.VolcService, + VolcECSRoleName: env.VolcRoleName, + }, p.next) + } + return nil, fmt.Errorf("unsupported auth type %v", c.Type) +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/prometheus_client.go b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_client.go new file mode 100644 index 0000000000..0cc45248d0 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_client.go @@ -0,0 +1,114 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "context" + "crypto/tls" + "net" + "net/http" + "net/url" + "time" + + prometheusapi "github.com/prometheus/client_golang/api" +) + +type AuthType string + +const ( + PrometheusBasicAuth AuthType = "Basic" + PrometheusBearerTokenAuth AuthType = "BearerToken" + PrometheusVolcEngineAuth AuthType = "VolcEngine" +) + +// PromConfig represents the config of prometheus +type PromConfig struct { + Address string + Timeout time.Duration + KeepAlive time.Duration + InsecureSkipVerify bool + Auth ClientAuth + + QueryConcurrency int + BRateLimit bool + MaxPointsLimitPerTimeSeries int + TLSHandshakeTimeoutInSecond time.Duration +} + +// ClientAuth holds the HTTP client identity info. +type ClientAuth struct { + Type string + Username string + BearerToken string + Password string +} + +type prometheusAuthClient struct { + client prometheusapi.Client +} + +// URL implements prometheus client interface +func (p *prometheusAuthClient) URL(ep string, args map[string]string) *url.URL { + return p.client.URL(ep, args) +} + +// Do implements prometheus client interface, wrapped with an auth info +func (p *prometheusAuthClient) Do(ctx context.Context, request *http.Request) (*http.Response, []byte, error) { + return p.client.Do(ctx, request) +} + +// NewPrometheusClient returns a prometheus.Client +func NewPrometheusClient(config *PromConfig) (prometheusapi.Client, error) { + + tlsConfig := &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify} + + var rt http.RoundTripper = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: config.Timeout, + KeepAlive: config.KeepAlive, + }).DialContext, + TLSHandshakeTimeout: config.TLSHandshakeTimeoutInSecond, + TLSClientConfig: tlsConfig, + } + + authProvider := NewAuthRoundTripperProvider(rt) + t, err := authProvider.GetAuthRoundTripper(&config.Auth) + if err != nil { + return nil, err + } + + pc := prometheusapi.Config{ + Address: config.Address, + RoundTripper: t, + } + return newPrometheusAuthClient(pc) + +} + +func newPrometheusAuthClient(config prometheusapi.Config) (prometheusapi.Client, error) { + c, err := prometheusapi.NewClient(config) + if err != nil { + return nil, err + } + + client := &prometheusAuthClient{ + client: c, + } + + return client, nil +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/prometheus_provider.go b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_provider.go new file mode 100644 index 0000000000..448aceb610 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_provider.go @@ -0,0 +1,148 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "context" + "fmt" + "time" + + promapiv1 "github.com/prometheus/client_golang/api/prometheus/v1" + datamodel "github.com/prometheus/common/model" + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" +) + +type prometheus struct { + promAPIClient promapiv1.API + config *PromConfig +} + +type PromDatasource interface { + datasource.Datasource + GetPromClient() promapiv1.API +} + +// NewPrometheus return a prometheus data source +func NewPrometheus(config *PromConfig) (PromDatasource, error) { + + client, err := NewPrometheusClient(config) + if err != nil { + return nil, err + } + promAPIClient := promapiv1.NewAPI(client) + + return &prometheus{promAPIClient: promAPIClient, config: config}, nil +} + +func (p *prometheus) GetPromClient() promapiv1.API { + return p.promAPIClient +} + +func (p *prometheus) ConvertMetricToQuery(metric datasource.Metric) (*datasource.Query, error) { + var queryExpr string + switch metric.Resource { + case v1.ResourceCPU: + queryExpr = GetContainerCpuUsageQueryExp(metric.Namespace, metric.WorkloadName, metric.Kind, metric.ContainerName, metric.Selectors) + case v1.ResourceMemory: + queryExpr = GetContainerMemUsageQueryExp(metric.Namespace, metric.WorkloadName, metric.Kind, metric.ContainerName, metric.Selectors) + default: + return nil, fmt.Errorf("query for resource type %v is not supported", metric.Resource) + } + convertedQuery := datasource.PrometheusQuery{ + Query: queryExpr, + } + return &datasource.Query{ + Prometheus: &convertedQuery, + }, nil +} + +func (p *prometheus) QueryTimeSeries(query *datasource.Query, start time.Time, end time.Time, step time.Duration) (*datasource.TimeSeries, error) { + klog.InfoS("QueryTimeSeries", "query", query, "start", start, "end", end) + timeoutCtx, cancelFunc := context.WithTimeout(context.Background(), p.config.Timeout) + defer cancelFunc() + timeSeries, err := p.queryRangeSync(timeoutCtx, query.Prometheus.Query, start, end, step) + if err != nil { + klog.ErrorS(err, "query", query, "start", start, "end", end) + return nil, err + } + return timeSeries, nil +} + +// queryRangeSync range query prometheus in sync way +func (p *prometheus) queryRangeSync(ctx context.Context, query string, start, end time.Time, step time.Duration) (*datasource.TimeSeries, error) { + r := promapiv1.Range{ + Start: start, + End: end, + Step: step, + } + klog.InfoS("Prom query", "query", query) + var ts *datasource.TimeSeries + results, warnings, err := p.promAPIClient.QueryRange(ctx, query, r) + if len(warnings) != 0 { + klog.InfoS("Prom query range warnings", "warnings", warnings) + } + if err != nil { + return ts, err + } + klog.V(5).InfoS("Prom query range result", "query", query, "result", results.String(), "resultsType", results.Type()) + + return p.convertPromResultsToTimeSeries(results) +} + +func (p *prometheus) convertPromResultsToTimeSeries(value datamodel.Value) (*datasource.TimeSeries, error) { + results := datasource.NewTimeSeries() + typeValue := value.Type() + switch typeValue { + case datamodel.ValMatrix: + if matrix, ok := value.(datamodel.Matrix); ok { + for _, sampleStream := range matrix { + if sampleStream == nil { + continue + } + for key, val := range sampleStream.Metric { + results.AppendLabel(string(key), string(val)) + } + for _, pair := range sampleStream.Values { + results.AppendSample(int64(pair.Timestamp/1000), float64(pair.Value)) + } + } + return results, nil + } else { + return results, fmt.Errorf("prometheus value type is %v, but assert failed", typeValue) + } + + case datamodel.ValVector: + if vector, ok := value.(datamodel.Vector); ok { + for _, sample := range vector { + if sample == nil { + continue + } + for key, val := range sample.Metric { + results.AppendLabel(string(key), string(val)) + } + results.AppendSample(int64(sample.Timestamp/1000), float64(sample.Value)) + } + return results, nil + } else { + return results, fmt.Errorf("prometheus value type is %v, but assert failed", typeValue) + } + } + return results, fmt.Errorf("prometheus return unsupported model value type %v", typeValue) +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/prometheus_provider_test.go b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_provider_test.go new file mode 100644 index 0000000000..20ec38b700 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_provider_test.go @@ -0,0 +1,453 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "context" + "fmt" + "reflect" + "testing" + "time" + + promapiv1 "github.com/prometheus/client_golang/api/prometheus/v1" + v1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" + corev1 "k8s.io/api/core/v1" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" +) + +func Test_prometheus_ConvertMetricToQuery(t *testing.T) { + p := &prometheus{} + tests := []struct { + name string + metric datasource.Metric + expectedQuery *datasource.Query + expectedError error + }{ + { + name: "CPU metric", + metric: datasource.Metric{ + Resource: corev1.ResourceCPU, + Namespace: "my-namespace", + WorkloadName: "my-workload", + Kind: "Deployment", + ContainerName: "my-container", + }, + expectedQuery: &datasource.Query{ + Prometheus: &datasource.PrometheusQuery{ + Query: GetContainerCpuUsageQueryExp("my-namespace", "my-workload", "Deployment", "my-container", ""), + }, + }, + expectedError: nil, + }, + { + name: "Memory metric", + metric: datasource.Metric{ + Resource: corev1.ResourceMemory, + Namespace: "my-namespace", + WorkloadName: "my-workload", + Kind: "Deployment", + ContainerName: "my-container", + }, + expectedQuery: &datasource.Query{ + Prometheus: &datasource.PrometheusQuery{ + Query: GetContainerMemUsageQueryExp("my-namespace", "my-workload", "Deployment", "my-container", ""), + }, + }, + expectedError: nil, + }, + { + name: "Unsupported metric", + metric: datasource.Metric{ + Resource: "Unsupported", + Namespace: "my-namespace", + WorkloadName: "my-workload", + Kind: "Deployment", + ContainerName: "my-container", + }, + expectedQuery: nil, + expectedError: fmt.Errorf("query for resource type Unsupported is not supported"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotQuery, gotErr := p.ConvertMetricToQuery(tt.metric) + + if !reflect.DeepEqual(gotQuery, tt.expectedQuery) { + t.Errorf("ConvertMetricToQuery() gotQuery = %v, want %v", gotQuery, tt.expectedQuery) + } + + if (gotErr == nil && tt.expectedError != nil) || (gotErr != nil && tt.expectedError == nil) || (gotErr != nil && tt.expectedError != nil && gotErr.Error() != tt.expectedError.Error()) { + t.Errorf("ConvertMetricToQuery() gotErr = %v, wantErr %v", gotErr, tt.expectedError) + } + }) + } +} + +func Test_prometheus_QueryTimeSeries(t *testing.T) { + type fields struct { + promAPIClient v1.API + config *PromConfig + } + type args struct { + query *datasource.Query + start time.Time + end time.Time + step time.Duration + } + tests := []struct { + name string + fields fields + args args + want *datasource.TimeSeries + wantErr bool + }{ + { + name: "Successful query time series", + fields: fields{ + promAPIClient: &mockPromAPIClient{ + QueryRangeFunc: func(ctx context.Context, query string, r promapiv1.Range) (model.Value, promapiv1.Warnings, error) { + matrix := model.Matrix{ + &model.SampleStream{ + Metric: model.Metric{ + "label1": "value1", + "label2": "value2", + }, + Values: []model.SamplePair{ + { + Timestamp: 1500000000, + Value: 10, + }, + { + Timestamp: 1500001000, + Value: 20, + }, + }, + }, + } + return matrix, nil, nil + }, + }, + config: &PromConfig{ + Timeout: time.Second * 5, + }, + }, + args: args{ + query: &datasource.Query{ + Prometheus: &datasource.PrometheusQuery{ + Query: "my_query", + }, + }, + start: time.Unix(1500000000, 0), + end: time.Unix(1500002000, 0), + step: time.Second, + }, + want: &datasource.TimeSeries{ + Samples: []datasource.Sample{ + { + Value: 10, + Timestamp: 1500000, + }, + { + Value: 20, + Timestamp: 1500001, + }, + }, + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + }, + wantErr: false, + }, + { + name: "Query time series error", + fields: fields{ + promAPIClient: &mockPromAPIClient{ + QueryRangeFunc: func(ctx context.Context, query string, r promapiv1.Range) (model.Value, promapiv1.Warnings, error) { + return nil, nil, fmt.Errorf("query error") + }, + }, + config: &PromConfig{ + Timeout: time.Second * 5, + }, + }, + args: args{ + query: &datasource.Query{ + Prometheus: &datasource.PrometheusQuery{ + Query: "my_query", + }, + }, + start: time.Unix(1500000000, 0), + end: time.Unix(1500002000, 0), + step: time.Second, + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &prometheus{ + promAPIClient: tt.fields.promAPIClient, + config: tt.fields.config, + } + got, err := p.QueryTimeSeries(tt.args.query, tt.args.start, tt.args.end, tt.args.step) + if (err != nil) != tt.wantErr { + t.Errorf("QueryTimeSeries() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("QueryTimeSeries() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_prometheus_convertPromResultsToTimeSeries(t *testing.T) { + type fields struct { + promAPIClient v1.API + config *PromConfig + } + type args struct { + value model.Value + } + tests := []struct { + name string + fields fields + args args + want *datasource.TimeSeries + wantErr bool + }{ + { + name: "Matrix value", + fields: fields{ + promAPIClient: nil, + config: nil, + }, + args: args{ + value: model.Matrix{ + &model.SampleStream{ + Metric: model.Metric{ + "label1": "value1", + "label2": "value2", + }, + Values: []model.SamplePair{ + { + Timestamp: 1500000000, + Value: 10, + }, + { + Timestamp: 1500001000, + Value: 20, + }, + }, + }, + }, + }, + want: &datasource.TimeSeries{ + Samples: []datasource.Sample{ + { + Value: 10, + Timestamp: 1500000, + }, + { + Value: 20, + Timestamp: 1500001, + }, + }, + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + }, + wantErr: false, + }, + { + name: "Vector value", + fields: fields{ + promAPIClient: nil, + config: nil, + }, + args: args{ + value: model.Vector{ + &model.Sample{ + Metric: model.Metric{ + "label1": "value1", + "label2": "value2", + }, + Timestamp: 1500000000, + Value: 10, + }, + }, + }, + want: &datasource.TimeSeries{ + Samples: []datasource.Sample{ + { + Value: 10, + Timestamp: 1500000, + }, + }, + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + }, + wantErr: false, + }, + { + name: "Unsupported value type", + fields: fields{ + promAPIClient: nil, + config: nil, + }, + args: args{ + value: &model.Scalar{}, + }, + want: datasource.NewTimeSeries(), + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &prometheus{ + promAPIClient: tt.fields.promAPIClient, + config: tt.fields.config, + } + got, err := p.convertPromResultsToTimeSeries(tt.args.value) + if (err != nil) != tt.wantErr { + t.Errorf("convertPromResultsToTimeSeries() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("convertPromResultsToTimeSeries() got = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_prometheus_queryRangeSync(t *testing.T) { + type fields struct { + promAPIClient v1.API + config *PromConfig + } + type args struct { + ctx context.Context + query string + start time.Time + end time.Time + step time.Duration + } + tests := []struct { + name string + fields fields + args args + want *datasource.TimeSeries + wantErr bool + }{ + { + name: "Successful query range", + fields: fields{ + promAPIClient: &mockPromAPIClient{ + QueryRangeFunc: func(ctx context.Context, query string, r promapiv1.Range) (model.Value, promapiv1.Warnings, error) { + matrix := model.Matrix{ + &model.SampleStream{ + Metric: model.Metric{ + "label1": "value1", + "label2": "value2", + }, + Values: []model.SamplePair{ + { + Timestamp: 1500000000, + Value: 10, + }, + { + Timestamp: 1500001000, + Value: 20, + }, + }, + }, + } + return matrix, nil, nil + }, + }, + config: nil, + }, + args: args{ + ctx: context.TODO(), + query: "my_query", + start: time.Unix(1500000000, 0), + end: time.Unix(1500002000, 0), + step: time.Second, + }, + want: &datasource.TimeSeries{ + Samples: []datasource.Sample{ + { + Value: 10, + Timestamp: 1500000, + }, + { + Value: 20, + Timestamp: 1500001, + }, + }, + Labels: map[string]string{ + "label1": "value1", + "label2": "value2", + }, + }, + wantErr: false, + }, + { + name: "Error in query range", + fields: fields{ + promAPIClient: &mockPromAPIClient{ + QueryRangeFunc: func(ctx context.Context, query string, r promapiv1.Range) (model.Value, promapiv1.Warnings, error) { + return nil, nil, fmt.Errorf("query error") + }, + }, + config: nil, + }, + args: args{ + ctx: context.TODO(), + query: "my_query", + start: time.Unix(1500000000, 0), + end: time.Unix(1500002000, 0), + step: time.Second, + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &prometheus{ + promAPIClient: tt.fields.promAPIClient, + config: tt.fields.config, + } + got, err := p.queryRangeSync(tt.args.ctx, tt.args.query, tt.args.start, tt.args.end, tt.args.step) + if (err != nil) != tt.wantErr { + t.Errorf("queryRangeSync() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("queryRangeSync() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/prometheus_query.go b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_query.go new file mode 100644 index 0000000000..002421518a --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_query.go @@ -0,0 +1,58 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "fmt" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" +) + +const ( + // ContainerCpuUsageQueryExpr is used to query container cpu usage by promql + ContainerCpuUsageQueryExpr = `rate(container_cpu_usage_seconds_total{container!="POD",namespace="%s",pod=~"%s",container="%s"%s}[30s])` + // ContainerMemUsageQueryExpr is used to query container cpu usage by promql + ContainerMemUsageQueryExpr = `container_memory_working_set_bytes{container!="POD",namespace="%s",pod=~"%s",container="%s"%s}` +) + +const ( + WorkloadSuffixRuleForDeployment = `[a-z0-9]+-[a-z0-9]{5}$` +) + +func GetContainerCpuUsageQueryExp(namespace string, workloadName string, kind string, containerName string, extraFilters string) string { + if len(extraFilters) != 0 { + extraLabels := fmt.Sprintf(",%s", extraFilters) + return fmt.Sprintf(ContainerCpuUsageQueryExpr, namespace, convertWorkloadNameToPods(workloadName, kind), containerName, extraLabels) + } + return fmt.Sprintf(ContainerCpuUsageQueryExpr, namespace, convertWorkloadNameToPods(workloadName, kind), containerName, extraFilters) +} + +func GetContainerMemUsageQueryExp(namespace string, workloadName string, kind string, containerName string, extraFilters string) string { + if len(extraFilters) != 0 { + extraLabels := fmt.Sprintf(",%s", extraFilters) + return fmt.Sprintf(ContainerMemUsageQueryExpr, namespace, convertWorkloadNameToPods(workloadName, kind), containerName, extraLabels) + } + return fmt.Sprintf(ContainerMemUsageQueryExpr, namespace, convertWorkloadNameToPods(workloadName, kind), containerName, extraFilters) +} + +func convertWorkloadNameToPods(workloadName string, workloadKind string) string { + switch workloadKind { + case string(datasource.WorkloadDeployment): + return fmt.Sprintf("^%s-%s", workloadName, WorkloadSuffixRuleForDeployment) + } + return fmt.Sprintf("^%s-%s", workloadName, `.*`) +} diff --git a/pkg/controller/resource-recommend/datasource/prometheus/prometheus_query_test.go b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_query_test.go new file mode 100644 index 0000000000..840fa1c9b2 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/prometheus/prometheus_query_test.go @@ -0,0 +1,136 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "testing" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" +) + +func TestGetContainerCpuUsageQueryExp(t *testing.T) { + type args struct { + namespace string + workloadName string + kind string + containerName string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "Deployment workload kind", + args: args{ + namespace: "my-namespace", + workloadName: "my-deployment", + kind: string(datasource.WorkloadDeployment), + containerName: "test", + }, + want: "rate(container_cpu_usage_seconds_total{container!=\"POD\",namespace=\"my-namespace\",pod=~\"^my-deployment-[a-z0-9]+-[a-z0-9]{5}$\",container=\"test\"}[30s])", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetContainerCpuUsageQueryExp(tt.args.namespace, tt.args.workloadName, tt.args.kind, tt.args.containerName, ""); got != tt.want { + t.Errorf("GetContainerCpuUsageQueryExp() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestGetContainerMemUsageQueryExp(t *testing.T) { + type args struct { + namespace string + workloadName string + kind string + containerName string + extraFilter string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "Deployment workload kind", + args: args{ + namespace: "my-namespace", + workloadName: "my-deployment", + kind: string(datasource.WorkloadDeployment), + containerName: "test", + extraFilter: "", + }, + want: "container_memory_working_set_bytes{container!=\"POD\",namespace=\"my-namespace\",pod=~\"^my-deployment-[a-z0-9]+-[a-z0-9]{5}$\",container=\"test\"}", + }, + { + name: "Deployment workload kind", + args: args{ + namespace: "my-namespace", + workloadName: "my-deployment", + kind: string(datasource.WorkloadDeployment), + containerName: "test2", + extraFilter: "key1=\"value\"", + }, + want: "container_memory_working_set_bytes{container!=\"POD\",namespace=\"my-namespace\",pod=~\"^my-deployment-[a-z0-9]+-[a-z0-9]{5}$\",container=\"test2\",key1=\"value\"}", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetContainerMemUsageQueryExp(tt.args.namespace, tt.args.workloadName, tt.args.kind, tt.args.containerName, tt.args.extraFilter); got != tt.want { + t.Errorf("GetContainerMemUsageQueryExp() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_convertWorkloadNameToPods(t *testing.T) { + type args struct { + workloadName string + workloadKind string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "Deployment workload kind", + args: args{ + workloadName: "my-deployment", + workloadKind: string(datasource.WorkloadDeployment), + }, + want: "^my-deployment-[a-z0-9]+-[a-z0-9]{5}$", + }, + { + name: "Unknown workload kind", + args: args{ + workloadName: "my-workload", + workloadKind: "unknown", + }, + want: "^my-workload-.*", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := convertWorkloadNameToPods(tt.args.workloadName, tt.args.workloadKind); got != tt.want { + t.Errorf("convertWorkloadNameToPods() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/datasource/types.go b/pkg/controller/resource-recommend/datasource/types.go new file mode 100644 index 0000000000..f499cab9ee --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/types.go @@ -0,0 +1,99 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package datasource + +import ( + "fmt" + "sort" + "strings" + + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" +) + +// Sample is a single timestamped value of the metric. +type Sample struct { + Value float64 + Timestamp int64 +} + +// TimeSeries represents a metric with given labels, with its values possibly changing in time. +type TimeSeries struct { + Labels map[string]string + Samples []Sample +} + +// WorkloadKind is k8s resource kind +type WorkloadKind string + +// Resource Name +const ( + // workload kind is deployment + WorkloadDeployment WorkloadKind = "Deployment" +) + +type Metric struct { + //to be extended when new datasource is added + Namespace string + Kind string + APIVersion string + WorkloadName string + PodName string + ContainerName string + Resource v1.ResourceName + Selectors string +} + +func (m *Metric) SetSelector(selectors map[string]string) { + keys := make([]string, 0, len(selectors)) + for key := range selectors { + keys = append(keys, key) + } + sort.Strings(keys) + + var formattedParams []string + for _, key := range keys { + value := selectors[key] + formattedParam := fmt.Sprintf("%s=\"%s\"", key, value) + formattedParams = append(formattedParams, formattedParam) + } + m.Selectors = strings.Join(formattedParams, ",") + klog.V(4).InfoS("Setting selectors", "selectors", m.Selectors) +} + +type Query struct { + //to be extended when new datasource is added + Prometheus *PrometheusQuery +} +type PrometheusQuery struct { + Query string +} + +func NewTimeSeries() *TimeSeries { + return &TimeSeries{ + Labels: make(map[string]string), + Samples: make([]Sample, 0), + } +} + +func (ts *TimeSeries) AppendLabel(key, val string) { + ts.Labels[key] = val +} + +func (ts *TimeSeries) AppendSample(timestamp int64, val float64) { + ts.Samples = append(ts.Samples, Sample{Timestamp: timestamp, Value: val}) +} diff --git a/pkg/controller/resource-recommend/datasource/utils/operator.go b/pkg/controller/resource-recommend/datasource/utils/operator.go new file mode 100644 index 0000000000..63586b3200 --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/utils/operator.go @@ -0,0 +1,86 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "sort" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" +) + +type SamplesOverview struct { + AvgValue float64 + MinValue float64 + MaxValue float64 + Percentile50thValue float64 + Percentile90thValue float64 + LastTimestamp int64 + FirstTimestamp int64 + Count int +} + +func GetSamplesOverview(timeSeries *datasource.TimeSeries) *SamplesOverview { + if timeSeries == nil || len(timeSeries.Samples) == 0 { + return nil + } + samples := timeSeries.Samples + sort.Slice(samples, func(i, j int) bool { + return samples[i].Value < samples[j].Value + }) + aggregationSample := &SamplesOverview{ + Count: len(samples), + MinValue: samples[0].Value, + MaxValue: samples[len(samples)-1].Value, + Percentile50thValue: calculateSamplesPercentile(samples, 0.5), + Percentile90thValue: calculateSamplesPercentile(samples, 0.9), + } + + sumValue := 0.0 + for _, sample := range samples { + if aggregationSample.LastTimestamp < sample.Timestamp { + aggregationSample.LastTimestamp = sample.Timestamp + } + if aggregationSample.FirstTimestamp == 0 || aggregationSample.FirstTimestamp > sample.Timestamp { + aggregationSample.FirstTimestamp = sample.Timestamp + } + sumValue += sample.Value + } + aggregationSample.AvgValue = sumValue / float64(len(timeSeries.Samples)) + return aggregationSample +} + +func CalculateSamplesPercentile(samples []datasource.Sample, percentile float64) float64 { + sort.Slice(samples, func(i, j int) bool { + return samples[i].Value < samples[j].Value + }) + + return calculateSamplesPercentile(samples, percentile) +} + +func calculateSamplesPercentile(samples []datasource.Sample, percentile float64) float64 { + length := float64(len(samples)) + index := (length - 1) * percentile + + // 判断 index 是否为整数 + if index == float64(int(index)) { + return samples[int(index)].Value + } + + lower := samples[int(index)].Value + upper := samples[int(index)+1].Value + return (lower + upper) / 2 +} diff --git a/pkg/controller/resource-recommend/datasource/utils/operator_test.go b/pkg/controller/resource-recommend/datasource/utils/operator_test.go new file mode 100644 index 0000000000..e8b7828b0b --- /dev/null +++ b/pkg/controller/resource-recommend/datasource/utils/operator_test.go @@ -0,0 +1,72 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "reflect" + "testing" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" +) + +func TestGetSamplesOverview(t *testing.T) { + type args struct { + timeSeries *datasource.TimeSeries + } + tests := []struct { + name string + args args + want *SamplesOverview + }{ + { + name: "Empty time series", + args: args{ + timeSeries: &datasource.TimeSeries{Samples: []datasource.Sample{}}, + }, + want: nil, + }, + { + name: "Non-empty time series", + args: args{ + timeSeries: &datasource.TimeSeries{ + Samples: []datasource.Sample{ + {Value: 10, Timestamp: 1630838400}, + {Value: 20, Timestamp: 1630842000}, + {Value: 15, Timestamp: 1630845600}, + }, + }, + }, + want: &SamplesOverview{ + AvgValue: 15, + MinValue: 10, + MaxValue: 20, + Percentile50thValue: 15, + Percentile90thValue: 17.5, + FirstTimestamp: 1630838400, + LastTimestamp: 1630845600, + Count: 3, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GetSamplesOverview(tt.args.timeSeries); !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetSamplesOverview() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/oom/oom_recorder.go b/pkg/controller/resource-recommend/oom/oom_recorder.go new file mode 100644 index 0000000000..b875603eb2 --- /dev/null +++ b/pkg/controller/resource-recommend/oom/oom_recorder.go @@ -0,0 +1,222 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oom + +import ( + "context" + "encoding/json" + "sort" + "sync" + "time" + + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + ConfigMapOOMRecordName = "oom-record" + ConfigMapDataOOMRecord = "oom-data" + ConfigMapOOMRecordNameSpace = "kube-system" + CacheCleanTimeDurationHour = 12 + DataRetentionHour = 168 +) + +type Recorder interface { + ListOOMRecords() []OOMRecord +} + +type OOMRecord struct { + Namespace string + Pod string + Container string + Memory resource.Quantity + OOMAt time.Time +} + +type PodOOMRecorder struct { + client.Client + + mu sync.Mutex + + OOMRecordMaxNumber int + cache []OOMRecord + Queue workqueue.Interface +} + +func (r *PodOOMRecorder) initOOMCacheFromConfigmap() { + r.mu.Lock() + defer r.mu.Unlock() + + oomRecords, err := r.ListOOMRecordsFromConfigmap() + if err != nil { + // TODO: add monitor metric + klog.ErrorS(err, "init cache from configmap failed") + } + r.cache = oomRecords +} + +func (r *PodOOMRecorder) ListOOMRecords() []OOMRecord { + return r.cache +} + +func (r *PodOOMRecorder) cleanOOMRecord() { + r.mu.Lock() + defer r.mu.Unlock() + oomCache := r.ListOOMRecords() + sort.Slice(oomCache, func(i, j int) bool { + return oomCache[i].OOMAt.Before(oomCache[j].OOMAt) + }) + now := time.Now() + index := 0 + for i := len(oomCache) - 1; i >= 0; i-- { + if oomCache[i].OOMAt.Before(now.Add(-DataRetentionHour * time.Hour)) { + break + } + index++ + if index >= r.OOMRecordMaxNumber { + break + } + } + r.cache = oomCache[len(oomCache)-index:] +} + +func (r *PodOOMRecorder) updateOOMRecordCache(oomRecord OOMRecord) bool { + r.mu.Lock() + defer r.mu.Unlock() + + oomCache := r.ListOOMRecords() + if oomCache == nil { + oomCache = []OOMRecord{} + } + + isFound := false + isUpdated := false + for i := range oomCache { + if oomCache[i].Namespace == oomRecord.Namespace && oomCache[i].Pod == oomRecord.Pod && oomCache[i].Container == oomRecord.Container { + if oomRecord.Memory.Value() >= oomCache[i].Memory.Value() && !oomRecord.OOMAt.Equal(oomCache[i].OOMAt) { + oomCache[i].Memory = oomRecord.Memory + oomCache[i].OOMAt = oomRecord.OOMAt + isUpdated = true + } + isFound = true + break + } + } + + if !isFound { + oomCache = append(oomCache, oomRecord) + isUpdated = true + } + if isUpdated { + r.cache = oomCache + } + return isUpdated +} + +func (r *PodOOMRecorder) updateOOMRecordConfigMap() error { + r.mu.Lock() + defer r.mu.Unlock() + + oomCache := r.ListOOMRecords() + cacheData, err := json.Marshal(oomCache) + if err != nil { + return err + } + oomConfigMap := &v1.ConfigMap{} + err = r.Client.Get(context.TODO(), types.NamespacedName{Namespace: ConfigMapOOMRecordNameSpace, + Name: ConfigMapOOMRecordName}, oomConfigMap) + if err != nil { + if client.IgnoreNotFound(err) != nil { + return err + } + oomConfigMap.Name = ConfigMapOOMRecordName + oomConfigMap.Namespace = ConfigMapOOMRecordNameSpace + oomConfigMap.Data = map[string]string{ + ConfigMapDataOOMRecord: string(cacheData), + } + return r.Client.Create(context.TODO(), oomConfigMap) + } + oomConfigMap.Data = map[string]string{ + ConfigMapDataOOMRecord: string(cacheData), + } + return r.Client.Update(context.TODO(), oomConfigMap) +} + +func (r *PodOOMRecorder) Run(stopCh <-chan struct{}) error { + r.initOOMCacheFromConfigmap() + cleanTicker := time.NewTicker(time.Duration(CacheCleanTimeDurationHour) * time.Hour) + go func() { + defer func() { + if r := recover(); r != nil { + if r := recover(); r != nil { + err := errors.Errorf("Run clean oom recorder panic: %v", r.(error)) + klog.Error(err) + panic(err) + } + } + }() + for range cleanTicker.C { + r.cleanOOMRecord() + } + }() + for { + select { + case <-stopCh: + return nil + default: + } + + record, shutdown := r.Queue.Get() + if shutdown { + return errors.New("queue of OOMRecord recorder is shutting down ! ") + } + oomRecord, ok := record.(OOMRecord) + if !ok { + klog.Error("type conversion failed") + r.Queue.Done(record) + continue + } + isUpdated := r.updateOOMRecordCache(oomRecord) + if !isUpdated { + r.Queue.Done(record) + continue + } + + err := r.updateOOMRecordConfigMap() + if err != nil { + klog.ErrorS(err, "Update oomRecord failed") + } + r.Queue.Done(record) + } +} + +func (r *PodOOMRecorder) ListOOMRecordsFromConfigmap() ([]OOMRecord, error) { + oomConfigMap := &v1.ConfigMap{} + err := r.Client.Get(context.TODO(), types.NamespacedName{Namespace: ConfigMapOOMRecordNameSpace, + Name: ConfigMapOOMRecordName}, oomConfigMap) + if err != nil { + return nil, client.IgnoreNotFound(err) + } + oomRecords := make([]OOMRecord, 0) + err = json.Unmarshal([]byte(oomConfigMap.Data[ConfigMapDataOOMRecord]), &oomRecords) + return oomRecords, err +} diff --git a/pkg/controller/resource-recommend/oom/oom_recorder_test.go b/pkg/controller/resource-recommend/oom/oom_recorder_test.go new file mode 100644 index 0000000000..ccfe77c74c --- /dev/null +++ b/pkg/controller/resource-recommend/oom/oom_recorder_test.go @@ -0,0 +1,321 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package oom + +import ( + "context" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestCleanOOMRecord(t *testing.T) { + + oomRecordsList := []PodOOMRecorder{ + { + OOMRecordMaxNumber: 4, + cache: []OOMRecord{ + { + OOMAt: time.Now().Add(-140 * time.Hour), + }, + { + OOMAt: time.Now().Add(-150 * time.Hour), + }, + { + OOMAt: time.Now().Add(-150 * time.Hour), + }, + { + OOMAt: time.Now().Add(-150 * time.Hour), + }, + { + OOMAt: time.Now().Add(-150 * time.Hour), + }, + { + OOMAt: time.Now().Add(-160 * time.Hour), + }, + { + OOMAt: time.Now().Add(-170 * time.Hour), + }, + { + OOMAt: time.Now().Add(-180 * time.Hour), + }, + { + OOMAt: time.Now().Add(-190 * time.Hour), + }, + }, + }, + { + OOMRecordMaxNumber: 4, + cache: []OOMRecord{ + { + OOMAt: time.Now().Add(-150 * time.Hour), + }, + { + OOMAt: time.Now().Add(-140 * time.Hour), + }, + { + OOMAt: time.Now().Add(-130 * time.Hour), + }, + { + OOMAt: time.Now().Add(-120 * time.Hour), + }, + { + OOMAt: time.Now().Add(-110 * time.Hour), + }, + }, + }, + { + OOMRecordMaxNumber: 4, + cache: []OOMRecord{ + { + OOMAt: time.Now().Add(-170 * time.Hour), + }, + { + OOMAt: time.Now().Add(-180 * time.Hour), + }, + { + OOMAt: time.Now().Add(-190 * time.Hour), + }, + { + OOMAt: time.Now().Add(-190 * time.Hour), + }, + { + OOMAt: time.Now().Add(-200 * time.Hour), + }, + { + OOMAt: time.Now().Add(-210 * time.Hour), + }, + }, + }, + { + OOMRecordMaxNumber: 4, + cache: []OOMRecord{ + { + OOMAt: time.Now().Add(-160 * time.Hour), + }, + { + OOMAt: time.Now().Add(-170 * time.Hour), + }, + { + OOMAt: time.Now().Add(-180 * time.Hour), + }, + }, + }, + { + OOMRecordMaxNumber: 4, + cache: []OOMRecord{ + { + OOMAt: time.Now().Add(-170 * time.Hour), + }, + { + OOMAt: time.Now().Add(-180 * time.Hour), + }, + { + OOMAt: time.Now().Add(-190 * time.Hour), + }, + }, + }, + { + OOMRecordMaxNumber: 4, + cache: []OOMRecord{ + { + OOMAt: time.Now().Add(-160 * time.Hour), + }, + { + OOMAt: time.Now().Add(-150 * time.Hour), + }, + { + OOMAt: time.Now().Add(-140 * time.Hour), + }, + }, + }, + { + OOMRecordMaxNumber: 4, + cache: []OOMRecord{}, + }, + } + for index := range oomRecordsList { + splitTimePoints := time.Now().Add(-DataRetentionHour * time.Hour) + oomRecordsList[index].cleanOOMRecord() + if len(oomRecordsList[index].cache) > oomRecordsList[index].OOMRecordMaxNumber { + t.Errorf("Expected oomRecordsList length to be less than or equal to %d, but it is actually %d", + oomRecordsList[index].OOMRecordMaxNumber, len(oomRecordsList[index].cache)) + } + for _, record := range oomRecordsList[index].cache { + if record.OOMAt.Before(splitTimePoints) { + t.Errorf("Expected oomAt to be greater than %v, but it is actually %v", splitTimePoints, record.OOMAt) + } + } + } + +} + +func TestListOOMRecordsFromConfigmap(t *testing.T) { + dummyClient := fake.NewClientBuilder().WithObjects(&v1.ConfigMap{}).Build() + dummyPodOOMRecorder := PodOOMRecorder{ + Client: dummyClient, + } + oomRecords, _ := dummyPodOOMRecorder.ListOOMRecordsFromConfigmap() + if len(oomRecords) > 0 { + t.Errorf("Expected oomRecords length is zero, but actual oomRecords length is %v", len(oomRecords)) + } + oomConfigMap := &v1.ConfigMap{ + Data: map[string]string{ + ConfigMapDataOOMRecord: `[{"Namespace":"dummyNamespace","Pod":"dummyPod","Container":"dummyContainer","Memory":"600Mi","OOMAt":"2023-08-07T16:45:50+08:00"},{"Namespace":"dummyNamespace","Pod":"dummyPod","Container":"dummyContainer","Memory":"600Mi","OOMAt":"2023-08-07T16:46:07+08:00"}]`, + }, + } + oomConfigMap.SetName(ConfigMapOOMRecordName) + oomConfigMap.SetNamespace(ConfigMapOOMRecordNameSpace) + dummyPodOOMRecorder.Client.Create(context.TODO(), oomConfigMap) + oomRecords, err := dummyPodOOMRecorder.ListOOMRecordsFromConfigmap() + if err != nil || len(oomRecords) != 2 { + t.Errorf("Expected oomRecords length is 2 and err is nil, but actual oomRecords length is %v and err is %v", len(oomRecords), err) + } +} + +func TestUpdateOOMRecordCache(t *testing.T) { + now := time.Now() + dummyClient := fake.NewClientBuilder().WithObjects(&v1.ConfigMap{}).Build() + podOOMRecorderList := []PodOOMRecorder{ + { + Client: dummyClient, + cache: []OOMRecord{ + { + Namespace: "dummyNamespace1", + Pod: "dummyPod1", + Container: "dummyContainer1", + Memory: resource.MustParse("6Gi"), + OOMAt: now, + }, + }, + }, + { + Client: dummyClient, + cache: []OOMRecord{ + { + Namespace: "dummyNamespace", + Pod: "dummyPod", + Container: "dummyContainer", + Memory: resource.MustParse("6Gi"), + OOMAt: now, + }, + }, + }, + { + Client: dummyClient, + cache: []OOMRecord{ + { + Namespace: "dummyNamespace", + Pod: "dummyPod", + Container: "dummyContainer", + Memory: resource.MustParse("6Gi"), + OOMAt: now.Add(1 * time.Hour), + }, + }, + }, + { + Client: dummyClient, + cache: []OOMRecord{ + { + Namespace: "dummyNamespace", + Pod: "dummyPod", + Container: "dummyContainer", + Memory: resource.MustParse("7Gi"), + OOMAt: now.Add(1 * time.Hour), + }, + }, + }, + { + Client: dummyClient, + cache: []OOMRecord{ + { + Namespace: "dummyNamespace", + Pod: "dummyPod", + Container: "dummyContainer", + Memory: resource.MustParse("5Gi"), + OOMAt: now.Add(1 * time.Hour), + }, + }, + }, + } + oomRecord := OOMRecord{ + Namespace: "dummyNamespace", + Pod: "dummyPod", + Container: "dummyContainer", + Memory: resource.MustParse("6Gi"), + OOMAt: now, + } + resultList := []bool{true, false, true, false, true} + for index := range podOOMRecorderList { + isUpdated := podOOMRecorderList[index].updateOOMRecordCache(oomRecord) + if isUpdated != resultList[index] { + t.Errorf("Expected isUpdated %v, but it is actually %v", resultList[index], isUpdated) + } + } + +} + +func TestUpdateOOMRecordConfigMap(t *testing.T) { + dummyClient := fake.NewClientBuilder().WithObjects(&v1.ConfigMap{}).Build() + oomConfigMap := &v1.ConfigMap{ + Data: map[string]string{ + ConfigMapDataOOMRecord: `[]`, + }, + } + oomConfigMap.SetName(ConfigMapOOMRecordName) + oomConfigMap.SetNamespace(ConfigMapOOMRecordNameSpace) + dummyPodOOMRecorder := PodOOMRecorder{ + Client: dummyClient, + cache: []OOMRecord{ + { + Namespace: "dummyNamespace", + Pod: "dummyPod", + Container: "dummyContainer", + Memory: resource.MustParse("600Mi"), + OOMAt: time.Date(2012, time.March, 4, 0, 0, 0, 0, time.UTC), + }, + }, + } + err := dummyPodOOMRecorder.updateOOMRecordConfigMap() + if err != nil { + t.Errorf("Expected the configMap was successfully created,but actually an error:%v occurred.", err) + } + oomRecordList, _ := dummyPodOOMRecorder.ListOOMRecordsFromConfigmap() + for index := range oomRecordList { + if dummyPodOOMRecorder.cache[index] != oomRecordList[index] { + t.Errorf("Expected OOMRecord value is %v, actually OOMRecord value is %v", + dummyPodOOMRecorder.cache[index], oomRecordList[index]) + } + } + + dummyPodOOMRecorder.Client.DeleteAllOf(context.TODO(), oomConfigMap) + dummyPodOOMRecorder.Client.Create(context.TODO(), oomConfigMap) + err = dummyPodOOMRecorder.updateOOMRecordConfigMap() + if err != nil { + t.Errorf("Expected the configMap was successfully updated,but actually an error:%v occurred.", err) + } + oomRecordList, _ = dummyPodOOMRecorder.ListOOMRecordsFromConfigmap() + for index := range oomRecordList { + if dummyPodOOMRecorder.cache[index] != oomRecordList[index] { + t.Errorf("Expected OOMRecord value is %v, actually OOMRecord value is %v", + dummyPodOOMRecorder.cache[index], oomRecordList[index]) + } + } +} diff --git a/pkg/controller/resource-recommend/processor/common/task_key.go b/pkg/controller/resource-recommend/processor/common/task_key.go new file mode 100644 index 0000000000..a2d598f126 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/common/task_key.go @@ -0,0 +1,71 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" +) + +type TaskConfigStr string + +type TaskID string + +type ProcessKey struct { + ResourceRecommendNamespacedName types.NamespacedName + *datasource.Metric +} + +type ProcessConfig struct { + ProcessKey + Config TaskConfigStr +} + +func (pc *ProcessConfig) Validate() error { + if pc.Metric == nil { + return fmt.Errorf("metric is empty") + } + if pc.ContainerName == "" { + return fmt.Errorf("containerName is empty") + } + + if pc.Kind == "" { + return fmt.Errorf("kind is empty") + } + + if pc.WorkloadName == "" { + return fmt.Errorf("workloadName is empty") + } + if !(pc.Resource == v1.ResourceCPU || pc.Resource == v1.ResourceMemory) { + return fmt.Errorf("controlledResource only can be [%s, %s]", v1.ResourceCPU, v1.ResourceMemory) + } + return nil +} + +func (pc *ProcessConfig) GenerateTaskID() TaskID { + return TaskID(pc.ResourceRecommendNamespacedName.String() + "-" + + pc.Kind + "-" + + pc.APIVersion + "-" + + pc.WorkloadName + "-" + + pc.ContainerName + "-" + + string(pc.Resource) + "-" + + string(pc.Config)) +} diff --git a/pkg/controller/resource-recommend/processor/manager/processor_manager.go b/pkg/controller/resource-recommend/processor/manager/processor_manager.go new file mode 100644 index 0000000000..7aad864ab8 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/manager/processor_manager.go @@ -0,0 +1,75 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "context" + "runtime/debug" + "sync" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/utils/log" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type Manager struct { + processors map[v1alpha1.Algorithm]processor.Processor +} + +func NewManager(datasourceProxy *datasource.Proxy, c client.Client) *Manager { + percentileProcessor := percentile.NewProcessor(datasourceProxy, c) + return &Manager{ + processors: map[v1alpha1.Algorithm]processor.Processor{ + v1alpha1.AlgorithmPercentile: percentileProcessor, + }, + } +} + +func (m *Manager) StartProcess(ctx context.Context) { + var wg sync.WaitGroup + for processorName, dataProcessor := range m.processors { + wg.Add(1) + processorCtx := log.SetKeysAndValues(log.InitContext(ctx), "processor", processorName) + go func(ctx context.Context, processor processor.Processor) { + defer wg.Done() + defer func() { + if r := recover(); r != nil { + log.ErrorS(ctx, r.(error), "processor panic", "stack", string(debug.Stack())) + panic("processor panic") + } + }() + processor.Run(ctx) + }(processorCtx, dataProcessor) + } + + log.InfoS(ctx, "ProcessorManager started, all Processor running") + + wg.Wait() + + log.InfoS(ctx, "ProcessorManager stopped, all Processor end") +} + +func (m *Manager) GetProcessor(algorithm v1alpha1.Algorithm) processor.Processor { + if dataProcessor, ok := m.processors[algorithm]; ok { + return dataProcessor + } + return m.processors[v1alpha1.AlgorithmPercentile] +} diff --git a/pkg/controller/resource-recommend/processor/manager/processor_manager_test.go b/pkg/controller/resource-recommend/processor/manager/processor_manager_test.go new file mode 100644 index 0000000000..f41724a647 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/manager/processor_manager_test.go @@ -0,0 +1,118 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "context" + "reflect" + "testing" + "time" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + customError "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/error" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/utils/log" +) + +type mockProcessor struct { + algorithm v1alpha1.Algorithm + runMark string +} + +func (p *mockProcessor) Run(ctx context.Context) { + time.Sleep(1 * time.Second) + p.runMark = string(p.algorithm) + "processor" + "-run" + log.InfoS(ctx, "mockProcessor Run", "algorithm", p.algorithm) +} + +func (p *mockProcessor) Register(_ *common.ProcessConfig) *customError.CustomError { + return nil +} + +func (p *mockProcessor) Cancel(_ *common.ProcessKey) *customError.CustomError { return nil } + +func (p *mockProcessor) QueryProcessedValues(_ *common.ProcessKey) (float64, error) { + return 0, nil +} + +func TestManager_StartProcess(t *testing.T) { + mockProcessor1 := mockProcessor{algorithm: "mockAlgorithm1"} + mockProcessor2 := mockProcessor{algorithm: "mockAlgorithm2"} + mockProcessor3 := mockProcessor{algorithm: "mockAlgorithm3"} + tests := []struct { + name string + processors []*mockProcessor + }{ + { + name: "case1", + processors: []*mockProcessor{&mockProcessor1, &mockProcessor2, &mockProcessor3}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manager := &Manager{processors: map[v1alpha1.Algorithm]processor.Processor{}} + for _, p := range tt.processors { + manager.processors[p.algorithm] = p + } + manager.StartProcess(context.Background()) + for _, p := range tt.processors { + if p.runMark != string(p.algorithm)+"processor"+"-run" { + t.Errorf("StartProcess() failed, %s processor not run", p.algorithm) + } + } + }) + } +} + +func TestManager_GetProcessor(t *testing.T) { + manager1 := NewManager(nil, nil) + type args struct { + algorithm v1alpha1.Algorithm + } + tests := []struct { + name string + manager *Manager + args args + want processor.Processor + }{ + { + name: "Unknown_Algorithm", + manager: manager1, + args: args{ + algorithm: "Unknown", + }, + want: manager1.processors[v1alpha1.AlgorithmPercentile], + }, + { + name: "get_AlgorithmPercentile_processor", + manager: manager1, + args: args{ + algorithm: v1alpha1.AlgorithmPercentile, + }, + want: manager1.processors[v1alpha1.AlgorithmPercentile], + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.manager.GetProcessor(tt.args.algorithm) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetProcessor() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/processor/percentile/process_gc.go b/pkg/controller/resource-recommend/processor/percentile/process_gc.go new file mode 100644 index 0000000000..1df8642ddf --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/process_gc.go @@ -0,0 +1,128 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "context" + "runtime/debug" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + + spendsmartv1alpha1 "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/utils/log" +) + +func (p *Processor) GarbageCollector(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + errMsg := "garbage collect goroutine panic" + log.ErrorS(ctx, r.(error), errMsg, "stack", string(debug.Stack())) + panic(errMsg) + } + }() + + ticker := time.Tick(DefaultGarbageCollectInterval) + for range ticker { + if err := p.garbageCollect(NewContext()); err != nil { + log.ErrorS(ctx, err, "garbageCollect failed") + } + } +} + +func (p *Processor) garbageCollect(ctx context.Context) error { + defer func() { + if r := recover(); r != nil { + errMsg := "garbage collect run panic" + log.ErrorS(ctx, r.(error), errMsg, "stack", string(debug.Stack())) + } + }() + + log.InfoS(ctx, "garbage collect start") + + p.mutex.Lock() + defer p.mutex.Unlock() + + // Tasks that have not been running for too long are considered invalid and will be cleared + p.AggregateTasks.Range(func(ki, vi any) bool { + taskID := ki.(common.TaskID) + t, ok := vi.(*task.HistogramTask) + if !ok { + p.AggregateTasks.Delete(taskID) + return true + } + if t.IsTimeoutNotExecute() { + p.AggregateTasks.Delete(taskID) + } + return true + }) + + // List all ResourceRecommend CR up to now + resourceRecommendList := &spendsmartv1alpha1.ResourceRecommendList{} + err := p.Client.List(ctx, resourceRecommendList, &k8sclient.ListOptions{Raw: &metav1.ListOptions{ResourceVersion: "0"}}) + if err != nil { + log.ErrorS(ctx, err, "garbage collect list all ResourceRecommend failed") + return err + } + + //p.ClearingNoAttributionTask(resourceRecommendList) + klog.InfoS("percentile processor garbage collect list ResourceRecommend", + "ResourceRecommend Count", len(resourceRecommendList.Items)) + // Convert the ResourceRecommend list to map for quick check whether it exists + existResourceRecommends := make(map[types.NamespacedName]spendsmartv1alpha1.CrossVersionObjectReference) + for _, existResourceRecommend := range resourceRecommendList.Items { + namespacedName := types.NamespacedName{ + Name: existResourceRecommend.Name, + Namespace: existResourceRecommend.Namespace, + } + existResourceRecommends[namespacedName] = existResourceRecommend.Spec.TargetRef + } + + // Delete tasks belong to not exist ResourceRecommend or task not match the targetRef in the ResourceRecommend + for namespacedName, tasks := range p.ResourceRecommendTaskIDsMap { + if tasks == nil { + delete(p.ResourceRecommendTaskIDsMap, namespacedName) + continue + } + targetRef, isExist := existResourceRecommends[namespacedName] + for metrics, taskID := range *tasks { + if !isExist || + targetRef.Kind != metrics.Kind || + targetRef.Name != metrics.WorkloadName || + targetRef.APIVersion != metrics.APIVersion { + p.AggregateTasks.Delete(taskID) + delete(*tasks, metrics) + } + // timeout task in p.AggregateTasks will clean, + // If taskID does not exist in p.AggregateTasks，mean is the task is cleared, this needs to be deleted from tasks + if _, found := p.AggregateTasks.Load(taskID); !found { + delete(*tasks, metrics) + } + } + if !isExist { + delete(p.ResourceRecommendTaskIDsMap, namespacedName) + } + } + + log.InfoS(ctx, "garbage collect end") + return nil +} diff --git a/pkg/controller/resource-recommend/processor/percentile/process_gc_test.go b/pkg/controller/resource-recommend/processor/percentile/process_gc_test.go new file mode 100644 index 0000000000..c7c6dfc9d0 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/process_gc_test.go @@ -0,0 +1,120 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "sync" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-api/pkg/client/clientset/versioned/scheme" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task" +) + +func TestProcessor_garbageCollect(t *testing.T) { + s := scheme.Scheme + s.AddKnownTypes(v1alpha1.SchemeGroupVersion, &v1alpha1.ResourceRecommend{}) + client := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() + processor := Processor{ + Client: client, + AggregateTasks: &sync.Map{}, + ResourceRecommendTaskIDsMap: make(map[types.NamespacedName]*map[datasource.Metric]common.TaskID), + } + + namespacedName0 := types.NamespacedName{ + Name: "testGC0", + Namespace: "testGC", + } + + // case1: clean TypeIllegal task + taskIDTaskTypeIllegal := common.TaskID("taskID_TaskTypeIllegal") + metricTaskTypeIllegal := datasource.Metric{ + Namespace: "testGCTaskTypeIllegal", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testGCTaskTypeIllegal", + ContainerName: "testGCTaskTypeIllegal-1", + Resource: v1.ResourceCPU, + } + + // case2: clean timeout task + taskID0 := common.TaskID("taskID0") + metric0 := datasource.Metric{ + Namespace: "testGC", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testGC0", + ContainerName: "testGC0-1", + Resource: v1.ResourceCPU, + } + task0, _ := task.NewTask(metric0, "") + task0.AddSample(time.Unix(1693572914, 0), 20, 1000) + + // case3: clean resourceRecommend cr not exist task + taskID1 := common.TaskID("taskID1") + metric1 := datasource.Metric{ + Namespace: "testGC", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testGC1", + ContainerName: "testGC1-1", + Resource: v1.ResourceCPU, + } + task1, _ := task.NewTask(metric1, "") + + processor.AggregateTasks.LoadOrStore(taskIDTaskTypeIllegal, 3) + processor.AggregateTasks.LoadOrStore(taskID0, task0) + processor.AggregateTasks.LoadOrStore(taskID1, task1) + processor.ResourceRecommendTaskIDsMap[namespacedName0] = &map[datasource.Metric]common.TaskID{ + metric0: taskID0, + metric1: taskID1, + metricTaskTypeIllegal: taskIDTaskTypeIllegal, + } + + tests := []struct { + name string + wantErr bool + }{ + {}, + } + for _, tt := range tests { + err := processor.garbageCollect(NewContext()) + if (err != nil) != tt.wantErr { + t.Errorf("garbageCollect() error = %v, wantErr %v", err, tt.wantErr) + return + } + if _, ok := processor.AggregateTasks.Load(taskIDTaskTypeIllegal); ok { + t.Errorf("garbageCollect() type tllegal task not cleaned up") + } + if _, ok := processor.AggregateTasks.Load(taskID0); ok { + t.Errorf("garbageCollect() timeout task not cleaned up") + } + if _, ok := processor.AggregateTasks.Load(taskID1); ok { + t.Errorf("garbageCollect() resourceRecommend cr not exist tasks(in AggregateTasks) not cleaned up") + } + if _, ok := processor.ResourceRecommendTaskIDsMap[namespacedName0]; ok { + t.Errorf("garbageCollect() resourceRecommend cr not exist tasks(in ResourceRecommendTaskIDsMap) not cleaned up") + } + } +} diff --git a/pkg/controller/resource-recommend/processor/percentile/process_tasks.go b/pkg/controller/resource-recommend/processor/percentile/process_tasks.go new file mode 100644 index 0000000000..72a6721f6f --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/process_tasks.go @@ -0,0 +1,99 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "context" + "runtime/debug" + "sync" + + "github.com/pkg/errors" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/utils/log" +) + +func (p *Processor) ProcessTasks(ctx context.Context) { + defer func() { + if r := recover(); r != nil { + errMsg := "process tasks goroutine run panic" + log.ErrorS(ctx, r.(error), errMsg, "stack", string(debug.Stack())) + panic(errMsg) + } + }() + + wg := &sync.WaitGroup{} + wg.Add(DefaultConcurrentTaskNum) + for i := 0; i < DefaultConcurrentTaskNum; i++ { + go func() { + defer wg.Done() + // Run a worker thread that just dequeues items, processes them, and marks them done. + for p.processTask(NewContext()) { + } + }() + } + + log.InfoS(ctx, "process workers running") + wg.Wait() + log.InfoS(ctx, "all process workers finished") +} + +func (p *Processor) processTask(ctx context.Context) bool { + obj, shutdown := p.TaskQueue.Get() + log.InfoS(ctx, "process task dequeue", "obj", obj, "shutdown", shutdown) + if shutdown { + err := errors.New("task queue is shutdown") + log.ErrorS(ctx, err, "process task failed") + // Stop working + return false + } + + defer p.TaskQueue.Done(obj) + + p.taskHandler(ctx, obj) + return true +} + +func (p *Processor) taskHandler(ctx context.Context, obj interface{}) { + log.InfoS(ctx, "task handler begin") + taskID, ok := obj.(common.TaskID) + if !ok { + err := errors.New("task key type error, drop") + log.ErrorS(ctx, err, "task err", "obj", obj) + p.TaskQueue.Forget(obj) + return + } + ctx = log.SetKeysAndValues(ctx, "taskID", taskID) + + task, err := p.getTaskForTaskID(taskID) + if err != nil { + log.ErrorS(ctx, err, "task not found, drop it") + p.TaskQueue.Forget(obj) + return + } + + if nextRunInterval, err := task.Run(ctx, p.DatasourceProxy); err != nil { + log.ErrorS(ctx, err, "task handler err") + p.TaskQueue.AddRateLimited(taskID) + } else { + log.InfoS(ctx, "task handler finished") + p.TaskQueue.Forget(obj) + if nextRunInterval > 0 { + p.TaskQueue.AddAfter(taskID, nextRunInterval) + } + } +} diff --git a/pkg/controller/resource-recommend/processor/percentile/process_tasks_test.go b/pkg/controller/resource-recommend/processor/percentile/process_tasks_test.go new file mode 100644 index 0000000000..6ca6ef6f4c --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/process_tasks_test.go @@ -0,0 +1,289 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "golang.org/x/time/rate" + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/util/workqueue" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task" +) + +func TestProcessor_processTask(t *testing.T) { + taskQueue1 := workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName) + //processor1 := Processor{ + // TaskQueue: taskQueue1, + //} + + taskQueue2 := workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName) + //processor2 := Processor{ + // TaskQueue: taskQueue2, + //} + taskIDTaskTypeIllegal := 3 + + taskQueue3 := workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName) + //processor3 := Processor{ + // TaskQueue: taskQueue3, + // AggregateTasks: &sync.Map{}, + //} + taskID3 := common.TaskID("case3") + + type testFunc func() + type checkFunc func() + tests := []struct { + name string + processor Processor + testFunc + isContinueToRun bool + checkFunc + }{ + { + name: "queue_shutdown", + processor: Processor{ + TaskQueue: taskQueue1, + }, + testFunc: func() { + taskQueue1.ShutDown() + }, + isContinueToRun: false, + checkFunc: func() {}, + }, + { + name: "task_type_illegal", + processor: Processor{ + TaskQueue: taskQueue2, + }, + testFunc: func() { + taskQueue2.Add(taskIDTaskTypeIllegal) + }, + isContinueToRun: true, + checkFunc: func() { + if taskQueue2.Len() != 0 { + t.Errorf("task type illegal not forget") + } + }, + }, + { + name: "task_type_illegal", + processor: Processor{ + TaskQueue: taskQueue3, + AggregateTasks: &sync.Map{}, + }, + testFunc: func() { + taskQueue3.Add(taskID3) + }, + isContinueToRun: true, + checkFunc: func() { + if taskQueue3.Len() != 0 { + t.Errorf("not found task not forget") + } + }, + }, + } + for i := 0; i < len(tests); i++ { + t.Run(tests[i].name, func(t *testing.T) { + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + isContinueToRun := tests[i].processor.processTask(NewContext()) + if tests[i].isContinueToRun != isContinueToRun { + t.Errorf("processTask() isShutdown = %v, want %v\n", isContinueToRun, tests[i].isContinueToRun) + return + } + }() + go func() { + defer wg.Done() + tests[i].testFunc() + }() + wg.Wait() + tests[i].checkFunc() + }) + } +} + +type MockDatasourceForProcessTasks struct{} + +func (m1 *MockDatasourceForProcessTasks) QueryTimeSeries(_ *datasource.Query, _ time.Time, _ time.Time, _ time.Duration) (*datasource.TimeSeries, error) { + return &datasource.TimeSeries{Samples: []datasource.Sample{ + { + Timestamp: 1694270256, + Value: 1, + }, + }}, nil +} + +func (m1 *MockDatasourceForProcessTasks) ConvertMetricToQuery(metric datasource.Metric) (*datasource.Query, error) { + return nil, nil +} + +func TestProcessor_processTask1(t *testing.T) { + queueRateLimiter := workqueue.NewMaxOfRateLimiter( + workqueue.NewItemExponentialFailureRateLimiter(time.Second, 32*time.Second), + // 10 qps, 100 bucket size. This is only for retry speed and its only the overall factor (not per item) + &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)}, + ) + taskQueue1 := workqueue.NewNamedRateLimitingQueue(queueRateLimiter, ProcessorName) + taskID1 := common.TaskID("task_run_err") + aggregateTasks1 := &sync.Map{} + aggregateTasks1.Store(taskID1, &task.HistogramTask{}) + + taskQueue2 := workqueue.NewNamedRateLimitingQueue(queueRateLimiter, ProcessorName) + taskID := common.TaskID("task_run_err") + mockTask, _ := task.NewTask(datasource.Metric{Resource: v1.ResourceCPU}, "") + mockTask.ProcessInterval = time.Second * 2 + aggregateTasks2 := &sync.Map{} + aggregateTasks2.Store(taskID, mockTask) + + proxy := datasource.NewProxy() + proxy.RegisterDatasource(datasource.PrometheusDatasource, &MockDatasourceForProcessTasks{}) + + type testFunc func() + type runFunc func() + tests := []struct { + name string + processor Processor + testFunc + runFunc + runTimes int + wantRunSecond int64 + }{ + { + name: "run_err", + processor: Processor{ + TaskQueue: taskQueue1, + AggregateTasks: aggregateTasks1, + }, + testFunc: func() { + taskQueue1.Add(taskID1) + }, + runTimes: 4, + wantRunSecond: 7, + }, + { + name: "run", + processor: Processor{ + TaskQueue: taskQueue2, + AggregateTasks: aggregateTasks2, + DatasourceProxy: proxy, + }, + testFunc: func() { + taskQueue2.Add(taskID) + }, + runTimes: 3, + wantRunSecond: 4, + }, + } + for i := 0; i < len(tests); i++ { + t.Run(tests[i].name, func(t *testing.T) { + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + beginTime := time.Now() + for j := 0; j < tests[i].runTimes; j++ { + _ = tests[i].processor.processTask(NewContext()) + } + runtime := int64(time.Now().Sub(beginTime) / time.Second) + if runtime != tests[i].wantRunSecond { + t.Errorf("processTask() runtime = %ds, want %ds", runtime, tests[i].wantRunSecond) + } + }() + go func() { + defer wg.Done() + tests[i].testFunc() + }() + wg.Wait() + }) + } +} + +type MockDatasource1ForProcessTasks struct{} + +func (m1 *MockDatasource1ForProcessTasks) QueryTimeSeries(_ *datasource.Query, _ time.Time, _ time.Time, _ time.Duration) (*datasource.TimeSeries, error) { + time.Sleep(2 * time.Second) + return &datasource.TimeSeries{Samples: []datasource.Sample{ + { + Timestamp: 1694270256, + Value: 1, + }, + }}, nil +} + +func (m1 *MockDatasource1ForProcessTasks) ConvertMetricToQuery(metric datasource.Metric) (*datasource.Query, error) { + return nil, nil +} + +func TestProcessor_ProcessTasks(t *testing.T) { + q := workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName) + aggregateTasks := &sync.Map{} + proxy := datasource.NewProxy() + proxy.RegisterDatasource(datasource.PrometheusDatasource, &MockDatasource1ForProcessTasks{}) + + taskIDList := make([]common.TaskID, 0) + for i := 0; i < DefaultConcurrentTaskNum+1; i++ { + taskID := common.TaskID(fmt.Sprintf("task-%d", i)) + taskIDList = append(taskIDList, taskID) + mockTask, _ := task.NewTask(datasource.Metric{Resource: v1.ResourceCPU}, "") + aggregateTasks.Store(taskID, mockTask) + } + mockP := Processor{ + TaskQueue: q, + AggregateTasks: aggregateTasks, + DatasourceProxy: proxy, + } + type testFunc func() + tests := []struct { + name string + testFunc + }{ + { + name: "case1", + testFunc: func() { + for _, id := range taskIDList { + q.Add(id) + } + time.Sleep(5 * time.Second) + q.ShutDown() + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + defer wg.Done() + mockP.ProcessTasks(context.Background()) + }() + go func() { + defer wg.Done() + tt.testFunc() + }() + wg.Wait() + }) + } +} diff --git a/pkg/controller/resource-recommend/processor/percentile/process_util.go b/pkg/controller/resource-recommend/processor/percentile/process_util.go new file mode 100644 index 0000000000..022a0fb382 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/process_util.go @@ -0,0 +1,62 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "context" + + "github.com/pkg/errors" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/utils/log" +) + +func NewContext() context.Context { + return log.SetKeysAndValues(log.InitContext(context.Background()), "processor", ProcessorName) +} + +func (p *Processor) getTaskForProcessKey(processKey *common.ProcessKey) (*task.HistogramTask, error) { + if processKey == nil { + return nil, errors.Errorf("ProcessKey is nil") + } + if processKey.Metric == nil { + return nil, errors.Errorf("Metric is nil") + } + if tasks, ok := p.ResourceRecommendTaskIDsMap[processKey.ResourceRecommendNamespacedName]; !ok { + return nil, errors.Errorf("not found percentile process task ID for ResourceRecommend(%s)", + processKey.ResourceRecommendNamespacedName) + } else { + if taskID, exist := (*tasks)[*processKey.Metric]; exist { + return p.getTaskForTaskID(taskID) + } + return nil, errors.Errorf("not found process task ID for container(%s) in ResourceRecommend(%s)", + processKey.ContainerName, processKey.ResourceRecommendNamespacedName) + } +} + +func (p *Processor) getTaskForTaskID(taskID common.TaskID) (*task.HistogramTask, error) { + data, found := p.AggregateTasks.Load(taskID) + if !found { + return nil, errors.New("process task not found") + } + t, ok := data.(*task.HistogramTask) + if !ok { + return nil, errors.New("process task type illegal") + } + return t, nil +} diff --git a/pkg/controller/resource-recommend/processor/percentile/process_util_test.go b/pkg/controller/resource-recommend/processor/percentile/process_util_test.go new file mode 100644 index 0000000000..2a8d92c07f --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/process_util_test.go @@ -0,0 +1,91 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "reflect" + "testing" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task" +) + +func TestProcessor_getTaskForProcessKey(t *testing.T) { + type newProcessor func() *Processor + tests := []struct { + name string + newProcessor newProcessor + processKey *common.ProcessKey + want *task.HistogramTask + wantErr bool + }{ + { + name: "processConfig is nil", + processKey: nil, + want: nil, + wantErr: true, + }, + { + name: "processConfig.Metric is nil", + processKey: &metricIsNilProcessorKey, + want: nil, + wantErr: true, + }, + { + name: "not found NamespacedName", + processKey: ¬ExistNamespacedNameProcessorKey, + want: nil, + wantErr: true, + }, + { + name: "not found Metric", + processKey: ¬ExistMetricProcessorKey, + want: nil, + wantErr: true, + }, + { + name: "not found Task", + processKey: ¬FoundTaskProcessorKey, + want: nil, + wantErr: true, + }, + { + name: "task type illegal", + processKey: &valueTypeIllegalProcessorKey, + want: nil, + wantErr: true, + }, + { + name: "success", + processKey: &mockProcessKey1, + want: mockTask1, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := mockProcessor.getTaskForProcessKey(tt.processKey) + if (err != nil) != tt.wantErr { + t.Errorf("getTaskForProcessKey() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getTaskForProcessKey() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/processor/percentile/processor.go b/pkg/controller/resource-recommend/processor/percentile/processor.go new file mode 100644 index 0000000000..ac67c10873 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/processor.go @@ -0,0 +1,219 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "context" + "runtime/debug" + "sync" + "time" + + "github.com/pkg/errors" + "golang.org/x/time/rate" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task" + customError "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/error" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/utils" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/utils/log" +) + +const ( + ProcessorName = "percentile" + // DefaultConcurrentTaskNum is num of default concurrent task + DefaultConcurrentTaskNum = 100 + DefaultPercentile = 0.9 + DefaultGarbageCollectInterval = 1 * time.Hour + ExceptionRequeueBaseDelay = time.Minute + ExceptionRequeueMaxDelay = 30 * time.Minute +) + +type Processor struct { + mutex sync.Mutex + + client.Client + + DatasourceProxy *datasource.Proxy + + TaskQueue workqueue.RateLimitingInterface + + AggregateTasks *sync.Map + + // Stores taskID corresponding to Metrics in the ResourceRecommend + ResourceRecommendTaskIDsMap map[types.NamespacedName]*map[datasource.Metric]common.TaskID +} + +var DefaultQueueRateLimiter = workqueue.NewMaxOfRateLimiter( + workqueue.NewItemExponentialFailureRateLimiter(ExceptionRequeueBaseDelay, ExceptionRequeueMaxDelay), + // 10 qps, 100 bucket size. This is only for retry speed and its only the overall factor (not per item) + &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(10), 100)}, +) + +func NewProcessor(datasourceProxy *datasource.Proxy, c client.Client) processor.Processor { + return &Processor{ + DatasourceProxy: datasourceProxy, + TaskQueue: workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName), + Client: c, + AggregateTasks: &sync.Map{}, + ResourceRecommendTaskIDsMap: make(map[types.NamespacedName]*map[datasource.Metric]common.TaskID), + } +} + +func (p *Processor) Register(processConfig *common.ProcessConfig) (cErr *customError.CustomError) { + defer func() { + if cErr != nil { + klog.ErrorS(cErr, "Percentile task register failed", "ResourceRecommend", processConfig.ResourceRecommendNamespacedName) + } + if r := recover(); r != nil { + errMsg := "percentile process register panic" + klog.ErrorS(r.(error), errMsg, "stack", string(debug.Stack())) + cErr = customError.RegisterProcessTaskPanic() + } + }() + + if err := processConfig.Validate(); err != nil { + return customError.RegisterProcessTaskValidateError(err) + } + + taskID := processConfig.GenerateTaskID() + + // Check whether a task has been registered and avoid repeated registration + _, ok := p.AggregateTasks.Load(taskID) + if ok { + klog.V(4).InfoS("The Percentile Processor task already registered", "processConfig", utils.StructToString(processConfig)) + return nil + } + + p.mutex.Lock() + defer p.mutex.Unlock() + + klog.InfoS("Register Percentile Processor Task", "processConfig", utils.StructToString(processConfig)) + + metric := *processConfig.Metric + + t, err := task.NewTask(metric, processConfig.Config) + if err != nil { + cErr := customError.NewProcessTaskError(err) + return cErr + } + + _, loaded := p.AggregateTasks.LoadOrStore(taskID, t) + if !loaded { + p.TaskQueue.Add(taskID) + } + + // Record the taskID corresponding to the Metric with the same ResourceRecommendID into ResourceRecommendTaskIDsMap + // To get the taskID from the ResourceRecommendID and Metric + if tasks, ok := p.ResourceRecommendTaskIDsMap[processConfig.ResourceRecommendNamespacedName]; !ok { + p.ResourceRecommendTaskIDsMap[processConfig.ResourceRecommendNamespacedName] = &map[datasource.Metric]common.TaskID{ + metric: taskID, + } + } else { + if existingTaskID, exist := (*tasks)[metric]; exist { + if existingTaskID == taskID { + return nil + } + // existingTaskID != taskID means that the config of the task has changed. + // Need to delete the old task of config + p.AggregateTasks.Delete(existingTaskID) + } + (*tasks)[metric] = taskID + } + + return nil +} + +func (p *Processor) Cancel(processKey *common.ProcessKey) (cErr *customError.CustomError) { + if processKey == nil { + return nil + } + + defer func() { + if cErr != nil { + klog.ErrorS(cErr, "Percentile task cancel failed", "ResourceRecommend", processKey.ResourceRecommendNamespacedName) + } + if r := recover(); r != nil { + errMsg := "percentile process cancel panic" + klog.ErrorS(r.(error), errMsg, "stack", string(debug.Stack())) + cErr = customError.CancelProcessTaskPanic() + } + }() + + p.mutex.Lock() + defer p.mutex.Unlock() + + tasks, ok := p.ResourceRecommendTaskIDsMap[processKey.ResourceRecommendNamespacedName] + if !ok { + klog.InfoS("Cancel task failed, percentile process task not found", "ResourceRecommend", + processKey.ResourceRecommendNamespacedName) + return customError.NotFoundTasksError(processKey.ResourceRecommendNamespacedName) + } + if processKey.Metric == nil { + klog.InfoS("delete percentile process tasks", "processConfig", processKey) + for _, taskID := range *tasks { + p.AggregateTasks.Delete(taskID) + } + delete(p.ResourceRecommendTaskIDsMap, processKey.ResourceRecommendNamespacedName) + } else { + if taskID, exist := (*tasks)[*processKey.Metric]; exist { + klog.InfoS("percentile process task delete for", "processConfig", processKey) + p.AggregateTasks.Delete(taskID) + delete(*tasks, *processKey.Metric) + } else { + klog.InfoS("task for metric cannot be found, don't deleted", "processConfig", processKey, "metric", processKey.Metric) + } + if tasks == nil || len(*tasks) == 0 { + delete(p.ResourceRecommendTaskIDsMap, processKey.ResourceRecommendNamespacedName) + } + } + + return nil +} + +func (p *Processor) Run(ctx context.Context) { + log.InfoS(ctx, "percentile processor starting") + + // Get task from queue and run it + go p.ProcessTasks(ctx) + + // Garbage collect every hour. Clearing timeout or no attribution task + go p.GarbageCollector(ctx) + + log.InfoS(ctx, "percentile processor running") + + <-ctx.Done() + + log.InfoS(ctx, "percentile processor end") +} + +func (p *Processor) QueryProcessedValues(processKey *common.ProcessKey) (float64, error) { + t, err := p.getTaskForProcessKey(processKey) + if err != nil { + return 0, errors.Wrapf(err, "internal err, process task not found") + } + percentileValue, err := t.QueryPercentileValue(NewContext(), DefaultPercentile) + if err != nil { + return 0, err + } + return percentileValue, nil +} diff --git a/pkg/controller/resource-recommend/processor/percentile/processor_mock.go b/pkg/controller/resource-recommend/processor/percentile/processor_mock.go new file mode 100644 index 0000000000..92dd58145d --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/processor_mock.go @@ -0,0 +1,131 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "sync" + "time" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/percentile/task" +) + +var mockNamespacedName1 = types.NamespacedName{ + Namespace: "testNamespace1", + Name: "testName1", +} +var mockNamespacedName2 = types.NamespacedName{ + Namespace: "testNamespace2", + Name: "testName2", +} +var notExistNamespacedName = types.NamespacedName{ + Namespace: "notExistNamespace", + Name: "notExistName", +} +var mockMetric1 = datasource.Metric{ + Namespace: "testNamespace1", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testWorkload1", + ContainerName: "testContainer1", + Resource: "cpu", +} +var emptyTaskMetric = datasource.Metric{ + Namespace: "testNamespace1", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testWorkload1", + ContainerName: "testContainer1", + Resource: "memory", +} +var notExistTaskIDMetric = datasource.Metric{ + Namespace: "testNamespace1", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testWorkload2", + ContainerName: "testContainer123", + Resource: "cpu", +} +var valueTypeIllegalMetric = datasource.Metric{ + Namespace: "testNamespace1", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testWorkload2", + ContainerName: "testContainer123", + Resource: "memory", +} +var notExistMetric = datasource.Metric{ + Namespace: "testNamespace1", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testWorkload2", + ContainerName: "testContainer876", + Resource: "memory", +} +var mockProcessKey1 = common.ProcessKey{ResourceRecommendNamespacedName: mockNamespacedName1, Metric: &mockMetric1} +var emptyTaskProcessorKey = common.ProcessKey{ResourceRecommendNamespacedName: mockNamespacedName1, Metric: &emptyTaskMetric} +var metricIsNilProcessorKey = common.ProcessKey{ResourceRecommendNamespacedName: mockNamespacedName1, Metric: nil} +var notExistNamespacedNameProcessorKey = common.ProcessKey{ResourceRecommendNamespacedName: notExistNamespacedName, Metric: ¬ExistTaskIDMetric} +var notExistMetricProcessorKey = common.ProcessKey{ResourceRecommendNamespacedName: mockNamespacedName1, Metric: ¬ExistMetric} +var notFoundTaskProcessorKey = common.ProcessKey{ResourceRecommendNamespacedName: mockNamespacedName2, Metric: ¬ExistTaskIDMetric} +var valueTypeIllegalProcessorKey = common.ProcessKey{ResourceRecommendNamespacedName: mockNamespacedName2, Metric: &valueTypeIllegalMetric} + +var mockProcessConfig1 = common.ProcessConfig{ProcessKey: mockProcessKey1} +var emptyProcessConfig = common.ProcessConfig{ProcessKey: emptyTaskProcessorKey} +var valueTypeIllegalProcessConfig = common.ProcessConfig{ProcessKey: valueTypeIllegalProcessorKey} + +var mockTaskID1 = mockProcessConfig1.GenerateTaskID() +var emptyTaskID = emptyProcessConfig.GenerateTaskID() +var notExistTaskID = "NotExistTaskID" +var valueTypeIllegalTaskID = valueTypeIllegalProcessConfig.GenerateTaskID() + +var mockTask1, _ = task.NewTask(mockMetric1, "") +var mockTask195PercentileValue = 20.40693 +var emptyTask, _ = task.NewTask(emptyTaskMetric, "") + +var mockAggregateTasks = sync.Map{} + +var mockResourceRecommendTaskIDsMap = map[types.NamespacedName]*map[datasource.Metric]common.TaskID{} + +var mockProcessor = Processor{ + AggregateTasks: &mockAggregateTasks, + TaskQueue: workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName), + ResourceRecommendTaskIDsMap: mockResourceRecommendTaskIDsMap, +} + +func init() { + mockAggregateTasks.LoadOrStore(mockTaskID1, mockTask1) + mockAggregateTasks.LoadOrStore(emptyTaskID, emptyTask) + mockAggregateTasks.LoadOrStore(valueTypeIllegalTaskID, 3) + + mockResourceRecommendTaskIDsMap[mockNamespacedName1] = &map[datasource.Metric]common.TaskID{ + mockMetric1: mockTaskID1, + emptyTaskMetric: emptyTaskID, + } + mockResourceRecommendTaskIDsMap[mockNamespacedName2] = &map[datasource.Metric]common.TaskID{ + notExistTaskIDMetric: common.TaskID(notExistTaskID), + valueTypeIllegalMetric: valueTypeIllegalTaskID, + } + + sampleTime1 := time.Now() + mockTask1.AddSample(sampleTime1.Add(-time.Hour*24), 20, 1000) + mockTask1.AddSample(sampleTime1, 10, 1) +} diff --git a/pkg/controller/resource-recommend/processor/percentile/processor_test.go b/pkg/controller/resource-recommend/processor/percentile/processor_test.go new file mode 100644 index 0000000000..d31f5ee406 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/processor_test.go @@ -0,0 +1,370 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package percentile + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/workqueue" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + customError "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/error" +) + +func TestProcessor_Register(t *testing.T) { + tests := []struct { + name string + processConfig *common.ProcessConfig + wantCErr *customError.CustomError + }{ + { + name: "validate err", + processConfig: &common.ProcessConfig{ + ProcessKey: common.ProcessKey{ + ResourceRecommendNamespacedName: types.NamespacedName{ + Namespace: "ns1", + Name: "n1", + }, + }, + }, + wantCErr: customError.RegisterProcessTaskValidateError(errors.New("")), + }, + { + name: "loaded", + processConfig: &mockProcessConfig1, + wantCErr: nil, + }, + { + name: "case1", + processConfig: &common.ProcessConfig{ProcessKey: common.ProcessKey{ + ResourceRecommendNamespacedName: types.NamespacedName{ + Namespace: "testRegisterNamespace1", + Name: "testRegisterName1", + }, + Metric: &datasource.Metric{ + Namespace: "testRegister", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testRegister", + ContainerName: "testRegister", + Resource: "cpu", + }, + }, + }, + wantCErr: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotCErr := mockProcessor.Register(tt.processConfig) + if gotCErr != nil { + if gotCErr.Phase != tt.wantCErr.Phase || gotCErr.Code != tt.wantCErr.Code { + t.Errorf("Register() = %v, want %v", gotCErr, tt.wantCErr) + } + return + } + tasks, ok := mockProcessor.ResourceRecommendTaskIDsMap[tt.processConfig.ResourceRecommendNamespacedName] + if !ok { + t.Errorf("Register() failed, namespaceName not exist") + return + } + id := (*tasks)[*tt.processConfig.Metric] + if id != tt.processConfig.GenerateTaskID() { + t.Errorf("Register() failed, Metric not exist") + return + } + + if _, ok := mockProcessor.AggregateTasks.Load(id); !ok { + t.Errorf("Register() failed, task not store") + return + } + + }) + } +} + +func TestProcessor_Cancel(t *testing.T) { + processor := Processor{ + TaskQueue: workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName), + AggregateTasks: &sync.Map{}, + ResourceRecommendTaskIDsMap: make(map[types.NamespacedName]*map[datasource.Metric]common.TaskID), + } + + ns0 := types.NamespacedName{Namespace: "CancelNamespace0", Name: "CancelName0"} + ns1 := types.NamespacedName{Namespace: "CancelNamespace1", Name: "CancelName1"} + ns2 := types.NamespacedName{Namespace: "CancelNamespace2", Name: "CancelName2"} + ns3 := types.NamespacedName{Namespace: "CancelNamespace3", Name: "CancelName3"} + notFoundNs := types.NamespacedName{Namespace: "NamespacedNameNotFound", Name: "NamespacedNameNotFound"} + + pk0 := common.ProcessKey{ + ResourceRecommendNamespacedName: ns0, + Metric: &datasource.Metric{ + Namespace: "testCancel", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testCancel0", + ContainerName: "testCancel0-1", + Resource: v1.ResourceCPU, + }, + } + pk1 := common.ProcessKey{ + ResourceRecommendNamespacedName: ns1, + Metric: &datasource.Metric{ + Namespace: "testCancel1", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testCancel1", + ContainerName: "testCancel1-1", + Resource: v1.ResourceCPU, + }, + } + pk2 := common.ProcessKey{ + ResourceRecommendNamespacedName: ns1, + Metric: &datasource.Metric{ + Namespace: "testCancel1", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testCancel1", + ContainerName: "testCancel1-1", + Resource: v1.ResourceMemory, + }, + } + pk3 := common.ProcessKey{ + ResourceRecommendNamespacedName: ns2, + Metric: &datasource.Metric{ + Namespace: "testCancel", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testCancel2", + ContainerName: "testCancel2-1", + Resource: v1.ResourceMemory, + }, + } + pk4 := common.ProcessKey{ + ResourceRecommendNamespacedName: ns2, + Metric: &datasource.Metric{ + Namespace: "testCancel", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testCancel2", + ContainerName: "testCancel2-2", + Resource: v1.ResourceMemory, + }, + } + pk5 := common.ProcessKey{ + ResourceRecommendNamespacedName: ns3, + Metric: &datasource.Metric{ + Namespace: "testCancel", + Kind: "deployment", + APIVersion: "v1", + WorkloadName: "testCancel3", + ContainerName: "testCancel3-1", + Resource: v1.ResourceMemory, + }, + } + processConfig0 := common.ProcessConfig{ProcessKey: pk0} + processConfig1 := common.ProcessConfig{ProcessKey: pk1} + processConfig2 := common.ProcessConfig{ProcessKey: pk2} + processConfig3 := common.ProcessConfig{ProcessKey: pk3} + processConfig4 := common.ProcessConfig{ProcessKey: pk4} + processConfig5 := common.ProcessConfig{ProcessKey: pk5} + + _ = processor.Register(&processConfig0) + _ = processor.Register(&processConfig1) + _ = processor.Register(&processConfig2) + _ = processor.Register(&processConfig3) + _ = processor.Register(&processConfig4) + _ = processor.Register(&processConfig5) + + type want struct { + existTaskIDs []common.TaskID + notExistTaskIDs []common.TaskID + namespaceTaskIsNil bool + } + tests := []struct { + name string + testProcessKey *common.ProcessKey + wantCErr *customError.CustomError + want want + }{ + { + name: "case1", + testProcessKey: nil, + wantCErr: nil, + }, + { + name: "NamespacedName not found", + testProcessKey: &common.ProcessKey{ResourceRecommendNamespacedName: notFoundNs}, + wantCErr: customError.NotFoundTasksError(notFoundNs), + }, + { + name: "metric_not_found", + testProcessKey: &common.ProcessKey{ + ResourceRecommendNamespacedName: ns0, + Metric: &datasource.Metric{WorkloadName: "metricNotFound"}, + }, + want: want{ + existTaskIDs: []common.TaskID{processConfig0.GenerateTaskID()}, + notExistTaskIDs: []common.TaskID{}, + namespaceTaskIsNil: false, + }, + }, + { + name: "delete_task", + testProcessKey: &pk1, + wantCErr: nil, + want: want{ + existTaskIDs: []common.TaskID{processConfig2.GenerateTaskID()}, + notExistTaskIDs: []common.TaskID{processConfig1.GenerateTaskID()}, + namespaceTaskIsNil: false, + }, + }, + { + name: "delete_all_task_belong_to_NamespacedName", + testProcessKey: &common.ProcessKey{ResourceRecommendNamespacedName: ns2}, + wantCErr: nil, + want: want{ + existTaskIDs: []common.TaskID{}, + notExistTaskIDs: []common.TaskID{processConfig3.GenerateTaskID(), processConfig4.GenerateTaskID()}, + namespaceTaskIsNil: true, + }, + }, + { + name: "delete_task_and_NamespacedName_in_map", + testProcessKey: &pk5, + wantCErr: nil, + want: want{ + existTaskIDs: []common.TaskID{}, + notExistTaskIDs: []common.TaskID{processConfig5.GenerateTaskID()}, + namespaceTaskIsNil: true, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotCErr := processor.Cancel(tt.testProcessKey) + if gotCErr != nil { + if gotCErr.Phase != tt.wantCErr.Phase || gotCErr.Code != tt.wantCErr.Code { + t.Errorf("Cancel() = %v, want %v", gotCErr, tt.wantCErr) + } + return + } + if tt.testProcessKey == nil && tt.wantCErr == nil { + return + } + tasks, ok := processor.ResourceRecommendTaskIDsMap[tt.testProcessKey.ResourceRecommendNamespacedName] + if tt.want.namespaceTaskIsNil || tt.testProcessKey.Metric == nil { + if ok { + t.Errorf("Cancel() failed, tasks for namespaceName(%s) exist", tt.testProcessKey.ResourceRecommendNamespacedName) + return + } + } else { + if !ok || tasks == nil { + t.Errorf("Cancel() failed, tasks for namespaceName(%s) not exist", tt.testProcessKey.ResourceRecommendNamespacedName) + return + } + if _, ok := (*tasks)[*tt.testProcessKey.Metric]; ok { + t.Errorf("Cancel() failed, task for Metric(%s) exist", *tt.testProcessKey.Metric) + } + } + for _, id := range tt.want.notExistTaskIDs { + if _, ok := processor.AggregateTasks.Load(id); ok { + t.Errorf("Cancel() failed, task for taskID(%s) exist", id) + } + } + for _, id := range tt.want.existTaskIDs { + if _, ok := processor.AggregateTasks.Load(id); !ok { + t.Errorf("Cancel() failed, task for taskID(%s) not exist", id) + } + } + }) + } +} + +func TestProcessor_Run(t *testing.T) { + processor := Processor{ + TaskQueue: workqueue.NewNamedRateLimitingQueue(DefaultQueueRateLimiter, ProcessorName), + AggregateTasks: &sync.Map{}, + ResourceRecommendTaskIDsMap: make(map[types.NamespacedName]*map[datasource.Metric]common.TaskID), + } + type args struct { + ctx context.Context + } + + tests := []struct { + name string + args args + }{ + { + name: "case", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx1, cancelFunc := context.WithCancel(context.Background()) + go processor.Run(ctx1) + time.Sleep(2 * time.Second) + cancelFunc() + }) + } +} + +func TestProcessor_QueryProcessedValues(t *testing.T) { + tests := []struct { + name string + processKey *common.ProcessKey + want float64 + wantErr bool + }{ + { + name: "case1", + processKey: &metricIsNilProcessorKey, + wantErr: true, + }, + { + name: "case2", + processKey: &emptyTaskProcessorKey, + wantErr: true, + }, + { + name: "case3", + processKey: &mockProcessKey1, + wantErr: false, + want: mockTask195PercentileValue, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := mockProcessor.QueryProcessedValues(tt.processKey) + if (err != nil) != tt.wantErr { + t.Errorf("QueryProcessedValues() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr { + assert.InEpsilon(t, tt.want, got, 1e-5) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/processor/percentile/task/config.go b/pkg/controller/resource-recommend/processor/percentile/task/config.go new file mode 100644 index 0000000000..2ebbc988b0 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/task/config.go @@ -0,0 +1,133 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "time" + + "github.com/pkg/errors" + "gopkg.in/yaml.v3" + v1 "k8s.io/api/core/v1" + vpautil "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/util" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" +) + +const ( + // DefaultSampleWeight is the default weight of any sample (prior to including decaying factor) + DefaultSampleWeight = float64(1) + // DefaultMinSampleWeight is the minimal sample weight of any sample (prior to including decaying factor) + DefaultMinSampleWeight = float64(0.1) + // DefaultEpsilon is the minimal weight kept in histograms, it should be small enough that old samples + // (just inside MemoryAggregationWindowLength) added with DefaultMinSampleWeight are still kept + DefaultEpsilon = 0.001 * DefaultMinSampleWeight + // DefaultHistogramBucketSizeGrowth is the default value for HistogramBucketSizeGrowth. + DefaultHistogramBucketSizeGrowth = 0.05 // Make each bucket 5% larger than the previous one. + + // DefaultCPUHistogramMaxValue CPU histograms max bucket size is 1000.0 cores + DefaultCPUHistogramMaxValue = 1000.0 + // DefaultCPUHistogramFirstBucketSize CPU histograms smallest bucket size is 0.01 cores + DefaultCPUHistogramFirstBucketSize = 0.01 + + // DefaultMemHistogramMaxValue Mem histograms max bucket size is 1TB + DefaultMemHistogramMaxValue = 1e12 + // DefaultMemHistogramFirstBucketSize Mem histograms smallest bucket size is 10MB + DefaultMemHistogramFirstBucketSize = 1e7 + + DefaultCPUTaskProcessInterval = time.Minute * 10 + DefaultMemTaskProcessInterval = time.Hour * 1 + + // DefaultHistogramDecayHalfLife is the default value for HistogramDecayHalfLife. + DefaultHistogramDecayHalfLife = time.Hour * 24 + + // DefaultInitDataLength is default data query span for the first run of the task + DefaultInitDataLength = time.Hour * 25 + + // MaxOfAllowNotExecutionTime is maximum non-run tolerance time for a task + MaxOfAllowNotExecutionTime = 50 * time.Hour +) + +func cpuHistogramOptions() vpautil.HistogramOptions { + // CPU histograms use exponential bucketing scheme with the smallest bucket size of 0.01 core, max of 1000.0 cores + options, err := vpautil.NewExponentialHistogramOptions(DefaultCPUHistogramMaxValue, DefaultCPUHistogramFirstBucketSize, + 1.+DefaultHistogramBucketSizeGrowth, DefaultEpsilon) + if err != nil { + panic("Invalid CPU histogram options") // Should not happen. + } + return options +} + +func memoryHistogramOptions() vpautil.HistogramOptions { + // Memory histograms use exponential bucketing scheme with the smallest bucket size of 10MB, max of 1TB + options, err := vpautil.NewExponentialHistogramOptions(DefaultMemHistogramMaxValue, DefaultMemHistogramFirstBucketSize, + 1.+DefaultHistogramBucketSizeGrowth, DefaultEpsilon) + if err != nil { + panic("Invalid memory histogram options") // Should not happen. + } + return options +} + +func HistogramOptionsFactory(resourceName v1.ResourceName) (vpautil.HistogramOptions, error) { + switch resourceName { + case v1.ResourceCPU: + return cpuHistogramOptions(), nil + case v1.ResourceMemory: + return memoryHistogramOptions(), nil + default: + return nil, errors.Errorf("generate histogram options failed, unknow resource: %s", resourceName) + } +} + +func GetDefaultTaskProcessInterval(resourceName v1.ResourceName) (t time.Duration, err error) { + switch resourceName { + case v1.ResourceCPU: + return DefaultCPUTaskProcessInterval, nil + case v1.ResourceMemory: + return DefaultMemTaskProcessInterval, nil + default: + return t, errors.Errorf("get task process interval failed, unknow resource: %s", resourceName) + } +} + +type ProcessConfig struct { + DecayHalfLife time.Duration +} + +const ( + ProcessConfigHalfLifeKey = "decayHalfLife" +) + +func GetTaskConfig(extensions common.TaskConfigStr) (*ProcessConfig, error) { + processConfig := &ProcessConfig{ + DecayHalfLife: DefaultHistogramDecayHalfLife, + } + if extensions == "" { + return processConfig, nil + } + config := make(map[string]interface{}) + if err := yaml.Unmarshal([]byte(extensions), config); err != nil { + return nil, errors.Wrap(err, "unmarshal process config failed") + } + + if value, ok := config[ProcessConfigHalfLifeKey]; ok { + if halfLife, ok := value.(int); ok { + processConfig.DecayHalfLife = time.Hour * time.Duration(halfLife) + } + } + + return processConfig, nil +} diff --git a/pkg/controller/resource-recommend/processor/percentile/task/config_test.go b/pkg/controller/resource-recommend/processor/percentile/task/config_test.go new file mode 100644 index 0000000000..2fda8f90e8 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/task/config_test.go @@ -0,0 +1,190 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "fmt" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" +) + +func TestGetProcessConfig(t *testing.T) { + type args struct { + extensions common.TaskConfigStr + } + tests := []struct { + name string + args args + want *ProcessConfig + wantErr bool + }{ + { + name: "case-1", + args: args{ + extensions: common.TaskConfigStr(`{"decayHalfLife":222,"key2":123}`), + }, + want: &ProcessConfig{ + DecayHalfLife: time.Hour * 24, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetTaskConfig(tt.args.extensions) + if (err != nil) != tt.wantErr { + t.Errorf("GetTaskConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + fmt.Printf("%v", got) + }) + } +} + +func TestHistogramOptionsFactory(t *testing.T) { + type want struct { + BucketsNum int + BucketFind struct { + value float64 + bucket int + } + StartBucket struct { + bucket int + value float64 + } + Epsilon float64 + } + tests := []struct { + name string + args v1.ResourceName + want want + wantErr bool + }{ + { + name: "case1", + args: v1.ResourceCPU, + want: want{ + BucketsNum: 176, + BucketFind: struct { + value float64 + bucket int + }{value: 10, bucket: 80}, + StartBucket: struct { + bucket int + value float64 + }{bucket: 10, value: 0.1257789253554882}, + Epsilon: DefaultEpsilon, + }, + wantErr: false, + }, + { + name: "case2", + args: v1.ResourceMemory, + want: want{ + BucketsNum: 176, + BucketFind: struct { + value float64 + bucket int + }{value: 976875765, bucket: 36}, + StartBucket: struct { + bucket int + value float64 + }{bucket: 90, value: 1.5946073009825298e+10}, + Epsilon: DefaultEpsilon, + }, + wantErr: false, + }, + { + name: "case3", + args: v1.ResourceName("errName"), + want: want{}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := HistogramOptionsFactory(tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("HistogramOptionsFactory() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.wantErr { + return + } + if NumBuckets := got.NumBuckets(); NumBuckets != tt.want.BucketsNum { + t.Errorf("HistogramOptionsFactory() NumBuckets gotT = %v, want %v", NumBuckets, tt.want.BucketsNum) + //return + } + if gotT := got.FindBucket(tt.want.BucketFind.value); gotT != tt.want.BucketFind.bucket { + t.Errorf("HistogramOptionsFactory() FindBucket gotT = %v, want %v", gotT, tt.want.BucketFind.bucket) + //return + } + if gotT := got.GetBucketStart(tt.want.StartBucket.bucket); gotT != tt.want.StartBucket.value { + t.Errorf("HistogramOptionsFactory() GetBucketStart gotT = %v, want %v", gotT, tt.want.StartBucket.value) + //return + } + if gotT := got.Epsilon(); gotT != tt.want.Epsilon { + t.Errorf("HistogramOptionsFactory() Epsilon gotT = %v, want %v", gotT, tt.want.Epsilon) + //return + } + }) + } +} + +func TestGetDefaultTaskProcessInterval(t *testing.T) { + + tests := []struct { + name string + args v1.ResourceName + wantT time.Duration + wantErr bool + }{ + { + name: "case1", + args: v1.ResourceCPU, + wantT: DefaultCPUTaskProcessInterval, + wantErr: false, + }, + { + name: "case2", + args: v1.ResourceMemory, + wantT: DefaultMemTaskProcessInterval, + wantErr: false, + }, + { + name: "case3", + args: v1.ResourceName("errName"), + wantT: 0, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotT, err := GetDefaultTaskProcessInterval(tt.args) + if (err != nil) != tt.wantErr { + t.Errorf("GetDefaultTaskProcessInterval() error = %v, wantErr %v", err, tt.wantErr) + return + } + if gotT != tt.wantT { + t.Errorf("GetDefaultTaskProcessInterval() gotT = %v, want %v", gotT, tt.wantT) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/processor/percentile/task/histogram_task.go b/pkg/controller/resource-recommend/processor/percentile/task/histogram_task.go new file mode 100644 index 0000000000..1c879805e5 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/task/histogram_task.go @@ -0,0 +1,191 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "context" + "fmt" + "math" + "runtime/debug" + "sync" + "time" + + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + vpautil "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/util" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + datasourceutils "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/utils" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/utils/log" +) + +var ( + DataPreparingErr = errors.New("data preparing") + SampleExpirationErr = errors.New("no samples in the last 24 hours") + InsufficientSampleErr = errors.New("The data sample is insufficient to obtain the predicted value") + QuerySamplesIsEmptyErr = errors.New("the sample found is empty") + HistogramTaskRunPanicErr = errors.New("histogram task run panic") +) + +type HistogramTask struct { + mutex sync.Mutex + + histogram vpautil.Histogram + decayHalfLife time.Duration + firstSampleTime time.Time + lastSampleTime time.Time + totalSamplesCount int + + createTime time.Time + lastRunTime time.Time + + metric datasource.Metric + ProcessInterval time.Duration +} + +func NewTask(metric datasource.Metric, config common.TaskConfigStr) (*HistogramTask, error) { + processConfig, err := GetTaskConfig(config) + if err != nil { + return nil, errors.Wrapf(err, "New histogram task error, get process config failed, config: %s", config) + } + histogramOptions, err := HistogramOptionsFactory(metric.Resource) + if err != nil { + return nil, errors.Wrap(err, "get histogram options failed") + } + taskProcessInterval, err := GetDefaultTaskProcessInterval(metric.Resource) + if err != nil { + return nil, errors.Wrap(err, "get process interval failed") + } + return &HistogramTask{ + mutex: sync.Mutex{}, + metric: metric, + histogram: vpautil.NewDecayingHistogram(histogramOptions, processConfig.DecayHalfLife), + decayHalfLife: processConfig.DecayHalfLife, + createTime: time.Now(), + ProcessInterval: taskProcessInterval, + }, nil +} + +func (t *HistogramTask) AddSample(sampleTime time.Time, sampleValue float64, sampleWeight float64) { + t.histogram.AddSample(sampleValue, sampleWeight, sampleTime) + if t.lastSampleTime.Before(sampleTime) { + t.lastSampleTime = sampleTime + } + if t.firstSampleTime.IsZero() || t.firstSampleTime.After(sampleTime) { + t.firstSampleTime = sampleTime + } + t.totalSamplesCount++ +} + +func (t *HistogramTask) AddRangeSample(firstSampleTime, lastSampleTime time.Time, sampleValue float64, sampleWeight float64) { + t.histogram.AddSample(sampleValue, sampleWeight, lastSampleTime) + if t.lastSampleTime.Before(lastSampleTime) { + t.lastSampleTime = lastSampleTime + } + if t.firstSampleTime.IsZero() || t.firstSampleTime.After(firstSampleTime) { + t.firstSampleTime = firstSampleTime + } + t.totalSamplesCount++ +} + +func (t *HistogramTask) Run(ctx context.Context, datasourceProxy *datasource.Proxy) (nextRunInterval time.Duration, err error) { + defer func() { + if r := recover(); r != nil { + log.ErrorS(ctx, HistogramTaskRunPanicErr, fmt.Sprintf("%v", r), "stack", string(debug.Stack())) + err = HistogramTaskRunPanicErr + } + }() + + t.mutex.Lock() + defer t.mutex.Unlock() + + log.InfoS(ctx, "percentile process task run") + + runSectionBegin := t.lastRunTime + runSectionEnd := time.Now() + t.lastRunTime = runSectionEnd + if runSectionBegin.IsZero() { + runSectionBegin = runSectionEnd.Add(-DefaultInitDataLength) + } + ctx = log.SetKeysAndValues(ctx, "runSectionBegin", runSectionBegin.String(), "runSectionEnd", runSectionEnd.String()) + + timeSeries, err := datasourceProxy.QueryTimeSeries(datasource.PrometheusDatasource, t.metric, runSectionBegin, runSectionEnd, time.Minute) + if err != nil { + log.ErrorS(ctx, err, "task handler error, query samples failed") + return 0, err + } + samplesOverview := datasourceutils.GetSamplesOverview(timeSeries) + if samplesOverview == nil { + log.ErrorS(ctx, QuerySamplesIsEmptyErr, "No sample found, histogram task run termination") + return 0, QuerySamplesIsEmptyErr + } + log.InfoS(ctx, "percentile process task query samples", "samplesOverview", samplesOverview) + + switch t.metric.Resource { + case v1.ResourceMemory: + lastSampleTime := time.Unix(samplesOverview.LastTimestamp, 0) + firstSampleTime := time.Unix(samplesOverview.FirstTimestamp, 0) + t.AddRangeSample(firstSampleTime, lastSampleTime, samplesOverview.MaxValue, DefaultSampleWeight) + log.V(5).InfoS(ctx, "Mem peak sample added.", + "firstSampleTime", firstSampleTime, "lastSampleTime", lastSampleTime, + "SampleValue", samplesOverview.MaxValue, "SampleNum", len(timeSeries.Samples)) + case v1.ResourceCPU: + for _, sample := range timeSeries.Samples { + sampleTime := time.Unix(sample.Timestamp, 0) + sampleWeight := math.Max(sample.Value, DefaultMinSampleWeight) + t.AddSample(sampleTime, sample.Value, sampleWeight) + log.V(5).InfoS(ctx, "CPU sample added", + "sampleTime", sampleTime, "sampleWeight", sampleWeight, "SampleValue", sample.Value) + } + } + + log.InfoS(ctx, "percentile process task run finished") + return t.ProcessInterval, nil +} + +func (t *HistogramTask) QueryPercentileValue(ctx context.Context, percentile float64) (float64, error) { + t.mutex.Lock() + defer t.mutex.Unlock() + + if t.firstSampleTime.IsZero() { + return 0, DataPreparingErr + } + if time.Now().Sub(t.lastSampleTime) > 24*time.Hour { + return 0, SampleExpirationErr + } + if t.lastSampleTime.Sub(t.firstSampleTime) < time.Hour*24 { + return 0, InsufficientSampleErr + } + //TODO: Check whether the ratio between the count of samples and the running time meets the requirements + + percentileValue := t.histogram.Percentile(percentile) + + log.InfoS(ctx, "Query Processed Values", + "lastSampleTime", t.lastSampleTime, "firstSampleTime", t.firstSampleTime, + "totalSamplesCount", t.totalSamplesCount, "percentileValue", percentileValue) + return percentileValue, nil +} + +func (t *HistogramTask) IsTimeoutNotExecute() bool { + baseTime := t.createTime + if !t.lastRunTime.IsZero() { + baseTime = t.lastRunTime + } + // If there is no execution record within 10 task execution intervals, the task is regarded as invalid + return baseTime.Add(MaxOfAllowNotExecutionTime).Before(time.Now()) +} diff --git a/pkg/controller/resource-recommend/processor/percentile/task/histogram_task_test.go b/pkg/controller/resource-recommend/processor/percentile/task/histogram_task_test.go new file mode 100644 index 0000000000..3921920fd7 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/percentile/task/histogram_task_test.go @@ -0,0 +1,489 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package task + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" +) + +func TestHistogramTask_AddSample(t1 *testing.T) { + type newTask func() *HistogramTask + type sample struct { + sampleTime time.Time + sampleValue float64 + sampleWeight float64 + } + type want struct { + lastSampleTime time.Time + firstSampleTime time.Time + totalSamplesCount int + } + tests := []struct { + name string + newTask newTask + samples []sample + want + }{ + { + name: "case1", + newTask: func() *HistogramTask { + t, _ := NewTask(datasource.Metric{Resource: v1.ResourceCPU}, "") + return t + }, + samples: []sample{ + { + sampleTime: time.Unix(1694270256, 0), + sampleValue: 1, + sampleWeight: 1, + }, + { + sampleTime: time.Unix(1694270765, 0), + sampleValue: 1, + sampleWeight: 1, + }, + { + sampleTime: time.Unix(1694278758, 0), + sampleValue: 1, + sampleWeight: 1, + }, + { + sampleTime: time.Unix(1694278700, 0), + sampleValue: 1, + sampleWeight: 1, + }, + }, + want: want{ + lastSampleTime: time.Unix(1694278758, 0), + firstSampleTime: time.Unix(1694270256, 0), + totalSamplesCount: 4, + }, + }, + } + for _, tt := range tests { + t1.Run(tt.name, func(t1 *testing.T) { + t := tt.newTask() + for _, sample := range tt.samples { + t.AddSample(sample.sampleTime, sample.sampleValue, sample.sampleWeight) + } + if tt.lastSampleTime != t.lastSampleTime { + t1.Errorf("AddSample() got lastSampleTime=%s, want lastSampleTime=%s", t.lastSampleTime, tt.lastSampleTime) + } + if tt.firstSampleTime != t.firstSampleTime { + t1.Errorf("AddSample() got firstSampleTime=%s, want firstSampleTime=%s", t.firstSampleTime, tt.firstSampleTime) + } + if tt.totalSamplesCount != t.totalSamplesCount { + t1.Errorf("AddSample() got totalSamplesCount=%d, want totalSamplesCount=%d", t.totalSamplesCount, tt.totalSamplesCount) + } + }) + } +} + +func TestHistogramTask_AddRangeSample(t1 *testing.T) { + type newTask func() *HistogramTask + type sample struct { + firstSampleTime time.Time + lastSampleTime time.Time + sampleValue float64 + sampleWeight float64 + } + type want struct { + lastSampleTime time.Time + firstSampleTime time.Time + totalSamplesCount int + } + tests := []struct { + name string + newTask newTask + samples []sample + want + }{ + { + name: "case1", + newTask: func() *HistogramTask { + t, _ := NewTask(datasource.Metric{Resource: v1.ResourceCPU}, "") + return t + }, + samples: []sample{ + { + firstSampleTime: time.Unix(1694270256, 0), + lastSampleTime: time.Unix(1694270768, 0), + sampleValue: 1, + sampleWeight: 1, + }, + { + firstSampleTime: time.Unix(1694270769, 0), + lastSampleTime: time.Unix(1694271769, 0), + sampleValue: 1, + sampleWeight: 1, + }, + }, + want: want{ + lastSampleTime: time.Unix(1694271769, 0), + firstSampleTime: time.Unix(1694270256, 0), + totalSamplesCount: 2, + }, + }, + } + for _, tt := range tests { + t1.Run(tt.name, func(t1 *testing.T) { + t := tt.newTask() + for _, sample := range tt.samples { + t.AddRangeSample(sample.firstSampleTime, sample.lastSampleTime, sample.sampleValue, sample.sampleWeight) + } + if tt.lastSampleTime != t.lastSampleTime { + t1.Errorf("AddSample() got lastSampleTime=%s, want lastSampleTime=%s", t.lastSampleTime, tt.lastSampleTime) + } + if tt.firstSampleTime != t.firstSampleTime { + t1.Errorf("AddSample() got firstSampleTime=%s, want firstSampleTime=%s", t.firstSampleTime, tt.firstSampleTime) + } + if tt.totalSamplesCount != t.totalSamplesCount { + t1.Errorf("AddSample() got totalSamplesCount=%d, want totalSamplesCount=%d", t.totalSamplesCount, tt.totalSamplesCount) + } + }) + } +} + +type mockDatasourceEmptyQuery struct{} + +func (m1 *mockDatasourceEmptyQuery) QueryTimeSeries(_ *datasource.Query, _ time.Time, _ time.Time, _ time.Duration) (*datasource.TimeSeries, error) { + return &datasource.TimeSeries{Samples: []datasource.Sample{}}, nil +} + +func (m1 *mockDatasourceEmptyQuery) ConvertMetricToQuery(metric datasource.Metric) (*datasource.Query, error) { + return nil, nil +} + +type mockDatasourcePanic struct{} + +func (m1 *mockDatasourcePanic) QueryTimeSeries(_ *datasource.Query, _ time.Time, _ time.Time, _ time.Duration) (*datasource.TimeSeries, error) { + panic("test panic") +} + +func (m1 *mockDatasourcePanic) ConvertMetricToQuery(metric datasource.Metric) (*datasource.Query, error) { + return nil, nil +} + +type mockDatasource struct{} + +func (m1 *mockDatasource) QueryTimeSeries(_ *datasource.Query, _ time.Time, _ time.Time, _ time.Duration) (*datasource.TimeSeries, error) { + return &datasource.TimeSeries{Samples: []datasource.Sample{ + { + Timestamp: 1694270256, + Value: 1, + }, + { + Timestamp: 1694270765, + Value: 2, + }, + { + Timestamp: 1694278758, + Value: 3, + }, + { + Timestamp: 1694278700, + Value: 4, + }, + }}, nil +} + +func (m1 *mockDatasource) ConvertMetricToQuery(metric datasource.Metric) (*datasource.Query, error) { + return nil, nil +} + +func TestHistogramTask_Run(t1 *testing.T) { + type newTask func() *HistogramTask + type args struct { + ctx context.Context + datasourceProxy func() *datasource.Proxy + } + type want struct { + lastSampleTime time.Time + firstSampleTime time.Time + totalSamplesCount int + } + tests := []struct { + name string + newTask newTask + args args + wantNextRunInterval time.Duration + wantErr error + want want + }{ + { + name: "empty samples", + newTask: func() *HistogramTask { + task, _ := NewTask(datasource.Metric{Resource: v1.ResourceCPU}, "") + return task + }, + args: args{ + ctx: context.Background(), + datasourceProxy: func() *datasource.Proxy { + proxy := datasource.NewProxy() + proxy.RegisterDatasource(datasource.PrometheusDatasource, &mockDatasourceEmptyQuery{}) + return proxy + }, + }, + wantNextRunInterval: 0, + wantErr: QuerySamplesIsEmptyErr, + }, + { + name: "panic", + newTask: func() *HistogramTask { + task, _ := NewTask(datasource.Metric{Resource: v1.ResourceCPU}, "") + return task + }, + args: args{ + ctx: context.Background(), + datasourceProxy: func() *datasource.Proxy { + proxy := datasource.NewProxy() + proxy.RegisterDatasource(datasource.PrometheusDatasource, &mockDatasourcePanic{}) + return proxy + }, + }, + wantNextRunInterval: 0, + wantErr: HistogramTaskRunPanicErr, + }, + { + name: "cpu run", + newTask: func() *HistogramTask { + task, _ := NewTask(datasource.Metric{Resource: v1.ResourceCPU}, "") + return task + }, + args: args{ + ctx: context.Background(), + datasourceProxy: func() *datasource.Proxy { + proxy := datasource.NewProxy() + proxy.RegisterDatasource(datasource.PrometheusDatasource, &mockDatasource{}) + return proxy + }, + }, + wantNextRunInterval: DefaultCPUTaskProcessInterval, + wantErr: nil, + want: want{ + lastSampleTime: time.Unix(1694278758, 0), + firstSampleTime: time.Unix(1694270256, 0), + totalSamplesCount: 4, + }, + }, + { + name: "mem run", + newTask: func() *HistogramTask { + task, _ := NewTask(datasource.Metric{Resource: v1.ResourceMemory}, "") + return task + }, + args: args{ + ctx: context.Background(), + datasourceProxy: func() *datasource.Proxy { + proxy := datasource.NewProxy() + proxy.RegisterDatasource(datasource.PrometheusDatasource, &mockDatasource{}) + return proxy + }, + }, + wantNextRunInterval: DefaultMemTaskProcessInterval, + wantErr: nil, + want: want{ + lastSampleTime: time.Unix(1694278758, 0), + firstSampleTime: time.Unix(1694270256, 0), + totalSamplesCount: 1, + }, + }, + } + for _, tt := range tests { + t1.Run(tt.name, func(t1 *testing.T) { + t := tt.newTask() + gotNextRunInterval, err := t.Run(tt.args.ctx, tt.args.datasourceProxy()) + if tt.wantErr != nil && err.Error() != tt.wantErr.Error() { + t1.Errorf("Run() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.wantErr == nil { + if gotNextRunInterval != tt.wantNextRunInterval { + t1.Errorf("Run() gotNextRunInterval = %v, want %v", gotNextRunInterval, tt.wantNextRunInterval) + } + if tt.want.lastSampleTime != t.lastSampleTime { + t1.Errorf("Run() got lastSampleTime=%s, want lastSampleTime=%s", t.lastSampleTime, tt.want.lastSampleTime) + } + if tt.want.firstSampleTime != t.firstSampleTime { + t1.Errorf("Run() got firstSampleTime=%s, want firstSampleTime=%s", t.firstSampleTime, tt.want.firstSampleTime) + } + if tt.want.totalSamplesCount != t.totalSamplesCount { + t1.Errorf("Run() got totalSamplesCount=%d, want totalSamplesCount=%d", t.totalSamplesCount, tt.want.totalSamplesCount) + } + } + }) + } +} + +func TestHistogramTask_QueryPercentileValue(t1 *testing.T) { + type newTask func() *HistogramTask + type args struct { + ctx context.Context + percentile float64 + } + tests := []struct { + name string + newTask newTask + args args + want float64 + wantErr error + }{ + { + name: "case0", + newTask: func() *HistogramTask { + return &HistogramTask{ + lastSampleTime: time.Now().Add(-25 * time.Hour), + } + }, + args: args{ + ctx: context.Background(), + percentile: 0.9, + }, + want: 0.0, + wantErr: DataPreparingErr, + }, + { + name: "case1", + newTask: func() *HistogramTask { + return &HistogramTask{ + firstSampleTime: time.Now().Add(-26 * time.Hour), + lastSampleTime: time.Now().Add(-25 * time.Hour), + } + }, + args: args{ + ctx: context.Background(), + percentile: 0.9, + }, + want: 0.0, + wantErr: SampleExpirationErr, + }, + { + name: "case2", + newTask: func() *HistogramTask { + return &HistogramTask{ + lastSampleTime: time.Now(), + firstSampleTime: time.Now().Add(-20 * time.Hour), + } + }, + args: args{ + ctx: context.Background(), + percentile: 0.9, + }, + want: 0.0, + wantErr: InsufficientSampleErr, + }, + { + name: "case3", + newTask: func() *HistogramTask { + task, _ := NewTask(datasource.Metric{Resource: v1.ResourceCPU}, "") + sampleTime := time.Now() + task.AddSample(sampleTime.Add(-time.Hour*24), 20, 1000) + task.AddSample(sampleTime, 10, 1) + return task + }, + args: args{ + ctx: context.Background(), + percentile: 0.9, + }, + want: 20.40693, + wantErr: nil, + }, + { + name: "case4", + newTask: func() *HistogramTask { + task, _ := NewTask(datasource.Metric{Resource: v1.ResourceCPU}, "") + sampleTime := time.Now() + task.AddSample(sampleTime.Add(-time.Hour*24*20), 20, 1000) + task.AddSample(sampleTime, 10, 1) + return task + }, + args: args{ + ctx: context.Background(), + percentile: 0.9, + }, + want: 10.207902, + wantErr: nil, + }, + } + for _, tt := range tests { + t1.Run(tt.name, func(t1 *testing.T) { + t := tt.newTask() + got, err := t.QueryPercentileValue(tt.args.ctx, tt.args.percentile) + if tt.wantErr != nil && err.Error() != tt.wantErr.Error() { + t1.Errorf("QueryPercentileValue() error = %v, WantErr %v", err, tt.wantErr) + return + } + + if tt.wantErr == nil { + assert.InEpsilon(t1, tt.want, got, 1e-5) + } + }) + } +} + +func TestHistogramTask_IsTimeoutNotExecute(t1 *testing.T) { + type newTask func() *HistogramTask + tests := []struct { + name string + newFunc newTask + want bool + }{ + { + name: "case1", + newFunc: func() *HistogramTask { + lastRunTime := time.Now() + return &HistogramTask{ + lastRunTime: lastRunTime, + createTime: lastRunTime.Add(-48 * time.Minute), + } + }, + want: false, + }, + { + name: "case1", + newFunc: func() *HistogramTask { + return &HistogramTask{ + lastRunTime: time.Now().Add(-(MaxOfAllowNotExecutionTime + time.Minute)), + } + }, + want: true, + }, + { + name: "case2", + newFunc: func() *HistogramTask { + return &HistogramTask{ + createTime: time.Now().Add(-(MaxOfAllowNotExecutionTime + time.Minute)), + } + }, + want: true, + }, + } + for _, tt := range tests { + t1.Run(tt.name, func(t1 *testing.T) { + t := tt.newFunc() + if got := t.IsTimeoutNotExecute(); got != tt.want { + t1.Errorf("IsTimeoutNotExecute() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/processor/processor.go b/pkg/controller/resource-recommend/processor/processor.go new file mode 100644 index 0000000000..eef8499424 --- /dev/null +++ b/pkg/controller/resource-recommend/processor/processor.go @@ -0,0 +1,35 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package processor + +import ( + "context" + + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + customError "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/error" +) + +type Processor interface { + // Run performs the prediction routine. + Run(ctx context.Context) + + Register(processConfig *common.ProcessConfig) *customError.CustomError + + Cancel(processKey *common.ProcessKey) *customError.CustomError + + QueryProcessedValues(taskKey *common.ProcessKey) (float64, error) +} diff --git a/pkg/controller/resource-recommend/recommender/manager/recommender_manager.go b/pkg/controller/resource-recommend/recommender/manager/recommender_manager.go new file mode 100644 index 0000000000..dae6a6c0a2 --- /dev/null +++ b/pkg/controller/resource-recommend/recommender/manager/recommender_manager.go @@ -0,0 +1,47 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "k8s.io/klog/v2" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/oom" + processormanager "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/manager" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/recommender" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/recommender/recommenders" +) + +type Manager struct { + ProcessorManager processormanager.Manager + OomRecorder oom.Recorder +} + +func NewManager(ProcessorManager processormanager.Manager, OomRecorder oom.Recorder) *Manager { + return &Manager{ + ProcessorManager: ProcessorManager, + OomRecorder: OomRecorder} +} + +func (m *Manager) NewRecommender(algorithm v1alpha1.Algorithm) recommender.Recommender { + switch algorithm { + case v1alpha1.AlgorithmPercentile: + return recommenders.NewPercentileRecommender(m.ProcessorManager.GetProcessor(v1alpha1.AlgorithmPercentile), m.OomRecorder) + } + klog.InfoS("no recommender matched. fall through to default percentile recommender") + return recommenders.NewPercentileRecommender(m.ProcessorManager.GetProcessor(v1alpha1.AlgorithmPercentile), m.OomRecorder) +} diff --git a/pkg/controller/resource-recommend/recommender/recommender.go b/pkg/controller/resource-recommend/recommender/recommender.go new file mode 100644 index 0000000000..61f57f9a55 --- /dev/null +++ b/pkg/controller/resource-recommend/recommender/recommender.go @@ -0,0 +1,26 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommender + +import ( + customError "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/error" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/recommendation" +) + +type Recommender interface { + Recommend(recommendation *recommendation.Recommendation) *customError.CustomError +} diff --git a/pkg/controller/resource-recommend/recommender/recommenders/percentile_recommender.go b/pkg/controller/resource-recommend/recommender/recommenders/percentile_recommender.go new file mode 100644 index 0000000000..1a58ed19e7 --- /dev/null +++ b/pkg/controller/resource-recommend/recommender/recommenders/percentile_recommender.go @@ -0,0 +1,190 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommenders + +import ( + "strings" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + vpamodel "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/recommender/model" + "k8s.io/klog/v2" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource/prometheus/auth/volc-engine/env" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/oom" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/recommender" + customError "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/error" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/recommendation" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/utils" +) + +type PercentileRecommender struct { + recommender.Recommender + DataProcessor processor.Processor + OomRecorder oom.Recorder +} + +const ( + // OOMBumpUpRatio specifies how much memory will be added after observing OOM. + OOMBumpUpRatio float64 = 1.2 + // OOMMinBumpUp specifies minimal increase of memory after observing OOM. + OOMMinBumpUp float64 = 100 * 1024 * 1024 // 100MB +) + +// NewPercentileRecommender returns a +func NewPercentileRecommender(DataProcessor processor.Processor, OomRecorder oom.Recorder) *PercentileRecommender { + return &PercentileRecommender{ + DataProcessor: DataProcessor, + OomRecorder: OomRecorder} +} + +func (r *PercentileRecommender) Recommend(recommendation *recommendation.Recommendation) *customError.CustomError { + klog.InfoS("starting recommenders process", "recommendationConfig", recommendation.Config) + recommendationConfig := recommendation.Config + cluster := env.GetEnvWithDefault("VOLC_CLUSTER_ID", "") + for _, container := range recommendationConfig.Containers { + containerRecommendation := v1alpha1.ContainerResources{ + ContainerName: container.ContainerName, + } + requests := v1alpha1.ContainerResourceList{ + Target: map[v1.ResourceName]resource.Quantity{}, + } + for _, containerConfig := range container.ContainerConfigs { + taskKey := &common.ProcessKey{ + ResourceRecommendNamespacedName: types.NamespacedName{ + Name: recommendation.Name, + Namespace: recommendation.Namespace, + }, + Metric: &datasource.Metric{ + Namespace: recommendation.Namespace, + Kind: recommendation.TargetRef.Kind, + APIVersion: recommendation.TargetRef.APIVersion, + WorkloadName: recommendation.TargetRef.Name, + ContainerName: container.ContainerName, + Resource: containerConfig.ControlledResource, + }, + } + if len(cluster) != 0 { + selectors := make(map[string]string) + selectors["cluster"] = cluster + taskKey.SetSelector(selectors) + } + switch containerConfig.ControlledResource { + case v1.ResourceCPU: + cpuQuantity, err := r.getCpuTargetPercentileEstimationWithUsageBuffer(taskKey, float64(containerConfig.ResourceBufferPercent)/100) + if err != nil { + return customError.RecommendationNotReadyError(err.Error()) + } + klog.InfoS("got recommended cpu for container", "recommendedCPU", cpuQuantity.String(), "container", container.ContainerName) + requests.Target[v1.ResourceCPU] = *cpuQuantity + case v1.ResourceMemory: + memQuantity, err := r.getMemTargetPercentileEstimationWithUsageBuffer(taskKey, float64(containerConfig.ResourceBufferPercent)/100) + if err != nil { + return customError.RecommendationNotReadyError(err.Error()) + } + requests.Target[v1.ResourceMemory] = *memQuantity + } + } + containerRecommendation.Requests = &requests + recommendation.Recommendations = append(recommendation.Recommendations, containerRecommendation) + } + klog.InfoS("recommenders process done", "recommendation", utils.StructToString(recommendation.Recommendations)) + return nil +} + +func (r *PercentileRecommender) ScaleOnOOM(oomRecords []oom.OOMRecord, namespace string, workloadName string, containerName string) *resource.Quantity { + klog.InfoS("scaling on oom for namespace, workload, container", "namespace", namespace, "workload", workloadName, "container", containerName) + var oomRecord *oom.OOMRecord + for _, record := range oomRecords { + // use oomRecord for all pods in workload + if strings.HasPrefix(record.Pod, workloadName) && containerName == record.Container && namespace == record.Namespace { + oomRecord = &record + break + } + } + + // ignore too old oom events + if oomRecord != nil && time.Since(oomRecord.OOMAt) <= (time.Hour*24*7) { + memoryOOM := oomRecord.Memory.Value() + var memoryNeeded vpamodel.ResourceAmount + memoryNeeded = vpamodel.ResourceAmountMax(vpamodel.ResourceAmount(memoryOOM)+vpamodel.MemoryAmountFromBytes(OOMMinBumpUp), + vpamodel.ScaleResource(vpamodel.ResourceAmount(memoryOOM), OOMBumpUpRatio)) + + return r.getMemQuantity(float64(memoryNeeded)) + } + + return nil +} + +func (r *PercentileRecommender) getCpuTargetPercentileEstimationWithUsageBuffer(taskKey *common.ProcessKey, resourceBufferPercentage float64) (quantity *resource.Quantity, err error) { + klog.InfoS("getting cpu estimation for namespace, workload, container, with resource buffer", "namespace", taskKey.Namespace, "workload", taskKey.WorkloadName, "container", taskKey.ContainerName, "resourceBuffer", resourceBufferPercentage) + cpuRecommendedValue, err := r.DataProcessor.QueryProcessedValues(taskKey) + if err != nil { + return nil, err + } + klog.InfoS("got cpu recommended value from processor", "cpuRecommendedValue", cpuRecommendedValue) + //scale cpu resource based on usageBuffer + cpuRecommendedValue = cpuRecommendedValue * (1 + resourceBufferPercentage) + klog.InfoS("scaled cpu recommended value for container", "container", taskKey.ContainerName, "resourceBuffer", resourceBufferPercentage, "cpuRecommendedValue", cpuRecommendedValue) + cpuQuantity := resource.NewMilliQuantity(int64(cpuRecommendedValue*1000), resource.DecimalSI) + return cpuQuantity, nil +} + +func (r *PercentileRecommender) getMemTargetPercentileEstimationWithUsageBuffer(taskKey *common.ProcessKey, resourceBufferPercentage float64) (quantity *resource.Quantity, err error) { + klog.InfoS("getting mem estimation for namespace, workload, container, with resource buffer", "namespace", taskKey.Namespace, "workload", taskKey.WorkloadName, "container", taskKey.ContainerName, "resourceBuffer", resourceBufferPercentage) + memRecommendedValue, err := r.DataProcessor.QueryProcessedValues(taskKey) + if err != nil { + return nil, err + } + klog.InfoS("got mem recommended value from processor", "memRecommendedValue", memRecommendedValue) + //scale mem resource based on usageBuffer + memRecommendedValue = memRecommendedValue * (1 + resourceBufferPercentage) + klog.InfoS("scaled mem recommended value for container", "container", taskKey.ContainerName, "resourceBuffer", resourceBufferPercentage, "memRecommendedValue", memRecommendedValue) + memQuantity := r.getMemQuantity(memRecommendedValue) + klog.InfoS("got recommended memory for container", "container", taskKey.ContainerName, "memory", memQuantity.String()) + oomRecords := r.OomRecorder.ListOOMRecords() + oomScaledMem := r.ScaleOnOOM(oomRecords, taskKey.Namespace, taskKey.WorkloadName, taskKey.ContainerName) + if oomScaledMem != nil && !oomScaledMem.IsZero() && oomScaledMem.Cmp(*memQuantity) > 0 { + klog.InfoS("container using oomProtect Memory", "container", taskKey.ContainerName, "oomScaledMem", oomScaledMem.String()) + memQuantity = oomScaledMem + } + return memQuantity, nil +} + +func (r *PercentileRecommender) getMemQuantity(memRecommendedValue float64) (quantity *resource.Quantity) { + scale := int64(1) + quotient := int64(memRecommendedValue) + remainder := int64(0) + for scale < 1024*1024 { + if quotient < 1024 { + break + } + scale *= 1024 + quotient = int64(memRecommendedValue) / scale + remainder = int64(memRecommendedValue) % scale + } + if remainder == 0 { + return resource.NewQuantity(quotient*scale, resource.BinarySI) + } + return resource.NewQuantity((quotient+1)*scale, resource.BinarySI) +} diff --git a/pkg/controller/resource-recommend/recommender/recommenders/percentile_recommender_test.go b/pkg/controller/resource-recommend/recommender/recommenders/percentile_recommender_test.go new file mode 100644 index 0000000000..1efdb84f88 --- /dev/null +++ b/pkg/controller/resource-recommend/recommender/recommenders/percentile_recommender_test.go @@ -0,0 +1,242 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommenders + +import ( + "context" + "reflect" + "testing" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/datasource" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/oom" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/processor/common" + customError "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/error" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/recommendation" +) + +func TestRecommend(t *testing.T) { + recommendation := &recommendation.Recommendation{ + NamespacedName: types.NamespacedName{ + Name: "name1", + Namespace: "namespace1", + }, + Config: recommendation.Config{ + Containers: []recommendation.Container{ + { + ContainerName: "container1", + ContainerConfigs: []recommendation.ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: 10, + }, + { + ControlledResource: v1.ResourceMemory, + ResourceBufferPercent: 10, + }, + }, + }, + }, + TargetRef: v1alpha1.CrossVersionObjectReference{ + Kind: "deployment", + Name: "workload1", + }, + }, + } + + r := &PercentileRecommender{ + DataProcessor: dummyDataProcessor{}, + OomRecorder: dummyOomRecorder{}, + } + err := r.Recommend(recommendation) + + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + + if recommendation.Recommendations[0].Requests.Target.Cpu().String() != "1100" || recommendation.Recommendations[0].Requests.Target.Memory().String() != "2Ki" { + t.Errorf("Recommendations mismatch.") + } +} + +func TestGetCpuTargetPercentileEstimationWithUsageBuffer(t *testing.T) { + recommender := &PercentileRecommender{ + DataProcessor: dummyDataProcessor{}, + OomRecorder: dummyOomRecorder{}, + } + taskKey := &common.ProcessKey{ + ResourceRecommendNamespacedName: types.NamespacedName{ + Name: "name1", + Namespace: "namespace1", + }, + Metric: &datasource.Metric{ + Namespace: "namespace1", + Kind: "deployment1", + WorkloadName: "workload1", + ContainerName: "container1", + Resource: v1.ResourceCPU, + }, + } + resourceBufferPercentage := 0.1 + cpuQuantity, err := recommender.getCpuTargetPercentileEstimationWithUsageBuffer(taskKey, resourceBufferPercentage) + if err != nil { + t.Errorf("Expected no error, but got: %v", err) + } + + expectedCpuQuantity := resource.NewMilliQuantity(int64(1100*1000), resource.DecimalSI) // 设置期望的CPU数量 + if !cpuQuantity.Equal(*expectedCpuQuantity) { + t.Errorf("Expected cpu quantity %s, but got %s", cpuQuantity.String(), expectedCpuQuantity.String()) + } +} + +func TestGetMemTargetPercentileEstimationWithUsageBuffer(t *testing.T) { + recommender := &PercentileRecommender{ + DataProcessor: dummyDataProcessor{}, + OomRecorder: dummyOomRecorder{}, + } + taskKey := &common.ProcessKey{ + ResourceRecommendNamespacedName: types.NamespacedName{ + Name: "name1", + Namespace: "namespace1", + }, + Metric: &datasource.Metric{ + Namespace: "namespace1", + Kind: "deployment1", + WorkloadName: "workload1", + ContainerName: "container1", + Resource: v1.ResourceMemory, + }, + } + resourceBufferPercentage := 0.1 + memQuantity, err := recommender.getMemTargetPercentileEstimationWithUsageBuffer(taskKey, resourceBufferPercentage) + if err != nil { + t.Errorf("Expected no error, but got: %v", err) + } + + expectedMemQuantity := resource.NewQuantity((1100/1024+1)*1024, resource.BinarySI) // 设置期望的内存数量 + if !memQuantity.Equal(*expectedMemQuantity) { + t.Errorf("Expected memory quantity %s, but got %s", expectedMemQuantity.String(), memQuantity.String()) + } +} + +func TestScaleOnOOM(t *testing.T) { + oomRecords := []oom.OOMRecord{ + { + Pod: "workload-name-pod-1", + Container: "container-1", + Namespace: "namespace-1", + OOMAt: time.Now(), + Memory: *resource.NewQuantity(1024, resource.BinarySI), + }, + { + Pod: "workload-name-pod-2", + Container: "container-2", + Namespace: "namespace-2", + OOMAt: time.Now().Add(-time.Hour * 24 * 8), // too old event + Memory: *resource.NewQuantity(2048, resource.BinarySI), + }, + } + + r := &PercentileRecommender{} + namespace := "namespace-1" + workloadName := "workload-name" + containerName := "container-1" + + quantityPointer := r.ScaleOnOOM(oomRecords, namespace, workloadName, containerName) + + if quantityPointer == nil { + t.Errorf("Expected non-nil quantity, got nil") + return + } + quantity := *quantityPointer + + expectedValuePointer := r.getMemQuantity(float64(oomRecords[0].Memory.Value()) + OOMMinBumpUp) + if expectedValuePointer == nil { + t.Errorf("got expectedValuePointer is nil") + return + } + expectedValue := *expectedValuePointer + if !quantity.Equal(expectedValue) { + t.Errorf("Expected value %s, got %s", expectedValue.String(), quantity.String()) + return + } +} + +type dummyDataProcessor struct{} + +func (d dummyDataProcessor) Run(ctx context.Context) { + return +} + +func (d dummyDataProcessor) Register(processConfig *common.ProcessConfig) *customError.CustomError { + return nil +} + +func (d dummyDataProcessor) Cancel(processKey *common.ProcessKey) *customError.CustomError { + return nil +} + +func (d dummyDataProcessor) QueryProcessedValues(taskKey *common.ProcessKey) (float64, error) { + return 1000, nil +} + +type dummyOomRecorder struct{} + +func (d dummyOomRecorder) ListOOMRecords() []oom.OOMRecord { + return nil +} + +func (d dummyOomRecorder) ScaleOnOOM(oomRecords []oom.OOMRecord, namespace, workloadName, containerName string) *resource.Quantity { + return nil +} + +func TestPercentileRecommender_getMemQuantity(t *testing.T) { + tests := []struct { + name string + memRecommendedValue float64 + wantQuantity *resource.Quantity + }{ + { + name: "testKi", + memRecommendedValue: 1100, + wantQuantity: resource.NewQuantity(2*1024, resource.BinarySI), + }, + { + name: "testMi", + memRecommendedValue: 2 * 1024 * 1024, + wantQuantity: resource.NewQuantity(2*1024*1024, resource.BinarySI), + }, + { + name: "testMiNotDivisible", + memRecommendedValue: 2.66 * 1024 * 1024, + wantQuantity: resource.NewQuantity(3*1024*1024, resource.BinarySI), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &PercentileRecommender{} + if gotQuantity := r.getMemQuantity(tt.memRecommendedValue); !reflect.DeepEqual(gotQuantity, tt.wantQuantity) { + t.Errorf("getMemQuantity() = %v, want %v", gotQuantity, tt.wantQuantity) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/types/conditions/conditions.go b/pkg/controller/resource-recommend/types/conditions/conditions.go new file mode 100644 index 0000000000..5cc1cffdfa --- /dev/null +++ b/pkg/controller/resource-recommend/types/conditions/conditions.go @@ -0,0 +1,109 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conditions + +import ( + "sort" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + customError "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/error" +) + +// ResourceRecommendConditionsMap is map from recommend condition type to condition. +type ResourceRecommendConditionsMap map[v1alpha1.ResourceRecommendConditionType]v1alpha1.ResourceRecommendCondition + +func NewResourceRecommendConditionsMap() *ResourceRecommendConditionsMap { + convertedConditionsMap := make(ResourceRecommendConditionsMap) + return &convertedConditionsMap +} + +func (conditionsMap *ResourceRecommendConditionsMap) Set(condition v1alpha1.ResourceRecommendCondition) { + oldCondition, alreadyPresent := (*conditionsMap)[condition.Type] + if alreadyPresent && oldCondition.Type == condition.Type && + oldCondition.Status == condition.Status && + oldCondition.Reason == condition.Reason && + oldCondition.Message == condition.Message { + return + } else { + condition.LastTransitionTime = metav1.Now() + } + (*conditionsMap)[condition.Type] = condition +} + +func (conditionsMap *ResourceRecommendConditionsMap) AsList() []v1alpha1.ResourceRecommendCondition { + conditions := make([]v1alpha1.ResourceRecommendCondition, 0, len(*conditionsMap)) + for _, condition := range *conditionsMap { + conditions = append(conditions, condition) + } + + // Sort conditions by type to avoid elements floating on the list + sort.Slice(conditions, func(i, j int) bool { + return conditions[i].Type < conditions[j].Type + }) + + return conditions +} + +func (conditionsMap *ResourceRecommendConditionsMap) ConditionActive(conditionType v1alpha1.ResourceRecommendConditionType) bool { + condition, found := (*conditionsMap)[conditionType] + return found && condition.Status == v1.ConditionTrue +} + +func ValidationSucceededCondition() *v1alpha1.ResourceRecommendCondition { + return &v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Validated, + Status: v1.ConditionTrue, + } +} + +func InitializationSucceededCondition() *v1alpha1.ResourceRecommendCondition { + return &v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.Initialized, + Status: v1.ConditionTrue, + } +} + +func RecommendationReadyCondition() *v1alpha1.ResourceRecommendCondition { + return &v1alpha1.ResourceRecommendCondition{ + Type: v1alpha1.RecommendationProvided, + Status: v1.ConditionTrue, + } +} + +func ConvertCustomErrorToCondition(err customError.CustomError) *v1alpha1.ResourceRecommendCondition { + var conditionType v1alpha1.ResourceRecommendConditionType + switch err.Phase { + case customError.Validated: + conditionType = v1alpha1.Validated + case customError.ProcessRegister: + conditionType = v1alpha1.Initialized + case customError.RecommendationProvided: + conditionType = v1alpha1.RecommendationProvided + default: + conditionType = v1alpha1.ResourceRecommendConditionType(err.Phase) + } + + return &v1alpha1.ResourceRecommendCondition{ + Type: conditionType, + Status: v1.ConditionFalse, + Reason: string(err.Code), + Message: err.Message, + } +} diff --git a/pkg/controller/resource-recommend/types/error/errors.go b/pkg/controller/resource-recommend/types/error/errors.go new file mode 100644 index 0000000000..bf8ecd8c1f --- /dev/null +++ b/pkg/controller/resource-recommend/types/error/errors.go @@ -0,0 +1,34 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package error + +type Phase string + +type Code string + +type CustomError struct { + // Phase in which error occurs + Phase Phase + // Code of the current error + Code Code + // Message is a human-readable explanation containing details about the error + Message string +} + +func (err *CustomError) Error() string { + return err.Message +} diff --git a/pkg/controller/resource-recommend/types/error/process.go b/pkg/controller/resource-recommend/types/error/process.go new file mode 100644 index 0000000000..80f1a0dd11 --- /dev/null +++ b/pkg/controller/resource-recommend/types/error/process.go @@ -0,0 +1,93 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package error + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/types" +) + +const ( + ProcessRegister Phase = "ProcessRegister" + ProcessCancel Phase = "ProcessCancel" +) + +const ( + DataProcessRegisterFailed Code = "DataProcessRegisterFailed" + ValidateProcessConfigFailed Code = "ValidateProcessConfigFailed" + RegisterTaskPanic Code = "RegisterTaskPanic" + CancelTaskPanic Code = "CancelTaskPanic" + NewProcessTaskFailed Code = "NewProcessTaskFailed" + NotFoundTasks Code = "NotFoundTask" +) + +const ( + RegisterProcessTaskPanicMessage = "register data process task panic" + CancelProcessTaskPanicMessage = "cancel data process task panic" + ProcessTaskValidateErrorMessageTemplate = "register data process task failed, get error when validate, err: %s" + NewProcessTaskErrorMessageTemplate = "register data process task failed, fail to get new task, err: %s" + TasksNotFoundErrorMessage = "process tasks not found " +) + +func DataProcessRegisteredFailedError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: ProcessRegister, + Code: DataProcessRegisterFailed, + Message: fmt.Sprintf(msg, arg...), + } +} + +func RegisterProcessTaskValidateError(err error) *CustomError { + return &CustomError{ + Phase: ProcessRegister, + Code: ValidateProcessConfigFailed, + Message: fmt.Sprintf(ProcessTaskValidateErrorMessageTemplate, err), + } +} + +func RegisterProcessTaskPanic() *CustomError { + return &CustomError{ + Phase: ProcessRegister, + Code: RegisterTaskPanic, + Message: RegisterProcessTaskPanicMessage, + } +} + +func NewProcessTaskError(err error) *CustomError { + return &CustomError{ + Phase: ProcessRegister, + Code: NewProcessTaskFailed, + Message: fmt.Sprintf(NewProcessTaskErrorMessageTemplate, err), + } +} + +func CancelProcessTaskPanic() *CustomError { + return &CustomError{ + Phase: ProcessCancel, + Code: CancelTaskPanic, + Message: CancelProcessTaskPanicMessage, + } +} + +func NotFoundTasksError(namespacedName types.NamespacedName) *CustomError { + return &CustomError{ + Phase: ProcessCancel, + Code: NotFoundTasks, + Message: fmt.Sprintf(TasksNotFoundErrorMessage+"for ResourceRecommend(%s)", namespacedName), + } +} diff --git a/pkg/controller/resource-recommend/types/error/recommend.go b/pkg/controller/resource-recommend/types/error/recommend.go new file mode 100644 index 0000000000..8b02a8fbac --- /dev/null +++ b/pkg/controller/resource-recommend/types/error/recommend.go @@ -0,0 +1,35 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package error + +import "fmt" + +const ( + RecommendationProvided Phase = "RecommendationProvided" +) + +const ( + RecommendationNotReady Code = "RecommendationNotReady" +) + +func RecommendationNotReadyError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: RecommendationProvided, + Code: RecommendationNotReady, + Message: fmt.Sprintf(msg, arg...), + } +} diff --git a/pkg/controller/resource-recommend/types/error/validate.go b/pkg/controller/resource-recommend/types/error/validate.go new file mode 100644 index 0000000000..4e01c040c9 --- /dev/null +++ b/pkg/controller/resource-recommend/types/error/validate.go @@ -0,0 +1,189 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package error + +import "fmt" + +const ( + Validated Phase = "Validated" +) + +const ( + WorkloadNameIsEmpty Code = "WorkloadNameIsEmpty" + WorkloadsUnsupported Code = "WorkloadsUnsupported" + AlgorithmUnsupported Code = "AlgorithmUnsupported" + WorkloadMatchError Code = "WorkloadMatchError" + WorkloadNotFound Code = "WorkloadNotFound" + ContainerPoliciesNotFound Code = "ContainerPoliciesNotFound" + ContainersNotFound Code = "ContainersNotFound" + ContainersMatchErrorCode Code = "ContainersMatchErrorCode" + ContainerDuplicate Code = "ContainerDuplicate" + ContainerNameEmpty Code = "ContainerNameEmpty" + ControlledResourcesPoliciesEmpty Code = "ControlledResourcesPoliciesEmpty" + ResourceBuffersUnsupported Code = "ResourceBuffersUnsupported" + ResourceNameUnsupported Code = "ResourceNameUnsupported" + ControlledValuesUnsupported Code = "ControlledValuesUnsupported" + VCIValidationErrorCode Code = "VCIValidationErrorCode" + VCIUnsupported Code = "VCIUnsupported" +) + +const ( + WorkloadNameIsEmptyMessage = "spec.targetRef.name cannot be empty" + WorkloadsUnsupportedMessage = "spec.targetRef.kind %s is currently unsupported, only support in %s" + AlgorithmUnsupportedMessage = "spec.resourcePolicy.algorithmPolicy.algorithm %s is currently unsupported, only support in %s" + WorkloadNotFoundMessage = "workload not found" + WorkloadMatchedErrorMessage = "workload matched err" + ContainerPoliciesNotFoundMessage = "spec.containerPolicies cannot be empty" + VCIValidationErrorMessage = "Volcengine Container Instance validation err" + VCIUnsupportedMessage = "Volcengine Container Instance not supported" + ContainersMatchedErrorMessage = "containers matched err" + ContainerDuplicateMessage = "container name %s is duplicate" + ContainerNameEmptyMessage = "empty container name" + ControlledResourcesPoliciesEmptyMessage = "container(%s) Resources Policies is empty" + ContainerNotFoundMessage = "container %s is not found" + ResourceNameUnsupportedMessage = "unsupported ResourceName, current supported values: %s" + ControlledValuesUnsupportedMessage = "unsupported ControlledValues, current supported values: %s" + ResourceBuffersUnsupportedMessage = "resource buffers should be in the range from 0 to 100" +) + +func WorkloadNameIsEmptyError() *CustomError { + return &CustomError{ + Phase: Validated, + Code: WorkloadNameIsEmpty, + Message: WorkloadNameIsEmptyMessage, + } +} + +func WorkloadsUnsupportedError(kind string, supportedKinds []string) *CustomError { + return &CustomError{ + Phase: Validated, + Code: WorkloadsUnsupported, + Message: fmt.Sprintf(WorkloadsUnsupportedMessage, kind, supportedKinds), + } +} + +func AlgorithmUnsupportedError(algorithm string, supportedAlgorithm []string) *CustomError { + return &CustomError{ + Phase: Validated, + Code: AlgorithmUnsupported, + Message: fmt.Sprintf(AlgorithmUnsupportedMessage, algorithm, supportedAlgorithm), + } +} + +func ResourceNameUnsupportedError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ResourceNameUnsupported, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ControlledValuesUnsupportedError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ControlledValuesUnsupported, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ResourceBuffersUnsupportedError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ResourceBuffersUnsupported, + Message: fmt.Sprintf(msg, arg...), + } +} + +func WorkloadMatchedError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: WorkloadMatchError, + Message: fmt.Sprintf(msg, arg...), + } +} + +func WorkloadNotFoundError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: WorkloadNotFound, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ContainerPoliciesNotFoundError() *CustomError { + return &CustomError{ + Phase: Validated, + Code: ContainerPoliciesNotFound, + Message: ContainerPoliciesNotFoundMessage, + } +} + +func VCIValidationError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: VCIValidationErrorCode, + Message: fmt.Sprintf(msg, arg...), + } +} + +func VCIUnsupportedError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: VCIUnsupported, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ContainersMatchedError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ContainersMatchErrorCode, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ContainerNameEmptyError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ContainerNameEmpty, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ControlledResourcesPoliciesEmptyError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ControlledResourcesPoliciesEmpty, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ContainersNotFoundError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ContainersNotFound, + Message: fmt.Sprintf(msg, arg...), + } +} + +func ContainerDuplicateError(msg string, arg ...any) *CustomError { + return &CustomError{ + Phase: Validated, + Code: ContainerDuplicate, + Message: fmt.Sprintf(msg, arg...), + } +} diff --git a/pkg/controller/resource-recommend/types/recommendation/recommendation.go b/pkg/controller/resource-recommend/types/recommendation/recommendation.go new file mode 100644 index 0000000000..9b945927cc --- /dev/null +++ b/pkg/controller/resource-recommend/types/recommendation/recommendation.go @@ -0,0 +1,147 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommendation + +import ( + "context" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/conditions" + customError "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/error" +) + +const ( + // TargetRefKindDeployment is Deployment + TargetRefKindDeployment string = "Deployment" +) + +var TargetRefKinds = []string{TargetRefKindDeployment} + +const ( + DefaultRecommenderType = "default" +) + +var RecommenderTypes = []string{DefaultRecommenderType} + +const ( + PercentileAlgorithmType = "percentile" + //use percentile as the default algorithm + DefaultAlgorithmType = PercentileAlgorithmType +) + +var AlgorithmTypes = []string{PercentileAlgorithmType} + +var ResourceNames = []v1.ResourceName{v1.ResourceCPU, v1.ResourceMemory} + +var SupportControlledValues = []v1alpha1.ContainerControlledValues{v1alpha1.ContainerControlledValuesRequestsOnly} + +const ( + DefaultUsageBuffer = 30 + MaxUsageBuffer = 100 + MinUsageBuffer = 0 +) + +const ( + ContainerPolicySelectAllFlag = "*" +) + +type Recommendation struct { + types.NamespacedName + ObservedGeneration int64 + Config + // Map of the status conditions (keys are condition types). + Conditions *conditions.ResourceRecommendConditionsMap + Recommendations []v1alpha1.ContainerResources +} + +type Config struct { + TargetRef v1alpha1.CrossVersionObjectReference + Containers []Container + AlgorithmPolicy v1alpha1.AlgorithmPolicy +} + +type Container struct { + ContainerName string + ContainerConfigs []ContainerConfig +} + +type ContainerConfig struct { + ControlledResource v1.ResourceName + ResourceBufferPercent int32 +} + +func NewRecommendation(resourceRecommend *v1alpha1.ResourceRecommend) *Recommendation { + return &Recommendation{ + NamespacedName: types.NamespacedName{Name: resourceRecommend.Name, Namespace: resourceRecommend.Namespace}, + ObservedGeneration: resourceRecommend.Generation, + Conditions: conditions.NewResourceRecommendConditionsMap(), + } +} + +func (r *Recommendation) SetConfig(ctx context.Context, client k8sclient.Client, + resourceRecommend *v1alpha1.ResourceRecommend) *customError.CustomError { + + targetRef, customErr := ValidateAndExtractTargetRef(resourceRecommend.Spec.TargetRef) + if customErr != nil { + klog.Errorf("spec.targetRef validate error, "+ + "reason: %s, msg: %s", customErr.Code, customErr.Message) + return customErr + } + + algorithmPolicy, customErr := ValidateAndExtractAlgorithmPolicy(resourceRecommend.Spec.ResourcePolicy.AlgorithmPolicy) + if customErr != nil { + klog.Errorf("spec.resourcePolicy.algorithmPolicy validate error,"+ + " reason: %s, msg: %s", customErr.Code, customErr.Message) + return customErr + } + + containers, customErr := ValidateAndExtractContainers(ctx, client, resourceRecommend.Namespace, targetRef, resourceRecommend.Spec.ResourcePolicy.ContainerPolicies) + if customErr != nil { + klog.Errorf("spec.resourcePolicy.containerPolicies validate error, "+ + "reason: %s, msg: %s", customErr.Code, customErr.Message) + return customErr + } + + r.Config = Config{ + TargetRef: targetRef, + AlgorithmPolicy: algorithmPolicy, + Containers: containers, + } + return nil +} + +// AsStatus returns this objects equivalent of VPA Status. UpdateConditions +// should be called first. +func (r *Recommendation) AsStatus() v1alpha1.ResourceRecommendStatus { + status := v1alpha1.ResourceRecommendStatus{ + Conditions: r.Conditions.AsList(), + } + if r.Recommendations != nil { + now := metav1.Now() + status.LastRecommendationTime = &now + status.RecommendResources = &v1alpha1.RecommendResources{ + ContainerRecommendations: r.Recommendations, + } + } + return status +} diff --git a/pkg/controller/resource-recommend/types/recommendation/validate.go b/pkg/controller/resource-recommend/types/recommendation/validate.go new file mode 100644 index 0000000000..2b2b77f998 --- /dev/null +++ b/pkg/controller/resource-recommend/types/recommendation/validate.go @@ -0,0 +1,165 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommendation + +import ( + "context" + + "k8s.io/klog/v2" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + + spendsmartv1alpha1 "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + customError "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/error" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/utils" +) + +func ValidateAndExtractTargetRef(targetRefReq spendsmartv1alpha1.CrossVersionObjectReference) ( + spendsmartv1alpha1.CrossVersionObjectReference, *customError.CustomError) { + convertedTargetRef := spendsmartv1alpha1.CrossVersionObjectReference{} + if targetRefReq.Name == "" { + return convertedTargetRef, customError.WorkloadNameIsEmptyError() + } + if ok := utils.SliceContains(TargetRefKinds, targetRefReq.Kind); !ok { + return convertedTargetRef, customError.WorkloadsUnsupportedError(targetRefReq.Kind, TargetRefKinds) + } + convertedTargetRef.Kind = targetRefReq.Kind + convertedTargetRef.Name = targetRefReq.Name + convertedTargetRef.APIVersion = targetRefReq.APIVersion + return convertedTargetRef, nil +} + +func ValidateAndExtractAlgorithmPolicy(algorithmPolicyReq spendsmartv1alpha1.AlgorithmPolicy) ( + spendsmartv1alpha1.AlgorithmPolicy, *customError.CustomError) { + algorithmPolicy := spendsmartv1alpha1.AlgorithmPolicy{ + Recommender: DefaultRecommenderType, + } + + if algorithmPolicyReq.Algorithm == "" { + algorithmPolicy.Algorithm = DefaultAlgorithmType + } else { + if ok := utils.SliceContains(AlgorithmTypes, algorithmPolicyReq.Algorithm); !ok { + return algorithmPolicy, customError.AlgorithmUnsupportedError(string(algorithmPolicyReq.Algorithm), AlgorithmTypes) + } + algorithmPolicy.Algorithm = algorithmPolicyReq.Algorithm + } + + algorithmPolicy.Extensions = algorithmPolicyReq.Extensions + return algorithmPolicy, nil +} + +func ValidateAndExtractContainers(ctx context.Context, client k8sclient.Client, namespace string, + targetRef spendsmartv1alpha1.CrossVersionObjectReference, + containerPolicies []spendsmartv1alpha1.ContainerResourcePolicy) ( + []Container, *customError.CustomError) { + if len(containerPolicies) == 0 { + return nil, customError.ContainerPoliciesNotFoundError() + } + + resource, err := utils.ConvertAndGetResource(ctx, client, namespace, targetRef) + if err != nil { + klog.ErrorS(err, "ConvertAndGetResource err") + if k8sclient.IgnoreNotFound(err) == nil { + return nil, customError.WorkloadNotFoundError(customError.WorkloadNotFoundMessage) + } + return nil, customError.WorkloadMatchedError(customError.WorkloadMatchedErrorMessage) + } + + isVci, err := utils.IsVolcengineContainerInstance(ctx, client, resource) + if err != nil { + klog.ErrorS(err, "check is vci err") + return nil, customError.VCIValidationError(customError.VCIValidationErrorMessage) + } + if isVci { + return nil, customError.VCIUnsupportedError(customError.VCIUnsupportedMessage) + } + + existContainerList, err := utils.GetAllClaimedContainers(resource) + if err != nil { + klog.ErrorS(err, "get all claimed containers err") + return nil, customError.ContainersMatchedError(customError.ContainersMatchedErrorMessage) + } + + containers, validateErr := validateAndExtractContainers(containerPolicies, existContainerList) + if validateErr != nil { + return nil, validateErr + } + + return containers, nil +} + +func validateAndExtractContainers(containerPolicies []spendsmartv1alpha1.ContainerResourcePolicy, + existContainerList []string) ([]Container, *customError.CustomError) { + resourcePoliciesMap := make(map[string]spendsmartv1alpha1.ContainerResourcePolicy) + + for _, resourcePolicy := range containerPolicies { + containerName := resourcePolicy.ContainerName + if _, ok := resourcePoliciesMap[containerName]; ok { + return nil, customError.ContainerDuplicateError(customError.ContainerDuplicateMessage, containerName) + } else if containerName == "" { + return nil, customError.ContainerNameEmptyError(customError.ContainerNameEmptyMessage) + } else if !(containerName == ContainerPolicySelectAllFlag) && !utils.SliceContains(existContainerList, containerName) { + return nil, customError.ContainersNotFoundError(customError.ContainerNotFoundMessage, containerName) + } else if len(resourcePolicy.ControlledResourcesPolicies) == 0 { + return nil, customError.ControlledResourcesPoliciesEmptyError(customError.ControlledResourcesPoliciesEmptyMessage, containerName) + } + resourcePoliciesMap[containerName] = resourcePolicy + } + + if defaultPolicy, exist := resourcePoliciesMap[ContainerPolicySelectAllFlag]; exist { + for _, container := range existContainerList { + if _, ok := resourcePoliciesMap[container]; !ok { + defaultPolicy.ContainerName = container + resourcePoliciesMap[container] = defaultPolicy + } + } + delete(resourcePoliciesMap, "*") + } + + containers := make([]Container, 0, len(resourcePoliciesMap)) + for _, containerResourcePolicy := range resourcePoliciesMap { + container := Container{ + ContainerName: containerResourcePolicy.ContainerName, + ContainerConfigs: []ContainerConfig{}, + } + for _, resourcesPolicy := range containerResourcePolicy.ControlledResourcesPolicies { + if ok := utils.SliceContains(ResourceNames, resourcesPolicy.ResourceName); !ok { + return containers, customError.ResourceNameUnsupportedError(customError.ResourceNameUnsupportedMessage, ResourceNames) + } + if resourcesPolicy.ControlledValues != nil { + if ok := utils.SliceContains(SupportControlledValues, *resourcesPolicy.ControlledValues); !ok { + return containers, customError.ControlledValuesUnsupportedError(customError.ResourceNameUnsupportedMessage, SupportControlledValues) + } + } + + containerConfig := ContainerConfig{ + ControlledResource: resourcesPolicy.ResourceName, + } + resourceBufferPercent := resourcesPolicy.BufferPercent + if resourceBufferPercent == nil { + containerConfig.ResourceBufferPercent = DefaultUsageBuffer + } else if *resourceBufferPercent > MaxUsageBuffer || *resourceBufferPercent < MinUsageBuffer { + return containers, customError.ResourceBuffersUnsupportedError(customError.ResourceBuffersUnsupportedMessage) + } else { + containerConfig.ResourceBufferPercent = *resourceBufferPercent + } + + container.ContainerConfigs = append(container.ContainerConfigs, containerConfig) + } + containers = append(containers, container) + } + return containers, nil +} diff --git a/pkg/controller/resource-recommend/types/recommendation/validate_test.go b/pkg/controller/resource-recommend/types/recommendation/validate_test.go new file mode 100644 index 0000000000..37f801974e --- /dev/null +++ b/pkg/controller/resource-recommend/types/recommendation/validate_test.go @@ -0,0 +1,1004 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommendation + +import ( + "context" + "reflect" + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" + customError "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/types/error" + "github.com/kubewharf/katalyst-core/pkg/controller/resource-recommend/utils" +) + +func TestValidateAndExtractAlgorithmPolicy(t *testing.T) { + type args struct { + algorithmPolicyReq v1alpha1.AlgorithmPolicy + } + tests := []struct { + name string + args args + want v1alpha1.AlgorithmPolicy + wantErr *customError.CustomError + }{ + { + name: "algorithmPolicyReq.Algorithm empty", + args: args{ + algorithmPolicyReq: v1alpha1.AlgorithmPolicy{ + Recommender: "Recommender", + Algorithm: "", + Extensions: &runtime.RawExtension{ + Raw: []byte("Data"), + }, + }, + }, + want: v1alpha1.AlgorithmPolicy{ + Recommender: "default", + Algorithm: DefaultAlgorithmType, + Extensions: &runtime.RawExtension{ + Raw: []byte("Data"), + }, + }, + wantErr: nil, + }, + { + name: "algorithmPolicyReq.Algorithm unsupported", + args: args{ + algorithmPolicyReq: v1alpha1.AlgorithmPolicy{ + Recommender: "Recommender", + Algorithm: "mockAlgorithm", + Extensions: &runtime.RawExtension{ + Raw: []byte("Data"), + }, + }, + }, + want: v1alpha1.AlgorithmPolicy{ + Recommender: "default", + }, + wantErr: customError.AlgorithmUnsupportedError("mockAlgorithm", AlgorithmTypes), + }, + { + name: "algorithmPolicyReq.Algorithm supported", + args: args{ + algorithmPolicyReq: v1alpha1.AlgorithmPolicy{ + Recommender: "default", + Algorithm: PercentileAlgorithmType, + Extensions: &runtime.RawExtension{ + Raw: []byte("Data"), + }, + }, + }, + want: v1alpha1.AlgorithmPolicy{ + Recommender: "default", + Algorithm: PercentileAlgorithmType, + Extensions: &runtime.RawExtension{ + Raw: []byte("Data"), + }, + }, + wantErr: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotErr := ValidateAndExtractAlgorithmPolicy(tt.args.algorithmPolicyReq) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ValidateAndExtractAlgorithmPolicy() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("ValidateAndExtractAlgorithmPolicy() gotErr = %v, want %v", gotErr, tt.wantErr) + } + }) + } +} + +func TestValidateAndExtractContainers(t *testing.T) { + type args struct { + ctx context.Context + client client.Client + namespace string + targetRef v1alpha1.CrossVersionObjectReference + containerPolicies []v1alpha1.ContainerResourcePolicy + } + type env struct { + podLabels map[string]string + podAnnotations map[string]string + matchLabelKey string + matchLabelValue string + podName string + podNodeName string + unstructuredName string + unstructuredTemplateSpec map[string]interface{} + namespace string + kind string + apiVersion string + } + tests := []struct { + name string + args args + env env + want []Container + wantErr *customError.CustomError + }{ + { + name: customError.WorkloadNotFoundMessage, + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Kind: "kind", + Name: "Name", + APIVersion: "version", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + }, + }, + }, + want: nil, + wantErr: customError.WorkloadNotFoundError(customError.WorkloadNotFoundMessage), + }, + { + name: customError.WorkloadMatchedErrorMessage, + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Kind: "", + Name: "", + APIVersion: "", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + }, + }, + }, + }, + }, + want: nil, + wantErr: customError.WorkloadMatchedError(customError.WorkloadMatchedErrorMessage), + }, + { + name: "vci-node-name", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Name: "mockName1", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + }, + }, + }, + }, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels1", + }, + matchLabelKey: "app", + matchLabelValue: "mockPodLabels1", + podName: "vci-mockPodName1", + podNodeName: "vci-mockPodNodeName1", + unstructuredName: "mockName1", + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: nil, + wantErr: customError.VCIUnsupportedError(customError.VCIUnsupportedMessage), + }, + { + name: "vci-annotationKey-1", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Name: "mockName2", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + }, + }, + }, + }, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels2", + }, + podAnnotations: map[string]string{ + utils.VCIAnnotationKeyOne: utils.VCIAnnotationValueOne, + }, + matchLabelKey: "app", + matchLabelValue: "mockPodLabels2", + podName: "vci-mockPodName2", + podNodeName: "mockPodNodeName2", + unstructuredName: "mockName2", + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: nil, + wantErr: customError.VCIUnsupportedError(customError.VCIUnsupportedMessage), + }, + { + name: "vci-annotationKey-2", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Name: "mockName3", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + }, + }, + }, + }, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels3", + }, + podAnnotations: map[string]string{ + utils.VCIAnnotationKeyTwo: "VCIAnnotationValueTwo", + }, + matchLabelKey: "app", + matchLabelValue: "mockPodLabels3", + podName: "vci-mockPodName3", + podNodeName: "mockPodNodeName3", + unstructuredName: "mockName3", + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: nil, + wantErr: customError.VCIUnsupportedError(customError.VCIUnsupportedMessage), + }, + { + name: "vci-validation-error", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Name: "mockName4", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + }, + }, + }, + }, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels3", + }, + podAnnotations: map[string]string{ + utils.VCIAnnotationKeyOne: "VCIAnnotationValueTwo", + }, + matchLabelKey: "app", + matchLabelValue: "mockPodLabels4", + podName: "vci-mockPodName4", + podNodeName: "mockPodNodeName4", + unstructuredName: "mockName4", + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: nil, + wantErr: customError.VCIValidationError(customError.VCIValidationErrorMessage), + }, + { + name: "all right", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Name: "mockName5", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "container-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + }, + }, + }, + { + ContainerName: "container-2", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + }, + }, + }, + }, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels5", + }, + podAnnotations: map[string]string{ + utils.VCIAnnotationKeyOne: "VCIAnnotationValueTwo", + }, + matchLabelKey: "app", + matchLabelValue: "mockPodLabels5", + podName: "vci-mockPodName5", + podNodeName: "mockPodNodeName5", + unstructuredName: "mockName5", + unstructuredTemplateSpec: map[string]interface{}{ + "containers": []interface{}{ + map[string]interface{}{ + "name": "container-1", + "image": "image:latest", + }, + map[string]interface{}{ + "name": "container-2", + "image": "image:latest", + }, + }, + }, + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: []Container{ + { + ContainerName: "container-1", + ContainerConfigs: []ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: DefaultUsageBuffer, + }, + }, + }, + { + ContainerName: "container-2", + ContainerConfigs: []ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: DefaultUsageBuffer, + }, + }, + }, + }, + wantErr: nil, + }, + { + name: "controlled resources is empty", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Name: "mockName5", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + }, + }, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels5", + }, + podAnnotations: map[string]string{ + utils.VCIAnnotationKeyOne: "VCIAnnotationValueTwo", + }, + matchLabelKey: "app", + matchLabelValue: "mockPodLabels5", + podName: "vci-mockPodName5", + podNodeName: "mockPodNodeName5", + unstructuredName: "mockName5", + unstructuredTemplateSpec: map[string]interface{}{ + "containers": []interface{}{ + map[string]interface{}{ + "name": "container-1", + "image": "image:latest", + }, + map[string]interface{}{ + "name": "container-2", + "image": "image:latest", + }, + }, + }, + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: nil, + wantErr: customError.ControlledResourcesPoliciesEmptyError(customError.ControlledResourcesPoliciesEmptyMessage, "*"), + }, + { + name: customError.ContainersMatchedErrorMessage, + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Name: "mockName5", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "container-1", + }, + { + ContainerName: "container-2", + }, + }, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels5", + }, + podAnnotations: map[string]string{ + utils.VCIAnnotationKeyOne: "VCIAnnotationValueTwo", + }, + matchLabelKey: "app", + matchLabelValue: "mockPodLabels5", + podName: "vci-mockPodName5", + podNodeName: "mockPodNodeName5", + unstructuredName: "mockName5", + unstructuredTemplateSpec: map[string]interface{}{ + "container": []interface{}{ + map[string]interface{}{ + "name": "container-1", + "image": "image:latest", + }, + map[string]interface{}{ + "name": "container-2", + "image": "image:latest", + }, + }, + }, + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: nil, + wantErr: customError.ContainersMatchedError(customError.ContainersMatchedErrorMessage), + }, + { + name: "validate containers err", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "mockNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Name: "mockName5", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "container-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + }, + }, + }, + { + ContainerName: "container-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + }, + }, + }, + }, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels5", + }, + podAnnotations: map[string]string{ + utils.VCIAnnotationKeyOne: "VCIAnnotationValueTwo", + }, + matchLabelKey: "app", + matchLabelValue: "mockPodLabels5", + podName: "vci-mockPodName5", + podNodeName: "mockPodNodeName5", + unstructuredName: "mockName5", + unstructuredTemplateSpec: map[string]interface{}{ + "containers": []interface{}{ + map[string]interface{}{ + "name": "container-1", + "image": "image:latest", + }, + map[string]interface{}{ + "name": "container-2", + "image": "image:latest", + }, + }, + }, + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: nil, + wantErr: customError.ContainerDuplicateError(customError.ContainerDuplicateMessage, "container-1"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matchLabels := map[string]interface{}{ + tt.env.matchLabelKey: tt.env.matchLabelValue, + } + utils.CreateMockUnstructured(matchLabels, tt.env.unstructuredTemplateSpec, tt.env.unstructuredName, tt.env.namespace, tt.env.apiVersion, tt.env.kind, tt.args.client) + utils.CreateMockPod(tt.env.podLabels, tt.env.podAnnotations, tt.env.podName, tt.env.namespace, tt.env.podNodeName, nil, tt.args.client) + got, gotErr := ValidateAndExtractContainers(tt.args.ctx, tt.args.client, tt.args.namespace, tt.args.targetRef, tt.args.containerPolicies) + SortContainersByContainerName(got) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ValidateAndExtractContainers() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("ValidateAndExtractContainers() gotErr = %v, wantErr = %v", gotErr, tt.wantErr) + } + }) + } +} + +func TestValidateAndExtractTargetRef(t *testing.T) { + type args struct { + targetRefReq v1alpha1.CrossVersionObjectReference + } + tests := []struct { + name string + args args + want v1alpha1.CrossVersionObjectReference + wantErr *customError.CustomError + }{ + { + name: "targetRefReq.Name empty", + args: args{ + targetRefReq: v1alpha1.CrossVersionObjectReference{ + Name: "", + Kind: "Deployment", + }, + }, + want: v1alpha1.CrossVersionObjectReference{}, + wantErr: customError.WorkloadNameIsEmptyError(), + }, + { + name: "targetRefReq.Kind unsupported", + args: args{ + targetRefReq: v1alpha1.CrossVersionObjectReference{ + Name: "test", + Kind: "ReplicaSet", + }, + }, + want: v1alpha1.CrossVersionObjectReference{}, + wantErr: customError.WorkloadsUnsupportedError("ReplicaSet", TargetRefKinds), + }, + { + name: "all right", + args: args{ + targetRefReq: v1alpha1.CrossVersionObjectReference{ + Name: "test1", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + }, + want: v1alpha1.CrossVersionObjectReference{ + Name: "test1", + Kind: "Deployment", + APIVersion: "apps/v1", + }, + wantErr: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotErr := ValidateAndExtractTargetRef(tt.args.targetRefReq) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ValidateAndExtractTargetRef() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("ValidateAndExtractTargetRef() gotErr = %v, want %v", gotErr, tt.wantErr) + } + }) + } +} + +func Test_validateAndExtractContainers(t *testing.T) { + errControlledValues := v1alpha1.ContainerControlledValues("errControlledValues") + type args struct { + containerPolicies []v1alpha1.ContainerResourcePolicy + existContainerList []string + } + tests := []struct { + name string + args args + want []Container + wantErr *customError.CustomError + }{ + { + name: customError.ContainerDuplicateMessage, + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "mockContainerName-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: utils.PtrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: utils.PtrInt32(10), + }, + }, + }, + { + ContainerName: "mockContainerName-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: utils.PtrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: utils.PtrInt32(10), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1"}, + }, + want: nil, + wantErr: customError.ContainerDuplicateError(customError.ContainerDuplicateMessage, "mockContainerName-1"), + }, + { + name: customError.ContainerNameEmptyMessage, + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "mockContainerName-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: utils.PtrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: utils.PtrInt32(10), + }, + }, + }, + { + ContainerName: "", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: utils.PtrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: utils.PtrInt32(10), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1"}, + }, + want: nil, + wantErr: customError.ContainerNameEmptyError(customError.ContainerNameEmptyMessage), + }, + { + name: customError.ContainerDuplicateMessage, + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "mockContainerName-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: utils.PtrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: utils.PtrInt32(10), + }, + }, + }, + { + ContainerName: "mockContainerName-2", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: utils.PtrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: utils.PtrInt32(10), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1"}, + }, + want: nil, + wantErr: customError.ContainersNotFoundError(customError.ContainerNotFoundMessage, "mockContainerName-2"), + }, + { + name: "Container name is only *", + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: utils.PtrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: utils.PtrInt32(20), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1", "mockContainerName-2"}, + }, + want: []Container{ + { + ContainerName: "mockContainerName-1", + ContainerConfigs: []ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: 10, + }, + { + ControlledResource: v1.ResourceMemory, + ResourceBufferPercent: 20, + }, + }, + }, + { + ContainerName: "mockContainerName-2", + ContainerConfigs: []ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: 10, + }, + { + ControlledResource: v1.ResourceMemory, + ResourceBufferPercent: 20, + }, + }, + }, + }, + wantErr: nil, + }, + { + name: "Container name list includes *", + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: utils.PtrInt32(20), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: utils.PtrInt32(20), + }, + }, + }, + { + ContainerName: "mockContainerName-1", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: utils.PtrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: utils.PtrInt32(10), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1", "mockContainerName-2", "mockContainerName-3"}, + }, + want: []Container{ + { + ContainerName: "mockContainerName-1", + ContainerConfigs: []ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: 10, + }, + { + ControlledResource: v1.ResourceMemory, + ResourceBufferPercent: 10, + }, + }, + }, + { + ContainerName: "mockContainerName-2", + ContainerConfigs: []ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: 20, + }, + { + ControlledResource: v1.ResourceMemory, + ResourceBufferPercent: 20, + }, + }, + }, + { + ContainerName: "mockContainerName-3", + ContainerConfigs: []ContainerConfig{ + { + ControlledResource: v1.ResourceCPU, + ResourceBufferPercent: 20, + }, + { + ControlledResource: v1.ResourceMemory, + ResourceBufferPercent: 20, + }, + }, + }, + }, + wantErr: nil, + }, + { + name: customError.ResourceNameUnsupportedMessage, + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: utils.PtrInt32(10), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: utils.PtrInt32(20), + }, + { + ResourceName: "errResource", + BufferPercent: utils.PtrInt32(20), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1", "mockContainerName-2"}, + }, + want: []Container{}, + wantErr: customError.ResourceNameUnsupportedError(customError.ResourceNameUnsupportedMessage, ResourceNames), + }, + { + name: customError.ResourceBuffersUnsupportedMessage, + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: utils.PtrInt32(101), + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: utils.PtrInt32(20), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1", "mockContainerName-2"}, + }, + want: []Container{}, + wantErr: customError.ResourceBuffersUnsupportedError(customError.ResourceBuffersUnsupportedMessage), + }, + { + name: customError.ResourceBuffersUnsupportedMessage, + args: args{ + containerPolicies: []v1alpha1.ContainerResourcePolicy{ + { + ContainerName: "*", + ControlledResourcesPolicies: []v1alpha1.ContainerControlledResourcesPolicy{ + { + ResourceName: v1.ResourceCPU, + BufferPercent: utils.PtrInt32(101), + ControlledValues: &errControlledValues, + }, + { + ResourceName: v1.ResourceMemory, + BufferPercent: utils.PtrInt32(20), + }, + }, + }, + }, + existContainerList: []string{"mockContainerName-1", "mockContainerName-2"}, + }, + want: []Container{}, + wantErr: customError.ControlledValuesUnsupportedError(customError.ResourceNameUnsupportedMessage, SupportControlledValues), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotErr := validateAndExtractContainers(tt.args.containerPolicies, tt.args.existContainerList) + SortContainersByContainerName(got) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("validateAndExtractContainers() got = %v, want %v", got, tt.want) + } + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("validateAndExtractContainers() gotErr = %v, want %v", gotErr, tt.wantErr) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/types/recommendation/validate_test_util.go b/pkg/controller/resource-recommend/types/recommendation/validate_test_util.go new file mode 100644 index 0000000000..a1e9218aa6 --- /dev/null +++ b/pkg/controller/resource-recommend/types/recommendation/validate_test_util.go @@ -0,0 +1,31 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recommendation + +import ( + "sort" +) + +type ByContainerName []Container + +func (c ByContainerName) Len() int { return len(c) } +func (c ByContainerName) Less(i, j int) bool { return c[i].ContainerName < c[j].ContainerName } +func (c ByContainerName) Swap(i, j int) { c[i], c[j] = c[j], c[i] } + +func SortContainersByContainerName(containers []Container) { + sort.Sort(ByContainerName(containers)) +} diff --git a/pkg/controller/resource-recommend/utils/k8s_resource.go b/pkg/controller/resource-recommend/utils/k8s_resource.go new file mode 100644 index 0000000000..55411e246e --- /dev/null +++ b/pkg/controller/resource-recommend/utils/k8s_resource.go @@ -0,0 +1,160 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + k8sclient "sigs.k8s.io/controller-runtime/pkg/client" + + spendsmartv1alpha1 "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" +) + +const ( + VCIAnnotationKeyOne = "vke.volcengine.com/burst-to-vci" + VCIAnnotationValueOne = "enforce" + VCIAnnotationKeyTwo = "vci.vke.volcengine.com/instance-id" +) + +func ConvertAndGetResource(ctx context.Context, client k8sclient.Client, namespace string, targetRef spendsmartv1alpha1.CrossVersionObjectReference) (*unstructured.Unstructured, error) { + klog.V(5).Infof("Get resource in", "targetRef", targetRef, "namespace", namespace) + obj := &unstructured.Unstructured{} + obj.SetAPIVersion(targetRef.APIVersion) + obj.SetKind(targetRef.Kind) + if err := client.Get(ctx, k8stypes.NamespacedName{Namespace: namespace, Name: targetRef.Name}, obj); err != nil { + return nil, err + } + return obj, nil +} + +func IsVolcengineContainerInstance(ctx context.Context, client k8sclient.Client, controller *unstructured.Unstructured) (bool, error) { + klog.V(5).Infof("Is the pod declared by the controller volcengine container instance", "controller", controller) + matchLabels, found, err := unstructured.NestedMap(controller.Object, "spec", "selector", "matchLabels") + if err != nil { + return false, errors.Wrapf(err, "list spec.selector.matchLabels in namespace %v controller %v failed", controller.GetNamespace(), controller.GetName()) + } + if !found { + return false, errors.Errorf("spec.selector.matchLabels not found in the controller") + } + podList := &v1.PodList{} + labelSet := make(labels.Set) + for key, value := range matchLabels { + labelSet[key] = fmt.Sprintf("%v", value) + } + labelSelector := labels.SelectorFromSet(labelSet) + listOpts := &k8sclient.ListOptions{ + LabelSelector: labelSelector, + Namespace: controller.GetNamespace(), + } + err = client.List(ctx, podList, listOpts) + if err != nil { + return false, errors.Wrapf(err, "list pods in namespace %v with lables %v failed", controller.GetNamespace(), labelSelector) + } + if podList == nil || len(podList.Items) == 0 { + return false, errors.Errorf("pods in namespace %v with lables %v not found", controller.GetNamespace(), labelSelector) + } + for _, pod := range podList.Items { + if value, ok := pod.Annotations[VCIAnnotationKeyOne]; ok { + if value == VCIAnnotationValueOne { + return true, nil + } + } + if _, ok := pod.Annotations[VCIAnnotationKeyTwo]; ok { + return true, nil + } + if strings.HasPrefix(pod.Spec.NodeName, "vci-") { + return true, nil + } + } + return false, nil +} + +func GetAllClaimedContainers(controller *unstructured.Unstructured) ([]string, error) { + klog.V(5).InfoS("Get all controller claimed containers", "controller", controller) + templateSpec, found, err := unstructured.NestedMap(controller.Object, "spec", "template", "spec") + if err != nil { + return nil, errors.Wrapf(err, "unstructured.NestedMap err") + } + if !found { + return nil, errors.Errorf("spec.template.spec not found in the controller") + } + containersList, found, err := unstructured.NestedSlice(templateSpec, "containers") + if err != nil { + return nil, errors.Wrapf(err, "unstructured.NestedSlice err") + } + if !found { + return nil, errors.Errorf("failure to find containers in the controller") + } + + containerNames := make([]string, 0, len(containersList)) + for _, container := range containersList { + containerMap, ok := container.(map[string]interface{}) + if !ok { + klog.Errorf("Unable to convert container:%v to map[string]interface{}", container) + continue + } + name, found, err := unstructured.NestedString(containerMap, "name") + if err != nil || !found { + klog.Errorf("Container name not found or get container name err:%v in the containerMap %v", err, containerMap) + continue + } + containerNames = append(containerNames, name) + } + return containerNames, nil +} + +type patchRecord struct { + Op string `json:"op,inline"` + Path string `json:"path,inline"` + Value interface{} `json:"value"` +} + +func PatchUpdateResourceRecommend(client k8sclient.Client, namespaceName k8stypes.NamespacedName, + resourceRecommend *spendsmartv1alpha1.ResourceRecommend) error { + obj := &spendsmartv1alpha1.ResourceRecommend{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName.Name, + Namespace: namespaceName.Namespace, + }, + } + patches := []patchRecord{{ + Op: "replace", + Path: "/status", + Value: resourceRecommend.Status, + }} + + patch, err := json.Marshal(patches) + if err != nil { + return errors.Wrapf(err, "failed to Marshal resourceRecommend: %+v", resourceRecommend) + } + patchDate := k8sclient.RawPatch(k8stypes.JSONPatchType, patch) + err = client.Status().Patch(context.TODO(), obj, patchDate) + if err != nil { + return errors.Wrapf(err, "failed to patch resource") + } + return nil +} diff --git a/pkg/controller/resource-recommend/utils/k8s_resource_test.go b/pkg/controller/resource-recommend/utils/k8s_resource_test.go new file mode 100644 index 0000000000..2f38f1b82f --- /dev/null +++ b/pkg/controller/resource-recommend/utils/k8s_resource_test.go @@ -0,0 +1,656 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + "reflect" + "testing" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/kubewharf/katalyst-api/pkg/apis/recommendation/v1alpha1" +) + +func TestConvertAndGetResource(t *testing.T) { + type args struct { + ctx context.Context + client client.Client + namespace string + targetRef v1alpha1.CrossVersionObjectReference + } + type env struct { + namespace string + kind string + name string + apiVersion string + } + tests := []struct { + name string + args args + env env + want *unstructured.Unstructured + wantErr bool + }{ + { + name: "Get resource failed", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "fakeNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "mockResourceName", + APIVersion: "apps/v1", + }, + }, + env: env{ + namespace: "fakeNamespace", + kind: "Deployment", + name: "mockResourceName1", + apiVersion: "apps/v1", + }, + wantErr: true, + }, + { + name: "Get resource failed", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + namespace: "fakeNamespace", + targetRef: v1alpha1.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "mockResourceName", + APIVersion: "apps/v1", + }, + }, + env: env{ + namespace: "fakeNamespace", + kind: "Deployment", + name: "mockResourceName", + apiVersion: "apps/v1", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + CreateMockUnstructured(nil, nil, tt.env.name, tt.env.namespace, tt.env.apiVersion, tt.env.kind, tt.args.client) + _, err := ConvertAndGetResource(tt.args.ctx, tt.args.client, tt.args.namespace, tt.args.targetRef) + if (err != nil) != tt.wantErr { + t.Errorf("ConvertAndGetResource() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestGetAllClaimedContainers(t *testing.T) { + type args struct { + controller *unstructured.Unstructured + } + type env struct { + needCreate bool + unstructuredTemplateSpec map[string]interface{} + } + tests := []struct { + name string + args args + env env + want []string + wantErr bool + }{ + { + name: "spec.template.spec not found", + args: args{ + controller: &unstructured.Unstructured{}, + }, + env: env{ + needCreate: false, + unstructuredTemplateSpec: map[string]interface{}{}, + }, + want: nil, + wantErr: true, + }, + { + name: "nestedMap failed", + args: args{ + controller: &unstructured.Unstructured{}, + }, + env: env{ + needCreate: true, + unstructuredTemplateSpec: map[string]interface{}{ + "container": []interface{}{ + map[string]interface{}{ + "name": "container-1", + "image": "image:latest", + }, + map[string]interface{}{ + "name": "container-2", + "image": "image:latest", + }, + }, + }, + }, + want: nil, + wantErr: true, + }, + { + name: "container name not found", + args: args{ + controller: &unstructured.Unstructured{}, + }, + env: env{ + needCreate: true, + unstructuredTemplateSpec: map[string]interface{}{ + "containers": []interface{}{ + map[string]interface{}{ + "image": "image:latest", + }, + map[string]interface{}{ + "image": "image:latest", + }, + }, + }, + }, + want: []string{}, + wantErr: false, + }, + { + name: "all right", + args: args{ + controller: &unstructured.Unstructured{}, + }, + env: env{ + needCreate: true, + unstructuredTemplateSpec: map[string]interface{}{ + "containers": []interface{}{ + map[string]interface{}{ + "name": "container-1", + "image": "image:latest", + }, + map[string]interface{}{ + "name": "container-2", + "image": "image:latest", + }, + }, + }, + }, + want: []string{"container-1", "container-2"}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.env.needCreate { + tt.args.controller.SetName("mockName") + unstructured.SetNestedMap(tt.args.controller.Object, tt.env.unstructuredTemplateSpec, "spec", "template", "spec") + } + got, err := GetAllClaimedContainers(tt.args.controller) + if (err != nil) != tt.wantErr { + t.Errorf("GetAllClaimedContainers() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetAllClaimedContainers() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIsVolcengineContainerInstance(t *testing.T) { + type args struct { + ctx context.Context + client client.Client + controller *unstructured.Unstructured + } + type env struct { + podLabels map[string]string + podAnnotations map[string]string + matchLabels map[string]interface{} + podName string + podNodeName string + unstructuredName string + unstructuredTemplateSpec map[string]interface{} + namespace string + kind string + apiVersion string + } + tests := []struct { + name string + args args + env env + want bool + wantErr bool + }{ + { + name: "spec.selector.matchLabels not found", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + controller: &unstructured.Unstructured{}, + }, + env: env{}, + want: false, + wantErr: true, + }, + { + name: "pods not found", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + controller: &unstructured.Unstructured{}, + }, + env: env{ + podLabels: map[string]string{ + "app": "fakePodLabels", + }, + matchLabels: map[string]interface{}{ + "app": "mockPodLabels", + }, + podName: "mockPodName", + podNodeName: "mockPodNodeName", + unstructuredName: "mockUnstructuredName", + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: false, + wantErr: true, + }, + { + name: "vci - vke.volcengine.com/burst-to-vci:enforce", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + controller: &unstructured.Unstructured{}, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels", + }, + matchLabels: map[string]interface{}{ + "app": "mockPodLabels", + }, + podAnnotations: map[string]string{ + "vke.volcengine.com/burst-to-vci": "enforce", + }, + podName: "mockPodName", + podNodeName: "mockPodNodeName", + unstructuredName: "mockUnstructuredName", + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: true, + wantErr: false, + }, + { + name: "vci - vke.volcengine.com/burst-to-vci:no-enforce", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + controller: &unstructured.Unstructured{}, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels", + }, + matchLabels: map[string]interface{}{ + "app": "mockPodLabels", + }, + podAnnotations: map[string]string{ + "vke.volcengine.com/burst-to-vci": "no-enforce", + }, + podName: "mockPodName", + podNodeName: "mockPodNodeName", + unstructuredName: "mockUnstructuredName", + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: false, + wantErr: false, + }, + { + name: "vci - vke.volcengine.com/burst-to-vci:no-enforce", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + controller: &unstructured.Unstructured{}, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels", + }, + matchLabels: map[string]interface{}{ + "app": "mockPodLabels", + }, + podAnnotations: map[string]string{ + "vke.volcengine.com/burst-to-vci": "no-enforce", + }, + podName: "mockPodName", + podNodeName: "mockPodNodeName", + unstructuredName: "mockUnstructuredName", + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: false, + wantErr: false, + }, + { + name: "vci - NodeName", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + controller: &unstructured.Unstructured{}, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels", + }, + matchLabels: map[string]interface{}{ + "app": "mockPodLabels", + }, + podAnnotations: map[string]string{}, + podName: "mockPodName", + podNodeName: "vci-mockPodNodeName", + unstructuredName: "mockUnstructuredName", + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: true, + wantErr: false, + }, + { + name: "all right", + args: args{ + ctx: context.TODO(), + client: fake.NewClientBuilder().Build(), + controller: &unstructured.Unstructured{}, + }, + env: env{ + podLabels: map[string]string{ + "app": "mockPodLabels", + }, + matchLabels: map[string]interface{}{ + "app": "mockPodLabels", + }, + podAnnotations: map[string]string{}, + podName: "mockPodName", + podNodeName: "vc-mockPodNodeName", + unstructuredName: "mockUnstructuredName", + namespace: "mockNamespace", + kind: "Deployment", + apiVersion: "apps/v1", + }, + want: false, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + CreateMockUnstructured(tt.env.matchLabels, tt.env.unstructuredTemplateSpec, tt.env.unstructuredName, + tt.env.namespace, tt.env.apiVersion, tt.env.kind, tt.args.client) + CreateMockPod(tt.env.podLabels, tt.env.podAnnotations, tt.env.podName, tt.env.namespace, tt.env.podNodeName, nil, tt.args.client) + tt.args.controller.SetKind(tt.env.kind) + tt.args.controller.SetAPIVersion(tt.env.apiVersion) + tt.args.client.Get(tt.args.ctx, client.ObjectKey{ + Name: tt.env.unstructuredName, + Namespace: tt.env.namespace, + }, tt.args.controller) + got, err := IsVolcengineContainerInstance(tt.args.ctx, tt.args.client, tt.args.controller) + if (err != nil) != tt.wantErr { + t.Errorf("IsVolcengineContainerInstance() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("IsVolcengineContainerInstance() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPatchUpdateResourceRecommend(t *testing.T) { + type args struct { + client client.Client + namespaceName types.NamespacedName + resourceRecommend *v1alpha1.ResourceRecommend + } + tests := []struct { + name string + args args + wantErr bool + wantResourceRecommend *v1alpha1.ResourceRecommend + }{ + { + name: "all right 1", + args: args{ + client: fake.NewClientBuilder().Build(), + namespaceName: types.NamespacedName{ + Name: "mockName", + Namespace: "mockNamespace", + }, + resourceRecommend: &v1alpha1.ResourceRecommend{ + TypeMeta: metav1.TypeMeta{ + Kind: "ResourceRecommend", + APIVersion: "autoscaling.katalyst.kubewharf.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mockName", + Namespace: "mockNamespace", + }, + Status: v1alpha1.ResourceRecommendStatus{}, + }, + }, + wantResourceRecommend: &v1alpha1.ResourceRecommend{ + Status: v1alpha1.ResourceRecommendStatus{ + RecommendResources: &v1alpha1.RecommendResources{ + ContainerRecommendations: []v1alpha1.ContainerResources{ + { + ContainerName: "ContainerName1", + Requests: &v1alpha1.ContainerResourceList{}, + }, + { + ContainerName: "ContainerName2", + Requests: &v1alpha1.ContainerResourceList{}, + }, + { + ContainerName: "ContainerName3", + Requests: &v1alpha1.ContainerResourceList{}, + }, + { + ContainerName: "ContainerName4", + Requests: &v1alpha1.ContainerResourceList{}, + }, + }, + }, + Conditions: []v1alpha1.ResourceRecommendCondition{ + { + Type: v1alpha1.Validated, + Reason: "reason1", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason2", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason3", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason4", + Status: v1.ConditionTrue, + Message: "Message", + }, + }, + }, + }, + wantErr: false, + }, + { + name: "all right 2", + args: args{ + client: fake.NewClientBuilder().Build(), + namespaceName: types.NamespacedName{ + Name: "mockName", + Namespace: "mockNamespace", + }, + resourceRecommend: &v1alpha1.ResourceRecommend{ + TypeMeta: metav1.TypeMeta{ + Kind: "ResourceRecommend", + APIVersion: "autoscaling.katalyst.kubewharf.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "mockName", + Namespace: "mockNamespace", + }, + Status: v1alpha1.ResourceRecommendStatus{ + RecommendResources: &v1alpha1.RecommendResources{ + ContainerRecommendations: []v1alpha1.ContainerResources{ + { + ContainerName: "ContainerName0", + }, + { + ContainerName: "ContainerName1", + }, + { + ContainerName: "ContainerName2", + }, + { + ContainerName: "ContainerName3", + }, + }, + }, + Conditions: []v1alpha1.ResourceRecommendCondition{ + { + Type: v1alpha1.Validated, + Reason: "reason0", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason1", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason2", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason3", + Status: v1.ConditionTrue, + Message: "Message", + }, + }, + }, + }, + }, + wantResourceRecommend: &v1alpha1.ResourceRecommend{ + Status: v1alpha1.ResourceRecommendStatus{ + RecommendResources: &v1alpha1.RecommendResources{ + ContainerRecommendations: []v1alpha1.ContainerResources{ + { + ContainerName: "ContainerName1", + Requests: &v1alpha1.ContainerResourceList{}, + }, + { + ContainerName: "ContainerName2", + Requests: &v1alpha1.ContainerResourceList{}, + }, + { + ContainerName: "ContainerName3", + Requests: &v1alpha1.ContainerResourceList{}, + }, + { + ContainerName: "ContainerName4", + Requests: &v1alpha1.ContainerResourceList{}, + }, + }, + }, + Conditions: []v1alpha1.ResourceRecommendCondition{ + { + Type: v1alpha1.Validated, + Reason: "reason1", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason2", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason3", + Status: v1.ConditionTrue, + Message: "Message", + }, + { + Type: v1alpha1.Validated, + Reason: "reason4", + Status: v1.ConditionTrue, + Message: "Message", + }, + }, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v1alpha1.AddToScheme(tt.args.client.Scheme()) + tt.args.client.Create(context.TODO(), tt.args.resourceRecommend) + if err := PatchUpdateResourceRecommend(tt.args.client, tt.args.namespaceName, tt.args.resourceRecommend); (err != nil) != tt.wantErr { + t.Errorf("PatchUpdateResourceRecommend() error = %v, wantErr %v", err, tt.wantErr) + } + if err := PatchUpdateResourceRecommend(tt.args.client, tt.args.namespaceName, tt.wantResourceRecommend); (err != nil) != tt.wantErr { + t.Errorf("PatchUpdateResourceRecommend() error = %v, wantErr %v", err, tt.wantErr) + } + gotResourceRecommend := &v1alpha1.ResourceRecommend{} + tt.args.client.Get(context.TODO(), client.ObjectKey{ + Name: tt.args.namespaceName.Name, + Namespace: tt.args.namespaceName.Namespace, + }, gotResourceRecommend) + if !reflect.DeepEqual(gotResourceRecommend.Status, tt.wantResourceRecommend.Status) { + t.Errorf("PatchUpdateResourceRecommend() gotResourceRecommend.Status = %v, wantResourceRecommend.Status = %v", + gotResourceRecommend.Status, tt.wantResourceRecommend.Status) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/utils/k8s_resource_test_util.go b/pkg/controller/resource-recommend/utils/k8s_resource_test_util.go new file mode 100644 index 0000000000..10ba23248c --- /dev/null +++ b/pkg/controller/resource-recommend/utils/k8s_resource_test_util.go @@ -0,0 +1,52 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "context" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func CreateMockPod(labels, annotations map[string]string, name, namespace, nodeName string, containers []v1.Container, client client.Client) { + pod := &v1.Pod{ + Spec: v1.PodSpec{ + NodeName: nodeName, + Containers: containers, + }, + } + pod.SetLabels(labels) + pod.SetAnnotations(annotations) + pod.SetName(name) + pod.SetNamespace(namespace) + + client.Create(context.TODO(), pod) +} + +func CreateMockUnstructured(matchLabels, unstructuredTemplateSpec map[string]interface{}, name, namespace, apiVersion, kind string, client client.Client) { + collectorObject := &unstructured.Unstructured{} + collectorObject.SetName(name) + collectorObject.SetNamespace(namespace) + collectorObject.SetKind(kind) + collectorObject.SetAPIVersion(apiVersion) + unstructured.SetNestedMap(collectorObject.Object, matchLabels, "spec", "selector", "matchLabels") + unstructured.SetNestedMap(collectorObject.Object, unstructuredTemplateSpec, "spec", "template", "spec") + client.Create(context.TODO(), collectorObject) + +} diff --git a/pkg/controller/resource-recommend/utils/list.go b/pkg/controller/resource-recommend/utils/list.go new file mode 100644 index 0000000000..af7339b76f --- /dev/null +++ b/pkg/controller/resource-recommend/utils/list.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import "reflect" + +// SliceContains returns true if an element is present in a slice. +func SliceContains(list interface{}, elem interface{}) bool { + if list == nil || elem == nil { + return false + } + listV := reflect.ValueOf(list) + + if listV.Kind() == reflect.Slice { + for i := 0; i < listV.Len(); i++ { + item := listV.Index(i).Interface() + + target := reflect.ValueOf(elem).Convert(reflect.TypeOf(item)).Interface() + if ok := reflect.DeepEqual(item, target); ok { + return true + } + } + } + return false +} diff --git a/pkg/controller/resource-recommend/utils/list_test.go b/pkg/controller/resource-recommend/utils/list_test.go new file mode 100644 index 0000000000..14808cf045 --- /dev/null +++ b/pkg/controller/resource-recommend/utils/list_test.go @@ -0,0 +1,147 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "testing" +) + +func TestSliceContains(t *testing.T) { + type args struct { + list interface{} + elem interface{} + } + type myType struct { + name string + id int + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "int - true", + args: args{ + list: []int{1, 2, 3}, + elem: 1, + }, + want: true, + }, + { + name: "int - false", + args: args{ + list: []int{1, 2, 3}, + elem: 4, + }, + want: false, + }, + { + name: "string - true", + args: args{ + list: []string{"1", "2", "3"}, + elem: "1", + }, + want: true, + }, + { + name: "string - false", + args: args{ + list: []string{"1", "2", "3"}, + elem: "4", + }, + want: false, + }, + { + name: "string - false", + args: args{ + list: []string{"1", "2", "3"}, + elem: 4, + }, + want: false, + }, + { + name: "string - true", + args: args{ + list: []string{"1", "2", "3"}, + elem: "1", + }, + want: true, + }, + { + name: "myType - true", + args: args{ + list: []myType{ + { + name: "name", + id: 1, + }, + { + name: "name", + id: 2, + }, + }, + elem: myType{ + name: "name", + id: 2, + }, + }, + want: true, + }, + { + name: "myType - false", + args: args{ + list: []myType{ + { + name: "name", + id: 1, + }, + { + name: "name", + id: 2, + }, + }, + elem: myType{ + name: "name-1", + id: 2, + }, + }, + want: false, + }, + { + name: "list - nil", + args: args{ + elem: 1, + }, + want: false, + }, + { + name: "elem - nil", + args: args{ + list: []int{1, 2, 3}, + }, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := SliceContains(tt.args.list, tt.args.elem); got != tt.want { + t.Errorf("SliceContains() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/controller/resource-recommend/utils/log/logger.go b/pkg/controller/resource-recommend/utils/log/logger.go new file mode 100644 index 0000000000..8e37e66f7e --- /dev/null +++ b/pkg/controller/resource-recommend/utils/log/logger.go @@ -0,0 +1,104 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package log + +import ( + "context" + + "github.com/google/uuid" + "k8s.io/klog/v2" +) + +var ( + fieldsKey = &[1]int{1} +) + +const LinkIDKey = "linkId" + +// InitContext set context info such as accountId for logger +func InitContext(ctx context.Context) context.Context { + fields := []interface{}{ + LinkIDKey, uuid.New(), + } + return context.WithValue(ctx, fieldsKey, fields) +} + +// SetKeysAndValues set KeysAndValues into ctx +func SetKeysAndValues(ctx context.Context, keysAndValues ...interface{}) context.Context { + ctxFields := ctx.Value(fieldsKey) + if ctxFields == nil { + return context.WithValue(ctx, fieldsKey, keysAndValues) + } + if fields, ok := ctxFields.([]interface{}); !ok { + return context.WithValue(ctx, fieldsKey, keysAndValues) + } else { + fields = append(fields, keysAndValues...) + return context.WithValue(ctx, fieldsKey, fields) + } +} + +func GetCtxFields(ctx context.Context) []interface{} { + ctxFields := ctx.Value(fieldsKey) + if ctxFields == nil { + return []interface{}{} + } + if fields, ok := ctxFields.([]interface{}); ok { + return fields + } + return []interface{}{} +} + +// ErrorS print error log +func ErrorS(ctx context.Context, err error, msg string, keysAndValues ...interface{}) { + fields := GetCtxFields(ctx) + fields = append(fields, keysAndValues...) + if len(fields) == 0 { + klog.ErrorSDepth(1, err, msg) + } else { + klog.ErrorSDepth(1, err, msg, fields...) + } +} + +// InfoS print info log +func InfoS(ctx context.Context, msg string, keysAndValues ...interface{}) { + fields := GetCtxFields(ctx) + fields = append(fields, keysAndValues...) + if len(fields) == 0 { + klog.InfoSDepth(1, msg) + } else { + klog.InfoSDepth(1, msg, fields...) + } +} + +type Verbose struct { + klog.Verbose +} + +func V(level klog.Level) Verbose { + return Verbose{Verbose: klog.V(level)} +} + +// InfoS print info log +func (v Verbose) InfoS(ctx context.Context, msg string, keysAndValues ...interface{}) { + fields := GetCtxFields(ctx) + fields = append(fields, keysAndValues...) + if len(fields) == 0 { + v.Verbose.InfoSDepth(1, msg) + } else { + v.Verbose.InfoSDepth(1, msg, fields...) + } +} diff --git a/pkg/controller/resource-recommend/utils/log/logger_test.go b/pkg/controller/resource-recommend/utils/log/logger_test.go new file mode 100644 index 0000000000..f22c3c2241 --- /dev/null +++ b/pkg/controller/resource-recommend/utils/log/logger_test.go @@ -0,0 +1,343 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package log + +import ( + "context" + "flag" + "fmt" + "os" + "regexp" + "runtime" + "testing" + + "k8s.io/klog/v2" +) + +func TestInitContextAndGetCtxFields(t *testing.T) { + type args struct { + ctx context.Context + } + tests := []struct { + name string + args args + }{ + { + name: "case1", + args: args{ + ctx: context.Background(), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := InitContext(tt.args.ctx) + fields := GetCtxFields(ctx) + if len(fields) != 2 { + t.Errorf("The fields length must be 2") + } + if fields[0] != LinkIDKey { + t.Errorf("fields must be included: %s", LinkIDKey) + } + }) + } +} + +func TestSetKeysAndValues(t *testing.T) { + type tempStruct struct { + a int + b string + } + tempStruct1 := &tempStruct{111, "aaa"} + type args struct { + ctx context.Context + keysAndValues []interface{} + } + type want struct { + index int + wantValue interface{} + } + tests := []struct { + name string + args args + want []want + }{ + { + name: "case1", + args: args{ + ctx: InitContext(context.Background()), + keysAndValues: []interface{}{"k1", "v1", "k2", "v2"}, + }, + want: []want{ + { + index: 2, wantValue: "k1", + }, + { + index: 3, wantValue: "v1", + }, + { + index: 4, wantValue: "k2", + }, + { + index: 5, wantValue: "v2", + }, + }, + }, + { + name: "case2", + args: args{ + ctx: InitContext(context.Background()), + keysAndValues: []interface{}{"k1", tempStruct{111, "aaa"}}, + }, + want: []want{ + { + index: 2, wantValue: "k1", + }, + { + index: 3, wantValue: tempStruct{111, "aaa"}, + }, + }, + }, + { + name: "case3", + args: args{ + ctx: InitContext(context.Background()), + keysAndValues: []interface{}{"k1", tempStruct1}, + }, + want: []want{ + { + index: 2, wantValue: "k1", + }, + { + index: 3, wantValue: tempStruct1, + }, + }, + }, + { + name: "case4", + args: args{ + ctx: context.Background(), + keysAndValues: []interface{}{"k1", tempStruct1}, + }, + want: []want{ + { + index: 0, wantValue: "k1", + }, + { + index: 1, wantValue: tempStruct1, + }, + }, + }, + { + name: "case4", + args: args{ + ctx: InitContext(context.Background()), + keysAndValues: []interface{}{}, + }, + want: []want{}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := SetKeysAndValues(tt.args.ctx, tt.args.keysAndValues...) + fields := GetCtxFields(ctx) + for _, w := range tt.want { + if fields[w.index] != w.wantValue { + t.Errorf("set KeysAndValues error, index(%d) should be %v, got: %v", w.index, w.wantValue, fields[w.index]) + } else { + t.Logf("set KeysAndValues successful, index(%d) should be %v, got: %v", w.index, w.wantValue, fields[w.index]) + } + } + }) + } +} + +// 自定义的测试日志记录器，实现 io.Writer 接口 +type TestLogger struct { + RegularxEpression string + t *testing.T +} + +func (tl *TestLogger) Write(p []byte) (n int, err error) { + str := string(p) + + regex, err := regexp.Compile(tl.RegularxEpression) + if err != nil { + fmt.Println("Error compiling regex:", err) + return + } + isMatch := regex.MatchString(str) + if !isMatch { + tl.t.Errorf("match the log content error, regex is %s, got: %s", tl.RegularxEpression, str) + } + + return len(p), nil +} + +func TestErrorS(t *testing.T) { + type args struct { + ctx context.Context + err error + msg string + keysAndValues []interface{} + } + tests := []struct { + name string + args args + wantLogStr string + }{ + { + name: "case1", + args: args{ + ctx: SetKeysAndValues(context.Background(), "baseK1", "baseV1"), + err: fmt.Errorf("err1"), + msg: "errMsg1", + keysAndValues: []interface{}{"k1", "v1", "k2", "v2"}, + }, + wantLogStr: "E\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6} %d logger_test.go:%d] \"errMsg1\" err=\"err1\" baseK1=\"baseV1\" k1=\"v1\" k2=\"v2\"\n", + }, + { + name: "case2", + args: args{ + ctx: context.Background(), + err: fmt.Errorf("err1"), + msg: "errMsg1", + keysAndValues: []interface{}{}, + }, + wantLogStr: "E\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6} %d logger_test.go:%d] \"errMsg1\" err=\"err1\"\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, _, line, ok := runtime.Caller(0) + if !ok { + panic("Unable to retrieve caller information") + } + pid := os.Getpid() + klog.LogToStderr(false) + klog.SetOutput(&TestLogger{ + RegularxEpression: fmt.Sprintf(tt.wantLogStr, pid, line+10), + t: t, + }) + ErrorS(tt.args.ctx, tt.args.err, tt.args.msg, tt.args.keysAndValues...) + }) + } +} + +func TestInfoS(t *testing.T) { + type args struct { + ctx context.Context + msg string + keysAndValues []interface{} + } + tests := []struct { + name string + args args + wantLogStr string + }{ + { + name: "case1", + args: args{ + ctx: SetKeysAndValues(context.Background(), "baseK1", "baseV1"), + msg: "InfoMsg1", + keysAndValues: []interface{}{"k1", "v1", "k2", "v2"}, + }, + wantLogStr: "I\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6} %d logger_test.go:%d] \"InfoMsg1\" baseK1=\"baseV1\" k1=\"v1\" k2=\"v2\"\n", + }, + { + name: "case2", + args: args{ + ctx: context.Background(), + msg: "InfoMsg1", + keysAndValues: []interface{}{}, + }, + wantLogStr: "I\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6} %d logger_test.go:%d] \"InfoMsg1\"\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, _, line, ok := runtime.Caller(0) + if !ok { + panic("Unable to retrieve caller information") + } + pid := os.Getpid() + klog.LogToStderr(false) + klog.SetOutput(&TestLogger{ + RegularxEpression: fmt.Sprintf(tt.wantLogStr, pid, line+10), + t: t, + }) + InfoS(tt.args.ctx, tt.args.msg, tt.args.keysAndValues...) + }) + } +} + +func TestVerbose_InfoS(t *testing.T) { + logLevel := klog.Level(1) + klog.InitFlags(nil) + flag.Set("v", logLevel.String()) + flag.Parse() + + type fields struct { + Verbose klog.Verbose + } + type args struct { + ctx context.Context + msg string + keysAndValues []interface{} + } + tests := []struct { + name string + fields fields + args args + wantLogStr string + }{ + { + name: "case1", + args: args{ + ctx: SetKeysAndValues(context.Background(), "baseK1", "baseV1"), + msg: "InfoMsg1", + keysAndValues: []interface{}{"k1", "v1", "k2", "v2"}, + }, + wantLogStr: "I\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6} %d logger_test.go:%d] \"InfoMsg1\" baseK1=\"baseV1\" k1=\"v1\" k2=\"v2\"\n", + }, + { + name: "case2", + args: args{ + ctx: context.Background(), + msg: "InfoMsg1", + keysAndValues: []interface{}{}, + }, + wantLogStr: "I\\d{4} \\d{2}:\\d{2}:\\d{2}\\.\\d{6} %d logger_test.go:%d] \"InfoMsg1\"\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, _, line, ok := runtime.Caller(0) + if !ok { + panic("Unable to retrieve caller information") + } + pid := os.Getpid() + klog.LogToStderr(false) + klog.SetOutput(&TestLogger{ + RegularxEpression: fmt.Sprintf(tt.wantLogStr, pid, line+10), + t: t, + }) + V(logLevel).InfoS(tt.args.ctx, tt.args.msg, tt.args.keysAndValues...) + klog.Flush() + }) + } +} diff --git a/pkg/controller/resource-recommend/utils/ptr.go b/pkg/controller/resource-recommend/utils/ptr.go new file mode 100644 index 0000000000..46804ce6fd --- /dev/null +++ b/pkg/controller/resource-recommend/utils/ptr.go @@ -0,0 +1,25 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +func PtrString(str string) *string { + return &str +} + +func PtrInt32(i int32) *int32 { + return &i +} diff --git a/pkg/controller/resource-recommend/utils/string.go b/pkg/controller/resource-recommend/utils/string.go new file mode 100644 index 0000000000..83e25be632 --- /dev/null +++ b/pkg/controller/resource-recommend/utils/string.go @@ -0,0 +1,37 @@ +/* +Copyright 2022 The Katalyst Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "encoding/json" // nolint: byted_substitute_packages + "fmt" +) + +func StructToString(val interface{}) string { + if val == nil { + return "" + } + data, err := json.Marshal(val) + if err != nil { + return fmt.Sprintf("%+v", val) + } + return BytesToString(data) +} + +func BytesToString(b []byte) string { + return string(b) +}