diff --git a/cluster-autoscaler/core/scale_test_common.go b/cluster-autoscaler/core/scale_test_common.go index f8550e559dd..ce50583b850 100644 --- a/cluster-autoscaler/core/scale_test_common.go +++ b/cluster-autoscaler/core/scale_test_common.go @@ -36,7 +36,6 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroups" "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset" "k8s.io/autoscaler/cluster-autoscaler/processors/nodeinfos" - "k8s.io/autoscaler/cluster-autoscaler/processors/podtemplates" "k8s.io/autoscaler/cluster-autoscaler/processors/status" "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" @@ -143,7 +142,6 @@ func NewTestProcessors() *processors.AutoscalingProcessors { NodeGroupManager: nodegroups.NewDefaultNodeGroupManager(), NodeInfoProcessor: nodeinfos.NewDefaultNodeInfoProcessor(), NodeGroupConfigProcessor: nodegroupconfig.NewDefaultNodeGroupConfigProcessor(), - PodTemplateListProcessor: podtemplates.NewDefaultPodTemplateListProcessor(), } } diff --git a/cluster-autoscaler/core/static_autoscaler.go b/cluster-autoscaler/core/static_autoscaler.go index f6ad308825d..de7f2cf2604 100644 --- a/cluster-autoscaler/core/static_autoscaler.go +++ b/cluster-autoscaler/core/static_autoscaler.go @@ -251,14 +251,6 @@ func (a *StaticAutoscaler) RunOnce(currentTime time.Time) errors.AutoscalerError return errors.ToAutoscalerError(errors.ApiCallError, err) } - extraDaemonsets, err := a.processors.PodTemplateListProcessor.ExtraDaemonsets(autoscalingContext) - if err != nil { - klog.Errorf("Failed to get extra daemonset list: %v", err) - return errors.ToAutoscalerError(errors.ApiCallError, err) - } else if len(extraDaemonsets) > 0 { - daemonsets = append(daemonsets, extraDaemonsets...) - } - // Call CloudProvider.Refresh before any other calls to cloud provider. err = a.AutoscalingContext.CloudProvider.Refresh() if err != nil { diff --git a/cluster-autoscaler/main.go b/cluster-autoscaler/main.go index c245bef07be..f52e1e564ce 100644 --- a/cluster-autoscaler/main.go +++ b/cluster-autoscaler/main.go @@ -43,7 +43,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/metrics" ca_processors "k8s.io/autoscaler/cluster-autoscaler/processors" "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset" - "k8s.io/autoscaler/cluster-autoscaler/processors/podtemplates" + "k8s.io/autoscaler/cluster-autoscaler/processors/nodeinfos/podtemplates" "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" @@ -332,7 +332,7 @@ func buildAutoscaler() (core.Autoscaler, error) { // Enable PodTemplateListProcessor if needed if autoscalingOptions.ExtraDaemonsetsFromPodTemplates { - opts.Processors.PodTemplateListProcessor = podtemplates.NewActivePodTemplateListProcessor(kubeClient) + opts.Processors.NodeInfoProcessor = podtemplates.NewNodeInfoWithPodTemplateProcessor(kubeClient) } // These metrics should be published only once. diff --git a/cluster-autoscaler/processors/nodeinfos/podtemplates/node_info_with_podtemplates_processors.go b/cluster-autoscaler/processors/nodeinfos/podtemplates/node_info_with_podtemplates_processors.go new file mode 100644 index 00000000000..91a0a923e3c --- /dev/null +++ b/cluster-autoscaler/processors/nodeinfos/podtemplates/node_info_with_podtemplates_processors.go @@ -0,0 +1,151 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podtemplates + +import ( + "context" + "fmt" + "time" + + apiv1 "k8s.io/api/core/v1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + + client "k8s.io/client-go/kubernetes" + v1lister "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + + ca_context "k8s.io/autoscaler/cluster-autoscaler/context" + "k8s.io/autoscaler/cluster-autoscaler/processors/nodeinfos" + "k8s.io/autoscaler/cluster-autoscaler/simulator" + "k8s.io/autoscaler/cluster-autoscaler/utils/errors" + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" +) + +const ( + // PodTemplateDaemonSetLabelKey use as label key on PodTemplate corresponding to an extra Daemonset. + PodTemplateDaemonSetLabelKey = "cluster-autoscaler.kubernetes.io/daemonset-pod" + // PodTemplateDaemonSetLabelValueTrue use as PodTemplateDaemonSetLabelKey label value. + PodTemplateDaemonSetLabelValueTrue = "true" +) + +// NewNodeInfoWithPodTemplateProcessor returns a default instance of NodeInfoProcessor. +func NewNodeInfoWithPodTemplateProcessor(kubeClient client.Interface) nodeinfos.NodeInfoProcessor { + internalContext, cancelFunc := context.WithCancel(context.Background()) + + return &nodeInfoWithPodTemplateProcessor{ + ctx: internalContext, + cancelFunc: cancelFunc, + podTemplateLister: newPodTemplateLister(kubeClient, internalContext.Done()), + } +} + +// nodeInfoWithPodTemplateProcessor add possible PodTemplates in nodeInfos. +type nodeInfoWithPodTemplateProcessor struct { + podTemplateLister v1lister.PodTemplateLister + + ctx context.Context + cancelFunc func() +} + +// Process returns unchanged nodeInfos. +func (p *nodeInfoWithPodTemplateProcessor) Process(ctx *ca_context.AutoscalingContext, nodeInfosForNodeGroups map[string]*schedulerframework.NodeInfo) (map[string]*schedulerframework.NodeInfo, error) { + // here we can use empty snapshot + clusterSnapshot := simulator.NewBasicClusterSnapshot() + + // retrieve only once the podTemplates list. + // This list will be used for each NodeGroup. + podTemplates, err := p.podTemplateLister.List(labels.Everything()) + if err != nil { + return nil, errors.ToAutoscalerError(errors.InternalError, err) + } + + result := make(map[string]*schedulerframework.NodeInfo, len(nodeInfosForNodeGroups)) + var errs []error + for id, nodeInfo := range nodeInfosForNodeGroups { + newNodeInfo, err := getNodeInfoWithPodTemplates(nodeInfo, podTemplates, clusterSnapshot, ctx.PredicateChecker) + if err != nil { + errs = append(errs, err) + } + result[id] = newNodeInfo + } + + return result, utilerrors.NewAggregate(errs) +} + +// CleanUp cleans up processor's internal structuxres. +func (p *nodeInfoWithPodTemplateProcessor) CleanUp() { + p.cancelFunc() +} + +func newPodTemplateLister(kubeClient client.Interface, stopchannel <-chan struct{}) v1lister.PodTemplateLister { + podTemplateWatchOption := func(options *metav1.ListOptions) { + options.FieldSelector = fields.Everything().String() + options.LabelSelector = labels.SelectorFromSet(getDaemonsetPodTemplateLabelSet()).String() + } + listWatcher := cache.NewFilteredListWatchFromClient(kubeClient.CoreV1().RESTClient(), "podtemplates", apiv1.NamespaceAll, podTemplateWatchOption) + store, reflector := cache.NewNamespaceKeyedIndexerAndReflector(listWatcher, &apiv1.PodTemplate{}, time.Hour) + lister := v1lister.NewPodTemplateLister(store) + go reflector.Run(stopchannel) + return lister +} + +func getNodeInfoWithPodTemplates(baseNodeInfo *schedulerframework.NodeInfo, podTemplates []*apiv1.PodTemplate, clusterSnapshot *simulator.BasicClusterSnapshot, predicateChecker simulator.PredicateChecker) (*schedulerframework.NodeInfo, error) { + node := baseNodeInfo.Node() + var pods []*apiv1.Pod + + for _, podInfo := range baseNodeInfo.Pods { + pods = append(pods, podInfo.Pod) + } + if err := clusterSnapshot.AddNodeWithPods(node, pods); err != nil { + return nil, err + } + // clone baseNodeInfo to not modify the input object. + newNodeInfo := baseNodeInfo.Clone() + for _, podTpl := range podTemplates { + newPod := newPod(podTpl, node.Name) + err := predicateChecker.CheckPredicates(clusterSnapshot, newPod, node.Name) + if err == nil { + newNodeInfo.AddPod(newPod) + } else if err.ErrorType() == simulator.NotSchedulablePredicateError { + // ok; we are just skipping this daemonset + } else { + // unexpected error + return nil, fmt.Errorf("unexpected error while calling PredicateChecker; %v", err) + } + } + + return newNodeInfo, nil +} + +func getDaemonsetPodTemplateLabelSet() labels.Set { + daemonsetPodTemplateLabels := map[string]string{ + PodTemplateDaemonSetLabelKey: PodTemplateDaemonSetLabelValueTrue, + } + return labels.Set(daemonsetPodTemplateLabels) +} + +func newPod(pt *apiv1.PodTemplate, nodeName string) *apiv1.Pod { + newPod := &apiv1.Pod{Spec: pt.Template.Spec, ObjectMeta: pt.Template.ObjectMeta} + newPod.Namespace = pt.Namespace + newPod.Name = fmt.Sprintf("%s-pod", pt.Name) + newPod.Spec.NodeName = nodeName + return newPod +} diff --git a/cluster-autoscaler/processors/nodeinfos/podtemplates/node_info_with_podtemplates_processors_test.go b/cluster-autoscaler/processors/nodeinfos/podtemplates/node_info_with_podtemplates_processors_test.go new file mode 100644 index 00000000000..3bfbe45f86b --- /dev/null +++ b/cluster-autoscaler/processors/nodeinfos/podtemplates/node_info_with_podtemplates_processors_test.go @@ -0,0 +1,302 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podtemplates + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + apiv1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + + "k8s.io/client-go/kubernetes/fake" + v1lister "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" + + ca_context "k8s.io/autoscaler/cluster-autoscaler/context" + "k8s.io/autoscaler/cluster-autoscaler/simulator" +) + +func TestNewNodeInfoWithPodTemplateProcessor(t *testing.T) { + client := fake.NewSimpleClientset() + processor := NewNodeInfoWithPodTemplateProcessor(client) + defer processor.CleanUp() +} + +func Test_newPodTemplateLister(t *testing.T) { + client := fake.NewSimpleClientset() + ctx, cancelFunc := context.WithCancel(context.Background()) + _ = newPodTemplateLister(client, ctx.Done()) + defer cancelFunc() +} +func Test_getNodeInfoWithPodTemplates(t *testing.T) { + nodeName1 := "node-1" + nodePod1 := newTestPod("bar", "foo", &apiv1.PodSpec{}, nodeName1) + nodeInfo := newNodeInfo(nodeName1, 42, nodePod1) + nodeInfoUnschedulable := newNodeInfo(nodeName1, 0, nodePod1) + + type args struct { + baseNodeInfo *schedulerframework.NodeInfo + podTemplates []*apiv1.PodTemplate + } + tests := []struct { + name string + args args + wantFunc func() *schedulerframework.NodeInfo + wantErr bool + }{ + { + name: "nodeInfo should not be updated", + args: args{ + baseNodeInfo: nodeInfo.Clone(), + podTemplates: nil, + }, + wantFunc: func() *schedulerframework.NodeInfo { + return nodeInfo.Clone() + }, + }, + { + name: "nodeInfo contains one additional Pod", + args: args{ + baseNodeInfo: nodeInfo.Clone(), + podTemplates: []*apiv1.PodTemplate{ + newPodTemplate("extra-ns", "extra-name", nil), + }, + }, + wantFunc: func() *schedulerframework.NodeInfo { + nodeInfo := nodeInfo.Clone() + nodeInfo.AddPod(newTestPod("extra-ns", "extra-name-pod", nil, nodeName1)) + return nodeInfo + }, + }, + { + name: "nodeInfo unschedulable", + args: args{ + baseNodeInfo: nodeInfoUnschedulable.Clone(), + podTemplates: []*apiv1.PodTemplate{ + newPodTemplate("extra-ns", "extra-name", nil), + }, + }, + wantFunc: func() *schedulerframework.NodeInfo { + return nodeInfoUnschedulable.Clone() + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clusterSnapshot := simulator.NewBasicClusterSnapshot() + predicateChecker, err := simulator.NewTestPredicateChecker() + got, err := getNodeInfoWithPodTemplates(tt.args.baseNodeInfo, tt.args.podTemplates, clusterSnapshot, predicateChecker) + if (err != nil) != tt.wantErr { + t.Errorf("getNodeInfoWithPodTemplates() error = %v, wantErr %v", err, tt.wantErr) + return + } + want := tt.wantFunc() + got.Generation = want.Generation + assert.EqualValues(t, want, got, "getNodeInfoWithPodTemplates wrong expected value") + }) + } +} + +func Test_nodeInfoWithPodTemplateProcessor_Process(t *testing.T) { + namespace := "bar" + podName := "foo-dfsdfds" + nodeName1 := "node-1" + + tests := []struct { + name string + podTemplates []*apiv1.PodTemplate + podListerCreation func(pts []*apiv1.PodTemplate) (v1lister.PodTemplateLister, error) + nodeInfosForNodeGroups map[string]*schedulerframework.NodeInfo + want map[string]*schedulerframework.NodeInfo + wantErr bool + }{ + { + name: "1 pod added", + podTemplates: []*apiv1.PodTemplate{newPodTemplate(namespace, podName, nil)}, + podListerCreation: newTestDaemonSetLister, + nodeInfosForNodeGroups: map[string]*schedulerframework.NodeInfo{ + nodeName1: newNodeInfo(nodeName1, 42), + }, + want: map[string]*schedulerframework.NodeInfo{ + nodeName1: newNodeInfo(nodeName1, 42, newTestPod(namespace, fmt.Sprintf("%s-pod", podName), nil, nodeName1)), + }, + wantErr: false, + }, + { + name: "0 pod added: unschedulable node", + podTemplates: []*apiv1.PodTemplate{newPodTemplate(namespace, podName, nil)}, + podListerCreation: newTestDaemonSetLister, + nodeInfosForNodeGroups: map[string]*schedulerframework.NodeInfo{ + nodeName1: newNodeInfo(nodeName1, 0), + }, + want: map[string]*schedulerframework.NodeInfo{ + nodeName1: newNodeInfo(nodeName1, 0), + }, + wantErr: false, + }, + { + name: "pod lister error", + podTemplates: []*apiv1.PodTemplate{newPodTemplate(namespace, podName, nil)}, + podListerCreation: func(pts []*apiv1.PodTemplate) (v1lister.PodTemplateLister, error) { + podLister := &podTemplateListerMock{} + podLister.On("List").Return(pts, fmt.Errorf("unable to list")) + return podLister, nil + }, + nodeInfosForNodeGroups: map[string]*schedulerframework.NodeInfo{ + nodeName1: newNodeInfo(nodeName1, 0), + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + podLister, err := tt.podListerCreation(tt.podTemplates) + assert.NoError(t, err, "err should be nil") + + ctx, cancelFunc := context.WithCancel(context.Background()) + + p := &nodeInfoWithPodTemplateProcessor{ + podTemplateLister: podLister, + ctx: ctx, + cancelFunc: cancelFunc, + } + + ca_ctx, err := newTestClusterAutoscalerContext() + assert.NoError(t, err, "err should be nil") + + got, err := p.Process(ca_ctx, tt.nodeInfosForNodeGroups) + + resetNodeInfoGeneration(got) + resetNodeInfoGeneration(tt.want) + + if (err != nil) != tt.wantErr { + t.Errorf("nodeInfoWithPodTemplateProcessor.Process() error = %v, wantErr %v", err, tt.wantErr) + return + } + assert.EqualValues(t, tt.want, got, "nodeInfoWithPodTemplateProcessor.Process() wrong expected value") + }) + } +} + +func newTestDaemonSetLister(pts []*apiv1.PodTemplate) (v1lister.PodTemplateLister, error) { + store := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for _, pt := range pts { + err := store.Add(pt) + if err != nil { + return nil, fmt.Errorf("Error adding object to cache: %v", err) + } + } + return v1lister.NewPodTemplateLister(store), nil +} + +func newTestClusterAutoscalerContext() (*ca_context.AutoscalingContext, error) { + predicateChecker, err := simulator.NewTestPredicateChecker() + if err != nil { + return nil, err + } + + ctx := &ca_context.AutoscalingContext{ + PredicateChecker: predicateChecker, + } + return ctx, nil +} + +func newNode(name string, maxPods int64) *apiv1.Node { + // define a resource list to allow pod scheduling on this node. + newResourceList := apiv1.ResourceList{ + apiv1.ResourcePods: *resource.NewQuantity(maxPods, resource.DecimalSI), + } + return &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + }, + Status: apiv1.NodeStatus{ + Allocatable: newResourceList, + }, + } +} + +func newNodeInfo(nodeName string, maxPod int64, pods ...*v1.Pod) *schedulerframework.NodeInfo { + node1 := newNode(nodeName, maxPod) + nodeInfo := schedulerframework.NewNodeInfo(pods...) + nodeInfo.SetNode(node1) + + return nodeInfo +} + +func newPodTemplate(namespace, name string, spec *apiv1.PodTemplateSpec) *apiv1.PodTemplate { + pt := &apiv1.PodTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + if spec != nil { + pt.Template = *spec + } + + return pt +} + +func newTestPod(namespace, name string, spec *apiv1.PodSpec, nodeName string) *apiv1.Pod { + newPod := &apiv1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }} + newPod.Namespace = namespace + newPod.Name = name + if spec != nil { + newPod.Spec = *spec + } + newPod.Spec.NodeName = nodeName + return newPod +} + +func resetNodeInfoGeneration(nodeInfos map[string]*schedulerframework.NodeInfo) { + for _, nodeInfo := range nodeInfos { + nodeInfo.Generation = 0 + } +} + +type podTemplateListerMock struct { + mock.Mock +} + +// List lists all PodTemplates in the indexer. +func (p *podTemplateListerMock) List(selector labels.Selector) (ret []*apiv1.PodTemplate, err error) { + args := p.Called() + return args.Get(0).([]*apiv1.PodTemplate), args.Error(1) +} + +// PodTemplates returns an object that can list and get PodTemplates. +func (p *podTemplateListerMock) PodTemplates(namespace string) v1lister.PodTemplateNamespaceLister { + args := p.Called(namespace) + return args.Get(0).(v1lister.PodTemplateNamespaceLister) +} diff --git a/cluster-autoscaler/processors/podtemplates/podtemplate_list_processor.go b/cluster-autoscaler/processors/podtemplates/podtemplate_list_processor.go deleted file mode 100644 index b5c16b8022e..00000000000 --- a/cluster-autoscaler/processors/podtemplates/podtemplate_list_processor.go +++ /dev/null @@ -1,142 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package podtemplates contains PodTemplate processor used to simulate -// DaemonSet resources from a PodTemplate. -package podtemplates - -import ( - "context" - "fmt" - "time" - - appsv1 "k8s.io/api/apps/v1" - apiv1 "k8s.io/api/core/v1" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/apimachinery/pkg/labels" - ca_context "k8s.io/autoscaler/cluster-autoscaler/context" - client "k8s.io/client-go/kubernetes" - v1lister "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/tools/cache" -) - -const ( - // PodTemplateDaemonSetLabelKey use as label key on PodTemplate corresponding to an extra Daemonset. - PodTemplateDaemonSetLabelKey = "cluster-autoscaler.kubernetes.io/daemonset-pod" - // PodTemplateDaemonSetLabelValueTrue use as PodTemplateDaemonSetLabelKey label value. - PodTemplateDaemonSetLabelValueTrue = "true" -) - -// PodTemplateListProcessor processes lists of unschedulable pods. -type PodTemplateListProcessor interface { - ExtraDaemonsets(context *ca_context.AutoscalingContext) ([]*appsv1.DaemonSet, error) - CleanUp() -} - -// activePodTemplateListProcessor returning pod list generated from PodTemplate instance. -type activePodTemplateListProcessor struct { - podTemplateLister v1lister.PodTemplateLister - store cache.Store - reflector *cache.Reflector - - ctx context.Context - cancelFunc func() -} - -// NewActivePodTemplateListProcessor creates an instance of PodTemplateListProcessor. -func NewActivePodTemplateListProcessor(kubeClient client.Interface) PodTemplateListProcessor { - internalContext, cancelFunc := context.WithCancel(context.Background()) - - return &activePodTemplateListProcessor{ - ctx: internalContext, - cancelFunc: cancelFunc, - podTemplateLister: newPodTemplateLister(kubeClient, internalContext.Done()), - } -} - -// Process processes lists of extra Daemonset before simulating Daemonset pod node usage. -func (p *activePodTemplateListProcessor) ExtraDaemonsets( - context *ca_context.AutoscalingContext) ([]*appsv1.DaemonSet, error) { - // Get all PodTpls since we already filter pods in the listner - podTpls, err := p.podTemplateLister.List(labels.Everything()) - if err != nil { - return nil, err - } - - result := make([]*appsv1.DaemonSet, 0, len(podTpls)) - for _, podTpl := range podTpls { - result = append(result, newDaemonSet(podTpl)) - } - - return result, nil -} - -// CleanUp cleans up the processor's internal structures. -func (p *activePodTemplateListProcessor) CleanUp() { - p.cancelFunc() -} - -// -// noOpPodTemplateListProcessor is returning pod lists without processing them. -type noOpPodTemplateListProcessor struct{} - -// NewDefaultPodTemplateListProcessor creates an instance of PodTemplateListProcessor. -func NewDefaultPodTemplateListProcessor() PodTemplateListProcessor { - return &noOpPodTemplateListProcessor{} -} - -// Process processes lists of extra Daemonset before simulating Daemonset pod node usage. -func (p *noOpPodTemplateListProcessor) ExtraDaemonsets(context *ca_context.AutoscalingContext) ([]*appsv1.DaemonSet, error) { - return nil, nil -} - -// CleanUp cleans up the processor's internal structures. -func (p *noOpPodTemplateListProcessor) CleanUp() { -} - -// newDaemonSetLister builds a podTemplate lister. -func newPodTemplateLister(kubeClient client.Interface, stopchannel <-chan struct{}) v1lister.PodTemplateLister { - podTemplateWatchOption := func(options *metav1.ListOptions) { - options.FieldSelector = fields.Everything().String() - options.LabelSelector = labels.SelectorFromSet(getDaemonsetPodTemplateLabelSet()).String() - } - listWatcher := cache.NewFilteredListWatchFromClient(kubeClient.CoreV1().RESTClient(), "podtemplates", apiv1.NamespaceAll, podTemplateWatchOption) - store, reflector := cache.NewNamespaceKeyedIndexerAndReflector(listWatcher, &apiv1.PodTemplate{}, time.Hour) - lister := v1lister.NewPodTemplateLister(store) - go reflector.Run(stopchannel) - return lister -} - -func newDaemonSet(podTemplate *apiv1.PodTemplate) *appsv1.DaemonSet { - return &appsv1.DaemonSet{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-tpl", podTemplate.Name), - Namespace: podTemplate.Namespace, - }, - Spec: appsv1.DaemonSetSpec{ - Template: podTemplate.Template, - }, - } -} - -func getDaemonsetPodTemplateLabelSet() labels.Set { - daemonsetPodTemplateLabels := map[string]string{ - PodTemplateDaemonSetLabelKey: PodTemplateDaemonSetLabelValueTrue, - } - return labels.Set(daemonsetPodTemplateLabels) -} diff --git a/cluster-autoscaler/processors/podtemplates/podtemplate_list_processor_test.go b/cluster-autoscaler/processors/podtemplates/podtemplate_list_processor_test.go deleted file mode 100644 index 0972e3b884b..00000000000 --- a/cluster-autoscaler/processors/podtemplates/podtemplate_list_processor_test.go +++ /dev/null @@ -1,186 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package podtemplates - -import ( - "context" - "fmt" - "reflect" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - - appsv1 "k8s.io/api/apps/v1" - apiv1 "k8s.io/api/core/v1" - - "k8s.io/client-go/kubernetes/fake" - v1lister "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/tools/cache" - - ca_context "k8s.io/autoscaler/cluster-autoscaler/context" -) - -func TestNewActivePodTemplateListProcessor(t *testing.T) { - client := fake.NewSimpleClientset() - processor := NewActivePodTemplateListProcessor(client) - defer processor.CleanUp() -} - -func Test_activePodTemplateListProcessor_ExtraDaemonsets(t *testing.T) { - podTplMatch := &apiv1.PodTemplate{ - ObjectMeta: metav1.ObjectMeta{ - Name: "foo", - Namespace: "bar", - Labels: map[string]string{PodTemplateDaemonSetLabelKey: PodTemplateDaemonSetLabelValueTrue}, - }, - } - - daemonSetWant := newDaemonSet(podTplMatch) - - type args struct { - context *ca_context.AutoscalingContext - } - tests := []struct { - name string - initListerFunc func() (v1lister.PodTemplateLister, error) - args args - - want []*appsv1.DaemonSet - wantErr bool - }{ - { - name: "no PodTemplate, should return 0 Daemonset", - args: args{ - context: &ca_context.AutoscalingContext{}, - }, - initListerFunc: func() (v1lister.PodTemplateLister, error) { - return newTestDaemonSetLister(nil) - }, - want: []*appsv1.DaemonSet{}, - wantErr: false, - }, - { - name: "1 matching PodTemplate, should return 1 Daemonset", - args: args{ - context: &ca_context.AutoscalingContext{}, - }, - initListerFunc: func() (v1lister.PodTemplateLister, error) { - return newTestDaemonSetLister([]*apiv1.PodTemplate{podTplMatch}) - }, - want: []*appsv1.DaemonSet{daemonSetWant}, - wantErr: false, - }, - { - name: "Lister returns an error", - args: args{ - context: &ca_context.AutoscalingContext{}, - }, - initListerFunc: func() (v1lister.PodTemplateLister, error) { - podLister := &podTemplateListerMock{} - podLister.On("List").Return([]*apiv1.PodTemplate{}, fmt.Errorf("unable to list")) - return podLister, nil - }, - want: nil, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx, cancelFunc := context.WithCancel(context.Background()) - lister, err := tt.initListerFunc() - assert.Nil(t, err, "lister creation should not return an error") - p := activePodTemplateListProcessor{ - podTemplateLister: lister, - ctx: ctx, - cancelFunc: cancelFunc, - } - defer p.CleanUp() - got, err := p.ExtraDaemonsets(tt.args.context) - if (err != nil) != tt.wantErr { - t.Errorf("activePodTemplateListProcessor.ExtraDaemonsets() error = %v, wantErr %v", err, tt.wantErr) - return - } - assert.Equal(t, tt.want, got, "they should be equal") - t.Logf("got: %v", got) - }) - } -} - -func Test_noOpPodTemplateListProcessor_ExtraDaemonsets(t *testing.T) { - type args struct { - context *ca_context.AutoscalingContext - } - tests := []struct { - name string - args args - want []*appsv1.DaemonSet - wantErr bool - }{ - { - name: "always return empty list with no error", - args: args{ - context: &ca_context.AutoscalingContext{}, - }, - want: nil, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - p := NewDefaultPodTemplateListProcessor() - got, err := p.ExtraDaemonsets(tt.args.context) - if (err != nil) != tt.wantErr { - t.Errorf("noOpPodTemplateListProcessor.ExtraDaemonsets() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("noOpPodTemplateListProcessor.ExtraDaemonsets() = %v, want %v", got, tt.want) - } - }) - } -} - -func newTestDaemonSetLister(pts []*apiv1.PodTemplate) (v1lister.PodTemplateLister, error) { - store := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) - for _, pt := range pts { - err := store.Add(pt) - if err != nil { - return nil, fmt.Errorf("Error adding object to cache: %v", err) - } - } - return v1lister.NewPodTemplateLister(store), nil -} - -type podTemplateListerMock struct { - mock.Mock -} - -// List lists all PodTemplates in the indexer. -func (p *podTemplateListerMock) List(selector labels.Selector) (ret []*apiv1.PodTemplate, err error) { - args := p.Called() - return args.Get(0).([]*apiv1.PodTemplate), args.Error(1) -} - -// PodTemplates returns an object that can list and get PodTemplates. -func (p *podTemplateListerMock) PodTemplates(namespace string) v1lister.PodTemplateNamespaceLister { - args := p.Called(namespace) - return args.Get(0).(v1lister.PodTemplateNamespaceLister) -} diff --git a/cluster-autoscaler/processors/processors.go b/cluster-autoscaler/processors/processors.go index 618bf47b585..dd3f9a54bfa 100644 --- a/cluster-autoscaler/processors/processors.go +++ b/cluster-autoscaler/processors/processors.go @@ -23,7 +23,6 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/processors/nodeinfos" "k8s.io/autoscaler/cluster-autoscaler/processors/nodes" "k8s.io/autoscaler/cluster-autoscaler/processors/pods" - "k8s.io/autoscaler/cluster-autoscaler/processors/podtemplates" "k8s.io/autoscaler/cluster-autoscaler/processors/status" ) @@ -50,8 +49,6 @@ type AutoscalingProcessors struct { NodeInfoProcessor nodeinfos.NodeInfoProcessor // NodeGroupConfigProcessor provides config option for each NodeGroup. NodeGroupConfigProcessor nodegroupconfig.NodeGroupConfigProcessor - // PodTemplateListProcessor is used to provide a list of Pods considered as Daemonset pods. - PodTemplateListProcessor podtemplates.PodTemplateListProcessor } // DefaultProcessors returns default set of processors. @@ -67,7 +64,6 @@ func DefaultProcessors() *AutoscalingProcessors { NodeGroupManager: nodegroups.NewDefaultNodeGroupManager(), NodeInfoProcessor: nodeinfos.NewDefaultNodeInfoProcessor(), NodeGroupConfigProcessor: nodegroupconfig.NewDefaultNodeGroupConfigProcessor(), - PodTemplateListProcessor: podtemplates.NewDefaultPodTemplateListProcessor(), } } @@ -83,5 +79,4 @@ func (ap *AutoscalingProcessors) CleanUp() { ap.ScaleDownNodeProcessor.CleanUp() ap.NodeInfoProcessor.CleanUp() ap.NodeGroupConfigProcessor.CleanUp() - ap.PodTemplateListProcessor.CleanUp() }