diff --git a/charts/cluster-autoscaler/Chart.yaml b/charts/cluster-autoscaler/Chart.yaml index 596b9290cce0..40957eb849b9 100644 --- a/charts/cluster-autoscaler/Chart.yaml +++ b/charts/cluster-autoscaler/Chart.yaml @@ -17,4 +17,4 @@ name: cluster-autoscaler sources: - https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler type: application -version: 9.9.2 +version: 9.10.0 diff --git a/charts/cluster-autoscaler/templates/clusterrole.yaml b/charts/cluster-autoscaler/templates/clusterrole.yaml index 2ebadc3bb6c9..e2a73ec2a32d 100644 --- a/charts/cluster-autoscaler/templates/clusterrole.yaml +++ b/charts/cluster-autoscaler/templates/clusterrole.yaml @@ -48,6 +48,7 @@ rules: - "" resources: - pods + - podtemplates - services - replicationcontrollers - persistentvolumeclaims diff --git a/cluster-autoscaler/cloudprovider/alicloud/examples/cluster-autoscaler-standard.yaml b/cluster-autoscaler/cloudprovider/alicloud/examples/cluster-autoscaler-standard.yaml index 373b483370e4..e7011c0a2562 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/examples/cluster-autoscaler-standard.yaml +++ b/cluster-autoscaler/cloudprovider/alicloud/examples/cluster-autoscaler-standard.yaml @@ -33,7 +33,7 @@ rules: resources: ["nodes"] verbs: ["watch","list","get","update"] - apiGroups: [""] - resources: ["pods","services","replicationcontrollers","persistentvolumeclaims","persistentvolumes"] + resources: ["pods","podtemplates","services","replicationcontrollers","persistentvolumeclaims","persistentvolumes"] verbs: ["watch","list","get"] - apiGroups: ["extensions"] resources: ["replicasets","daemonsets"] diff --git a/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml b/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml index 7527653fcc29..4871c24b4c95 100644 --- a/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml +++ b/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-multi-asg.yaml b/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-multi-asg.yaml index 42e485950dc9..04566d01c295 100644 --- a/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-multi-asg.yaml +++ b/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-multi-asg.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-one-asg.yaml b/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-one-asg.yaml index 2da3b9e2bbe0..c337c0938d25 100644 --- a/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-one-asg.yaml +++ b/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-one-asg.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-run-on-control-plane.yaml b/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-run-on-control-plane.yaml index c200ec661e02..982dc274e570 100644 --- a/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-run-on-control-plane.yaml +++ b/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-run-on-control-plane.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-aks.yaml b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-aks.yaml index 4e1f1f17f3a6..c48581cb5a5e 100644 --- a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-aks.yaml +++ b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-aks.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-autodiscover.yaml b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-autodiscover.yaml index 5a680c238a8c..f1359511c15b 100644 --- a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-autodiscover.yaml +++ b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-autodiscover.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard-control-plane.yaml b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard-control-plane.yaml index f7738f1fd714..f12bf9fa0064 100644 --- a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard-control-plane.yaml +++ b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard-control-plane.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard-msi.yaml b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard-msi.yaml index 2dedce4dcf57..66e5fe766701 100644 --- a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard-msi.yaml +++ b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard-msi.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard.yaml b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard.yaml index b7efff22cb2b..77169cbfe228 100644 --- a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard.yaml +++ b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-standard.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss-control-plane.yaml b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss-control-plane.yaml index 0a343395a019..8085bd06e305 100644 --- a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss-control-plane.yaml +++ b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss-control-plane.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss-msi.yaml b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss-msi.yaml index bba365627856..60570da5a435 100644 --- a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss-msi.yaml +++ b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss-msi.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss.yaml b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss.yaml index 5a45508c964c..c6f3cbe1be36 100644 --- a/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss.yaml +++ b/cluster-autoscaler/cloudprovider/azure/examples/cluster-autoscaler-vmss.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/baiducloud/examples/cluster-autoscaler-multiple-asgs.yaml b/cluster-autoscaler/cloudprovider/baiducloud/examples/cluster-autoscaler-multiple-asgs.yaml index aab56c5ec158..92d57ad766bd 100644 --- a/cluster-autoscaler/cloudprovider/baiducloud/examples/cluster-autoscaler-multiple-asgs.yaml +++ b/cluster-autoscaler/cloudprovider/baiducloud/examples/cluster-autoscaler-multiple-asgs.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/baiducloud/examples/cluster-autoscaler-one-asg.yaml b/cluster-autoscaler/cloudprovider/baiducloud/examples/cluster-autoscaler-one-asg.yaml index 346ce05ccd45..31a3c5b5055c 100644 --- a/cluster-autoscaler/cloudprovider/baiducloud/examples/cluster-autoscaler-one-asg.yaml +++ b/cluster-autoscaler/cloudprovider/baiducloud/examples/cluster-autoscaler-one-asg.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/bizflycloud/manifest/rbac.yaml b/cluster-autoscaler/cloudprovider/bizflycloud/manifest/rbac.yaml index 34159c181fad..4ea931ea2b1e 100644 --- a/cluster-autoscaler/cloudprovider/bizflycloud/manifest/rbac.yaml +++ b/cluster-autoscaler/cloudprovider/bizflycloud/manifest/rbac.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/cloudstack/examples/cluster-autoscaler-standard.yaml b/cluster-autoscaler/cloudprovider/cloudstack/examples/cluster-autoscaler-standard.yaml index 235fe4dda790..000376a12ce6 100644 --- a/cluster-autoscaler/cloudprovider/cloudstack/examples/cluster-autoscaler-standard.yaml +++ b/cluster-autoscaler/cloudprovider/cloudstack/examples/cluster-autoscaler-standard.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/clusterapi/examples/deployment.yaml b/cluster-autoscaler/cloudprovider/clusterapi/examples/deployment.yaml index bdd92151b761..a9af73f387c0 100644 --- a/cluster-autoscaler/cloudprovider/clusterapi/examples/deployment.yaml +++ b/cluster-autoscaler/cloudprovider/clusterapi/examples/deployment.yaml @@ -72,6 +72,7 @@ rules: - persistentvolumeclaims - persistentvolumes - pods + - podtemplates - replicationcontrollers - services verbs: diff --git a/cluster-autoscaler/cloudprovider/exoscale/examples/cluster-autoscaler-run-on-control-plane.yaml b/cluster-autoscaler/cloudprovider/exoscale/examples/cluster-autoscaler-run-on-control-plane.yaml index 41abc7efbedf..17228d6563ee 100644 --- a/cluster-autoscaler/cloudprovider/exoscale/examples/cluster-autoscaler-run-on-control-plane.yaml +++ b/cluster-autoscaler/cloudprovider/exoscale/examples/cluster-autoscaler-run-on-control-plane.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml b/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml index 40c50e21cd57..9b802b4f3f25 100644 --- a/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml +++ b/cluster-autoscaler/cloudprovider/hetzner/examples/cluster-autoscaler-run-on-master.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/huaweicloud/examples/cluster-autoscaler-svcaccount.yaml b/cluster-autoscaler/cloudprovider/huaweicloud/examples/cluster-autoscaler-svcaccount.yaml index 1bcfe443ed4a..766e1cd6a27c 100644 --- a/cluster-autoscaler/cloudprovider/huaweicloud/examples/cluster-autoscaler-svcaccount.yaml +++ b/cluster-autoscaler/cloudprovider/huaweicloud/examples/cluster-autoscaler-svcaccount.yaml @@ -23,6 +23,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/ionoscloud/examples/cluster-autoscaler-standard.yaml b/cluster-autoscaler/cloudprovider/ionoscloud/examples/cluster-autoscaler-standard.yaml index 1abad729ecba..a07087dca39f 100644 --- a/cluster-autoscaler/cloudprovider/ionoscloud/examples/cluster-autoscaler-standard.yaml +++ b/cluster-autoscaler/cloudprovider/ionoscloud/examples/cluster-autoscaler-standard.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscaler-autodiscover.yaml b/cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscaler-autodiscover.yaml index 0d42426c1887..2cacad6c551e 100644 --- a/cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscaler-autodiscover.yaml +++ b/cluster-autoscaler/cloudprovider/linode/examples/cluster-autoscaler-autodiscover.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/magnum/examples/cluster-autoscaler-svcaccount.yaml b/cluster-autoscaler/cloudprovider/magnum/examples/cluster-autoscaler-svcaccount.yaml index 190bf9372ecf..cdd5d0337087 100644 --- a/cluster-autoscaler/cloudprovider/magnum/examples/cluster-autoscaler-svcaccount.yaml +++ b/cluster-autoscaler/cloudprovider/magnum/examples/cluster-autoscaler-svcaccount.yaml @@ -23,6 +23,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/packet/examples/cluster-autoscaler-deployment.yaml b/cluster-autoscaler/cloudprovider/packet/examples/cluster-autoscaler-deployment.yaml index cdf35f2a4040..173ba6fd4a4a 100644 --- a/cluster-autoscaler/cloudprovider/packet/examples/cluster-autoscaler-deployment.yaml +++ b/cluster-autoscaler/cloudprovider/packet/examples/cluster-autoscaler-deployment.yaml @@ -35,6 +35,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/cloudprovider/packet/examples/cluster-autoscaler-svcaccount.yaml b/cluster-autoscaler/cloudprovider/packet/examples/cluster-autoscaler-svcaccount.yaml index bb5672a53a00..e0a0dc8bafa0 100644 --- a/cluster-autoscaler/cloudprovider/packet/examples/cluster-autoscaler-svcaccount.yaml +++ b/cluster-autoscaler/cloudprovider/packet/examples/cluster-autoscaler-svcaccount.yaml @@ -23,6 +23,7 @@ rules: - apiGroups: [""] resources: - "pods" + - "podtemplates" - "services" - "replicationcontrollers" - "persistentvolumeclaims" diff --git a/cluster-autoscaler/config/autoscaling_options.go b/cluster-autoscaler/config/autoscaling_options.go index 4dcd95245f59..10ae5e83c1a4 100644 --- a/cluster-autoscaler/config/autoscaling_options.go +++ b/cluster-autoscaler/config/autoscaling_options.go @@ -165,4 +165,6 @@ type AutoscalingOptions struct { DaemonSetEvictionForOccupiedNodes bool // User agent to use for HTTP calls. UserAgent string + // NodeInfoProcessorPodTemplates Enable or disable the NodeInfoProcessor PodTemplate + NodeInfoProcessorPodTemplates bool } diff --git a/cluster-autoscaler/core/scale_test_common.go b/cluster-autoscaler/core/scale_test_common.go index 5d4832584890..5f3a702edb15 100644 --- a/cluster-autoscaler/core/scale_test_common.go +++ b/cluster-autoscaler/core/scale_test_common.go @@ -260,7 +260,6 @@ type mockAutoprovisioningNodeGroupListProcessor struct { func (p *mockAutoprovisioningNodeGroupListProcessor) Process(context *context.AutoscalingContext, nodeGroups []cloudprovider.NodeGroup, nodeInfos map[string]*schedulerframework.NodeInfo, unschedulablePods []*apiv1.Pod) ([]cloudprovider.NodeGroup, map[string]*schedulerframework.NodeInfo, error) { - machines, err := context.CloudProvider.GetAvailableMachineTypes() assert.NoError(p.t, err) diff --git a/cluster-autoscaler/core/static_autoscaler.go b/cluster-autoscaler/core/static_autoscaler.go index a0f54f960dd8..0db9ff1cf7b3 100644 --- a/cluster-autoscaler/core/static_autoscaler.go +++ b/cluster-autoscaler/core/static_autoscaler.go @@ -120,7 +120,6 @@ func NewStaticAutoscaler( expanderStrategy expander.Strategy, estimatorBuilder estimator.EstimatorBuilder, backoff backoff.Backoff) *StaticAutoscaler { - processorCallbacks := newStaticAutoscalerProcessorCallbacks() autoscalingContext := context.NewAutoscalingContext( opts, @@ -701,7 +700,6 @@ func (a *StaticAutoscaler) filterOutYoungPods(allUnschedulablePods []*apiv1.Pod, oldUnschedulablePods = append(oldUnschedulablePods, pod) } else { klog.V(3).Infof("Pod %s is %.3f seconds old, too new to consider unschedulable", pod.Name, podAge.Seconds()) - } } return oldUnschedulablePods diff --git a/cluster-autoscaler/core/utils/utils.go b/cluster-autoscaler/core/utils/utils.go index fce14d38a0a6..66531d1d3d2a 100644 --- a/cluster-autoscaler/core/utils/utils.go +++ b/cluster-autoscaler/core/utils/utils.go @@ -38,6 +38,15 @@ import ( klog "k8s.io/klog/v2" ) +const ( + // TemplateNodeForNamePrefix template node prefix use in the sanitizeNodeInfo() function. + TemplateNodeForNamePrefix = "template-node-for" + // TemplateNodeForNameFromTemplatePrefix sub-prefix when the template is generated based on a cloud-provider node template. + TemplateNodeForNameFromTemplatePrefix = "template" + // TemplateNodeForNameFromCopyPrefix sub-prefix when the template is generated from a copy of another exiting node. + TemplateNodeForNameFromCopyPrefix = "copy" +) + // GetNodeInfosForGroups finds NodeInfos for all node groups used to manage the given nodes. It also returns a node group to sample node mapping. func GetNodeInfosForGroups(nodes []*apiv1.Node, nodeInfoCache map[string]*schedulerframework.NodeInfo, cloudProvider cloudprovider.CloudProvider, listers kube_util.ListerRegistry, // TODO(mwielgus): This returns map keyed by url, while most code (including scheduler) uses node.Name for a key. @@ -67,7 +76,7 @@ func GetNodeInfosForGroups(nodes []*apiv1.Node, nodeInfoCache map[string]*schedu if err != nil { return false, "", err } - sanitizedNodeInfo, err := sanitizeNodeInfo(nodeInfo, id, ignoredTaints) + sanitizedNodeInfo, err := sanitizeNodeInfo(nodeInfo, TemplateNodeForNameFromCopyPrefix, id, ignoredTaints) if err != nil { return false, "", err } @@ -179,9 +188,10 @@ func GetNodeInfoFromTemplate(nodeGroup cloudprovider.NodeGroup, daemonsets []*ap for _, podInfo := range baseNodeInfo.Pods { pods = append(pods, podInfo.Pod) } + fullNodeInfo := schedulerframework.NewNodeInfo(pods...) fullNodeInfo.SetNode(baseNodeInfo.Node()) - sanitizedNodeInfo, typedErr := sanitizeNodeInfo(fullNodeInfo, id, ignoredTaints) + sanitizedNodeInfo, typedErr := sanitizeNodeInfo(fullNodeInfo, TemplateNodeForNameFromTemplatePrefix, id, ignoredTaints) if typedErr != nil { return nil, typedErr } @@ -226,9 +236,9 @@ func deepCopyNodeInfo(nodeInfo *schedulerframework.NodeInfo) (*schedulerframewor return newNodeInfo, nil } -func sanitizeNodeInfo(nodeInfo *schedulerframework.NodeInfo, nodeGroupName string, ignoredTaints taints.TaintKeySet) (*schedulerframework.NodeInfo, errors.AutoscalerError) { +func sanitizeNodeInfo(nodeInfo *schedulerframework.NodeInfo, nodeSource string, nodeGroupName string, ignoredTaints taints.TaintKeySet) (*schedulerframework.NodeInfo, errors.AutoscalerError) { // Sanitize node name. - sanitizedNode, err := sanitizeTemplateNode(nodeInfo.Node(), nodeGroupName, ignoredTaints) + sanitizedNode, err := sanitizeTemplateNode(nodeInfo.Node(), nodeSource, nodeGroupName, ignoredTaints) if err != nil { return nil, err } @@ -247,9 +257,9 @@ func sanitizeNodeInfo(nodeInfo *schedulerframework.NodeInfo, nodeGroupName strin return sanitizedNodeInfo, nil } -func sanitizeTemplateNode(node *apiv1.Node, nodeGroup string, ignoredTaints taints.TaintKeySet) (*apiv1.Node, errors.AutoscalerError) { +func sanitizeTemplateNode(node *apiv1.Node, nodeSource string, nodeGroup string, ignoredTaints taints.TaintKeySet) (*apiv1.Node, errors.AutoscalerError) { newNode := node.DeepCopy() - nodeName := fmt.Sprintf("template-node-for-%s-%d", nodeGroup, rand.Int63()) + nodeName := fmt.Sprintf("%s-%s-%s-%d", TemplateNodeForNamePrefix, nodeSource, nodeGroup, rand.Int63()) newNode.Labels = make(map[string]string, len(node.Labels)) for k, v := range node.Labels { if k != apiv1.LabelHostname { diff --git a/cluster-autoscaler/core/utils/utils_test.go b/cluster-autoscaler/core/utils/utils_test.go index dfb7eb49f963..ede6f9048f7f 100644 --- a/cluster-autoscaler/core/utils/utils_test.go +++ b/cluster-autoscaler/core/utils/utils_test.go @@ -227,7 +227,7 @@ func TestSanitizeNodeInfo(t *testing.T) { nodeInfo := schedulerframework.NewNodeInfo(pod) nodeInfo.SetNode(node) - res, err := sanitizeNodeInfo(nodeInfo, "test-group", nil) + res, err := sanitizeNodeInfo(nodeInfo, "template", "test-group", nil) assert.NoError(t, err) assert.Equal(t, 1, len(res.Pods)) } @@ -238,7 +238,7 @@ func TestSanitizeLabels(t *testing.T) { apiv1.LabelHostname: "abc", "x": "y", } - node, err := sanitizeTemplateNode(oldNode, "bzium", nil) + node, err := sanitizeTemplateNode(oldNode, "copy", "bzium", nil) assert.NoError(t, err) assert.NotEqual(t, node.Labels[apiv1.LabelHostname], "abc", nil) assert.Equal(t, node.Labels["x"], "y") @@ -277,7 +277,6 @@ func TestGetNodeResource(t *testing.T) { memory = getNodeResource(nodeWithNegativeCapacity, apiv1.ResourceMemory) assert.Equal(t, int64(0), memory) - } func TestGetNodeCoresAndMemory(t *testing.T) { diff --git a/cluster-autoscaler/main.go b/cluster-autoscaler/main.go index 37459a175bf1..74d184245c52 100644 --- a/cluster-autoscaler/main.go +++ b/cluster-autoscaler/main.go @@ -43,6 +43,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/metrics" ca_processors "k8s.io/autoscaler/cluster-autoscaler/processors" "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset" + "k8s.io/autoscaler/cluster-autoscaler/processors/nodeinfos/podtemplates" "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" @@ -180,6 +181,7 @@ var ( daemonSetEvictionForEmptyNodes = flag.Bool("daemonset-eviction-for-empty-nodes", false, "DaemonSet pods will be gracefully terminated from empty nodes") daemonSetEvictionForOccupiedNodes = flag.Bool("daemonset-eviction-for-occupied-nodes", true, "DaemonSet pods will be gracefully terminated from non-empty nodes") userAgent = flag.String("user-agent", "cluster-autoscaler", "User agent used for HTTP calls.") + nodeInfoProcessorPodTemplates = flag.Bool("node-info-processor-podtemplate", false, "Enable PodTemplate NodeInfoProcessor to consider specific PodTemplate as DaemonSet") ) func createAutoscalingOptions() config.AutoscalingOptions { @@ -257,6 +259,7 @@ func createAutoscalingOptions() config.AutoscalingOptions { DaemonSetEvictionForEmptyNodes: *daemonSetEvictionForEmptyNodes, DaemonSetEvictionForOccupiedNodes: *daemonSetEvictionForOccupiedNodes, UserAgent: *userAgent, + NodeInfoProcessorPodTemplates: *nodeInfoProcessorPodTemplates, } } @@ -331,6 +334,10 @@ func buildAutoscaler() (core.Autoscaler, error) { Comparator: nodeInfoComparatorBuilder(autoscalingOptions.BalancingExtraIgnoredLabels), } + if autoscalingOptions.NodeInfoProcessorPodTemplates { + opts.Processors.NodeInfoProcessor = podtemplates.NewNodeInfoWithPodTemplateProcessor(&opts) + } + // These metrics should be published only once. metrics.UpdateNapEnabled(autoscalingOptions.NodeAutoprovisioningEnabled) metrics.UpdateMaxNodesCount(autoscalingOptions.MaxNodesTotal) diff --git a/cluster-autoscaler/processors/nodeinfos/podtemplates/node_info_with_podtemplates_processors.go b/cluster-autoscaler/processors/nodeinfos/podtemplates/node_info_with_podtemplates_processors.go new file mode 100644 index 000000000000..4fcbc07d7907 --- /dev/null +++ b/cluster-autoscaler/processors/nodeinfos/podtemplates/node_info_with_podtemplates_processors.go @@ -0,0 +1,174 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podtemplates + +import ( + "context" + "fmt" + "strings" + "time" + + apiv1 "k8s.io/api/core/v1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + + client "k8s.io/client-go/kubernetes" + v1lister "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + + ca_context "k8s.io/autoscaler/cluster-autoscaler/context" + "k8s.io/autoscaler/cluster-autoscaler/core" + core_utils "k8s.io/autoscaler/cluster-autoscaler/core/utils" + "k8s.io/autoscaler/cluster-autoscaler/processors/nodeinfos" + "k8s.io/autoscaler/cluster-autoscaler/simulator" + "k8s.io/autoscaler/cluster-autoscaler/utils/errors" + scheduler_utils "k8s.io/autoscaler/cluster-autoscaler/utils/scheduler" + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" +) + +const ( + // PodTemplateDaemonSetLabelKey use as label key on PodTemplate corresponding to an extra Daemonset. + PodTemplateDaemonSetLabelKey = "cluster-autoscaler.kubernetes.io/daemonset-pod" + // PodTemplateDaemonSetLabelValueTrue use as PodTemplateDaemonSetLabelKey label value. + PodTemplateDaemonSetLabelValueTrue = "true" +) + +// NewNodeInfoWithPodTemplateProcessor returns a default instance of NodeInfoProcessor. +func NewNodeInfoWithPodTemplateProcessor(opts *core.AutoscalerOptions) nodeinfos.NodeInfoProcessor { + internalContext, cancelFunc := context.WithCancel(context.Background()) + + return &nodeInfoWithPodTemplateProcessor{ + ctx: internalContext, + cancelFunc: cancelFunc, + podTemplateLister: newPodTemplateLister(opts.KubeClient, internalContext.Done()), + } +} + +// nodeInfoWithPodTemplateProcessor add possible PodTemplates in nodeInfos. +type nodeInfoWithPodTemplateProcessor struct { + podTemplateLister v1lister.PodTemplateLister + + ctx context.Context + cancelFunc func() +} + +const templateNodeFromTemplatePrefix = core_utils.TemplateNodeForNamePrefix + "-" + core_utils.TemplateNodeForNameFromTemplatePrefix + +// Process returns unchanged nodeInfos. +func (p *nodeInfoWithPodTemplateProcessor) Process(ctx *ca_context.AutoscalingContext, nodeInfosForNodeGroups map[string]*schedulerframework.NodeInfo) (map[string]*schedulerframework.NodeInfo, error) { + // here we can use empty snapshot, since the NodeInfos that will be updated + // are from CloudProvider NodeTemplates. + clusterSnapshot := simulator.NewBasicClusterSnapshot() + + // retrieve only once the podTemplates list. + // This list will be used for each NodeGroup. + podTemplates, err := p.podTemplateLister.List(labels.Everything()) + if err != nil { + return nil, errors.ToAutoscalerError(errors.InternalError, err) + } + + result := make(map[string]*schedulerframework.NodeInfo, len(nodeInfosForNodeGroups)) + var errs []error + for id, nodeInfo := range nodeInfosForNodeGroups { + var newNodeInfo *schedulerframework.NodeInfo + + // only runs getNodeInfoWithPodTemplates() for NodeInfo created from + // cloudprovider.TemplateNodeInfo. + // If not a NodeTemplate, Pods from PodTemplates should already be present + // in the PodList attached to the Node. + if strings.HasPrefix(nodeInfo.Node().Name, templateNodeFromTemplatePrefix) { + var err error + newNodeInfo, err = getNodeInfoWithPodTemplates(nodeInfo, podTemplates, clusterSnapshot, ctx.PredicateChecker) + if err != nil { + errs = append(errs, err) + } + } else { + newNodeInfo = nodeInfosForNodeGroups[id] + } + + result[id] = newNodeInfo + } + + return result, utilerrors.NewAggregate(errs) +} + +// CleanUp cleans up processor's internal structuxres. +func (p *nodeInfoWithPodTemplateProcessor) CleanUp() { + p.cancelFunc() +} + +func newPodTemplateLister(kubeClient client.Interface, stopchannel <-chan struct{}) v1lister.PodTemplateLister { + podTemplateWatchOption := func(options *metav1.ListOptions) { + options.FieldSelector = fields.Everything().String() + options.LabelSelector = labels.SelectorFromSet(getDaemonsetPodTemplateLabelSet()).String() + } + listWatcher := cache.NewFilteredListWatchFromClient(kubeClient.CoreV1().RESTClient(), "podtemplates", apiv1.NamespaceAll, podTemplateWatchOption) + store, reflector := cache.NewNamespaceKeyedIndexerAndReflector(listWatcher, &apiv1.PodTemplate{}, time.Hour) + lister := v1lister.NewPodTemplateLister(store) + go reflector.Run(stopchannel) + return lister +} + +const nodeInfoDeepCopySuffix = "podtemplate" + +func getNodeInfoWithPodTemplates(baseNodeInfo *schedulerframework.NodeInfo, podTemplates []*apiv1.PodTemplate, clusterSnapshot *simulator.BasicClusterSnapshot, predicateChecker simulator.PredicateChecker) (*schedulerframework.NodeInfo, error) { + // clone baseNodeInfo to not modify the input object. + newNodeInfo := scheduler_utils.DeepCopyTemplateNode(baseNodeInfo, nodeInfoDeepCopySuffix) + node := newNodeInfo.Node() + var pods []*apiv1.Pod + + for _, podInfo := range baseNodeInfo.Pods { + pods = append(pods, podInfo.Pod) + } + + if err := clusterSnapshot.AddNodeWithPods(node, pods); err != nil { + return nil, err + } + + for _, podTpl := range podTemplates { + newPod := newPod(podTpl, node.Name) + err := predicateChecker.CheckPredicates(clusterSnapshot, newPod, node.Name) + if err == nil { + newNodeInfo.AddPod(newPod) + } else if err.ErrorType() == simulator.NotSchedulablePredicateError { + // ok; we are just skipping this daemonset + } else { + // unexpected error + return nil, fmt.Errorf("unexpected error while calling PredicateChecker; %v", err) + } + } + + return newNodeInfo, nil +} + +func getDaemonsetPodTemplateLabelSet() labels.Set { + daemonsetPodTemplateLabels := map[string]string{ + PodTemplateDaemonSetLabelKey: PodTemplateDaemonSetLabelValueTrue, + } + return labels.Set(daemonsetPodTemplateLabels) +} + +func newPod(pt *apiv1.PodTemplate, nodeName string) *apiv1.Pod { + newPod := &apiv1.Pod{Spec: pt.Template.Spec, ObjectMeta: pt.Template.ObjectMeta} + newPod.Namespace = pt.Namespace + newPod.Name = fmt.Sprintf("%s-pod", pt.Name) + newPod.Spec.NodeName = nodeName + return newPod +} diff --git a/cluster-autoscaler/processors/nodeinfos/podtemplates/node_info_with_podtemplates_processors_test.go b/cluster-autoscaler/processors/nodeinfos/podtemplates/node_info_with_podtemplates_processors_test.go new file mode 100644 index 000000000000..79953c7299ef --- /dev/null +++ b/cluster-autoscaler/processors/nodeinfos/podtemplates/node_info_with_podtemplates_processors_test.go @@ -0,0 +1,322 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package podtemplates + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + apiv1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + + v1lister "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" + + ca_context "k8s.io/autoscaler/cluster-autoscaler/context" + "k8s.io/autoscaler/cluster-autoscaler/simulator" + scheduler_utils "k8s.io/autoscaler/cluster-autoscaler/utils/scheduler" +) + +func Test_getNodeInfoWithPodTemplates(t *testing.T) { + nodeName1 := "template-node-template-for-node-1" + nodePod1 := newTestPod("bar", "foo", &apiv1.PodSpec{}, nodeName1) + nodeInfo := newNodeInfo(nodeName1, 42, nodePod1) + nodeInfoUnschedulable := newNodeInfo(nodeName1, 0, nodePod1) + + type args struct { + baseNodeInfo *schedulerframework.NodeInfo + podTemplates []*apiv1.PodTemplate + } + tests := []struct { + name string + args args + wantFunc func() *schedulerframework.NodeInfo + wantErr bool + }{ + { + name: "nodeInfo should not be updated", + args: args{ + baseNodeInfo: nodeInfo.Clone(), + podTemplates: nil, + }, + wantFunc: func() *schedulerframework.NodeInfo { + return scheduler_utils.DeepCopyTemplateNode(nodeInfo, nodeInfoDeepCopySuffix) + }, + }, + { + name: "nodeInfo contains one additional Pod", + args: args{ + baseNodeInfo: nodeInfo.Clone(), + podTemplates: []*apiv1.PodTemplate{ + newPodTemplate("extra-ns", "extra-name", nil), + }, + }, + wantFunc: func() *schedulerframework.NodeInfo { + nodeInfo := scheduler_utils.DeepCopyTemplateNode(nodeInfo, nodeInfoDeepCopySuffix) + nodeInfo.AddPod(newTestPod("extra-ns", "extra-name-pod", nil, nodeName1+"-podtemplate")) + return nodeInfo + }, + }, + { + name: "nodeInfo unschedulable", + args: args{ + baseNodeInfo: nodeInfoUnschedulable.Clone(), + podTemplates: []*apiv1.PodTemplate{ + newPodTemplate("extra-ns", "extra-name", nil), + }, + }, + wantFunc: func() *schedulerframework.NodeInfo { + return scheduler_utils.DeepCopyTemplateNode(nodeInfoUnschedulable, nodeInfoDeepCopySuffix) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clusterSnapshot := simulator.NewBasicClusterSnapshot() + predicateChecker, _ := simulator.NewTestPredicateChecker() + + got, err := getNodeInfoWithPodTemplates(tt.args.baseNodeInfo, tt.args.podTemplates, clusterSnapshot, predicateChecker) + if (err != nil) != tt.wantErr { + t.Errorf("getNodeInfoWithPodTemplates() error = %v, wantErr %v", err, tt.wantErr) + return + } + want := tt.wantFunc() + + got.Generation = want.Generation + + resetNodeInfoGeneratedFields(got) + resetNodeInfoGeneratedFields(want) + assert.EqualValues(t, want, got, "getNodeInfoWithPodTemplates wrong expected value") + }) + } +} + +func Test_nodeInfoWithPodTemplateProcessor_Process(t *testing.T) { + namespace := "bar" + podName := "foo-dfsdfds" + nodeName1 := "template-node-for-template-node-1" + wantNodeName1 := fmt.Sprintf("%s-%s", nodeName1, nodeInfoDeepCopySuffix) + nodeNameCopyNode := "template-node-for-copy-node-0" + + tests := []struct { + name string + podTemplates []*apiv1.PodTemplate + podListerCreation func(pts []*apiv1.PodTemplate) (v1lister.PodTemplateLister, error) + nodeInfosForNodeGroups map[string]*schedulerframework.NodeInfo + want map[string]*schedulerframework.NodeInfo + wantErr bool + }{ + { + name: "1 pod added", + podTemplates: []*apiv1.PodTemplate{newPodTemplate(namespace, podName, nil)}, + podListerCreation: newTestDaemonSetLister, + nodeInfosForNodeGroups: map[string]*schedulerframework.NodeInfo{ + nodeName1: newNodeInfo(nodeName1, 42), + }, + want: map[string]*schedulerframework.NodeInfo{ + nodeName1: newNodeInfo(wantNodeName1, 42, newTestPod(namespace, fmt.Sprintf("%s-pod", podName), nil, wantNodeName1)), + }, + wantErr: false, + }, + { + name: "0 pod added: unschedulable node", + podTemplates: []*apiv1.PodTemplate{newPodTemplate(namespace, podName, nil)}, + podListerCreation: newTestDaemonSetLister, + nodeInfosForNodeGroups: map[string]*schedulerframework.NodeInfo{ + nodeName1: newNodeInfo(nodeName1, 0), + }, + want: map[string]*schedulerframework.NodeInfo{ + nodeName1: newNodeInfo(wantNodeName1, 0), + }, + wantErr: false, + }, + { + name: "0 pod added: real node", + podTemplates: []*apiv1.PodTemplate{newPodTemplate(namespace, podName, nil)}, + podListerCreation: newTestDaemonSetLister, + nodeInfosForNodeGroups: map[string]*schedulerframework.NodeInfo{ + nodeNameCopyNode: newNodeInfo(nodeNameCopyNode, 0), + }, + want: map[string]*schedulerframework.NodeInfo{ + nodeNameCopyNode: newNodeInfo(nodeNameCopyNode, 0), + }, + wantErr: false, + }, + + { + name: "pod lister error", + podTemplates: []*apiv1.PodTemplate{newPodTemplate(namespace, podName, nil)}, + podListerCreation: func(pts []*apiv1.PodTemplate) (v1lister.PodTemplateLister, error) { + podLister := &podTemplateListerMock{} + podLister.On("List").Return(pts, fmt.Errorf("unable to list")) + return podLister, nil + }, + nodeInfosForNodeGroups: map[string]*schedulerframework.NodeInfo{ + nodeName1: newNodeInfo(nodeName1, 0), + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + podLister, err := tt.podListerCreation(tt.podTemplates) + assert.NoError(t, err, "err should be nil") + + ctx, cancelFunc := context.WithCancel(context.Background()) + + p := &nodeInfoWithPodTemplateProcessor{ + podTemplateLister: podLister, + ctx: ctx, + cancelFunc: cancelFunc, + } + + caCtx, err := newTestClusterAutoscalerContext() + assert.NoError(t, err, "err should be nil") + + got, err := p.Process(caCtx, tt.nodeInfosForNodeGroups) + + resetNodeInfosGeneratedFields(got) + resetNodeInfosGeneratedFields(tt.want) + + if (err != nil) != tt.wantErr { + t.Errorf("nodeInfoWithPodTemplateProcessor.Process() error = %v, wantErr %v", err, tt.wantErr) + return + } + assert.EqualValues(t, tt.want, got, "nodeInfoWithPodTemplateProcessor.Process() wrong expected value") + }) + } +} + +func newTestDaemonSetLister(pts []*apiv1.PodTemplate) (v1lister.PodTemplateLister, error) { + store := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for _, pt := range pts { + err := store.Add(pt) + if err != nil { + return nil, fmt.Errorf("Error adding object to cache: %v", err) + } + } + return v1lister.NewPodTemplateLister(store), nil +} + +func newTestClusterAutoscalerContext() (*ca_context.AutoscalingContext, error) { + predicateChecker, err := simulator.NewTestPredicateChecker() + if err != nil { + return nil, err + } + + ctx := &ca_context.AutoscalingContext{ + PredicateChecker: predicateChecker, + } + return ctx, nil +} + +func newNode(name string, maxPods int64) *apiv1.Node { + // define a resource list to allow pod scheduling on this node. + newResourceList := apiv1.ResourceList{ + apiv1.ResourcePods: *resource.NewQuantity(maxPods, resource.DecimalSI), + } + return &apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "kubernetes.io/hostname": name, + }, + }, + Status: apiv1.NodeStatus{ + Allocatable: newResourceList, + }, + } +} + +func newNodeInfo(nodeName string, maxPod int64, pods ...*v1.Pod) *schedulerframework.NodeInfo { + node1 := newNode(nodeName, maxPod) + nodeInfo := schedulerframework.NewNodeInfo(pods...) + nodeInfo.SetNode(node1) + + return nodeInfo +} + +func newPodTemplate(namespace, name string, spec *apiv1.PodTemplateSpec) *apiv1.PodTemplate { + pt := &apiv1.PodTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + if spec != nil { + pt.Template = *spec + } + + return pt +} + +func newTestPod(namespace, name string, spec *apiv1.PodSpec, nodeName string) *apiv1.Pod { + newPod := &apiv1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + newPod.Namespace = namespace + newPod.Name = name + if spec != nil { + newPod.Spec = *spec + } + newPod.Spec.NodeName = nodeName + return newPod +} + +func resetNodeInfosGeneratedFields(nodeInfos map[string]*schedulerframework.NodeInfo) { + for _, nodeInfo := range nodeInfos { + resetNodeInfoGeneratedFields(nodeInfo) + } +} + +func resetNodeInfoGeneratedFields(nodeInfo *schedulerframework.NodeInfo) { + nodeInfo.Generation = 0 + nodeInfo.Node().UID = "" + for _, podInfo := range nodeInfo.Pods { + podInfo.Pod.UID = "" + } +} + +type podTemplateListerMock struct { + mock.Mock +} + +// List lists all PodTemplates in the indexer. +func (p *podTemplateListerMock) List(selector labels.Selector) (ret []*apiv1.PodTemplate, err error) { + args := p.Called() + return args.Get(0).([]*apiv1.PodTemplate), args.Error(1) +} + +// PodTemplates returns an object that can list and get PodTemplates. +func (p *podTemplateListerMock) PodTemplates(namespace string) v1lister.PodTemplateNamespaceLister { + args := p.Called(namespace) + return args.Get(0).(v1lister.PodTemplateNamespaceLister) +}