diff --git a/cluster-autoscaler/core/scaleup/orchestrator/async_initializer.go b/cluster-autoscaler/core/scaleup/orchestrator/async_initializer.go index 588e1cf1dede..22e50e23c36f 100644 --- a/cluster-autoscaler/core/scaleup/orchestrator/async_initializer.go +++ b/cluster-autoscaler/core/scaleup/orchestrator/async_initializer.go @@ -22,10 +22,10 @@ import ( appsv1 "k8s.io/api/apps/v1" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/autoscaler/cluster-autoscaler/context" - "k8s.io/autoscaler/cluster-autoscaler/core/utils" "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroups" "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset" "k8s.io/autoscaler/cluster-autoscaler/processors/status" + "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" "k8s.io/autoscaler/cluster-autoscaler/utils/taints" @@ -75,7 +75,7 @@ func (s *asyncNodeGroupInitializer) InitializeNodeGroup(result nodegroups.AsyncN mainCreatedNodeGroup := result.CreationResult.MainCreatedNodeGroup // If possible replace candidate node-info with node info based on crated node group. The latter // one should be more in line with nodes which will be created by node group. - nodeInfo, aErr := utils.GetNodeInfoFromTemplate(mainCreatedNodeGroup, s.daemonSets, s.taintConfig) + nodeInfo, aErr := simulator.TemplateNodeInfoFromNodeGroupTemplate(mainCreatedNodeGroup, s.daemonSets, s.taintConfig) if aErr != nil { klog.Warningf("Cannot build node info for newly created main node group %s. Using fallback. Error: %v", mainCreatedNodeGroup.Id(), aErr) nodeInfo = s.nodeInfo diff --git a/cluster-autoscaler/core/scaleup/orchestrator/orchestrator.go b/cluster-autoscaler/core/scaleup/orchestrator/orchestrator.go index 8eb316c594b2..80020cf319e5 100644 --- a/cluster-autoscaler/core/scaleup/orchestrator/orchestrator.go +++ b/cluster-autoscaler/core/scaleup/orchestrator/orchestrator.go @@ -27,7 +27,6 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/context" "k8s.io/autoscaler/cluster-autoscaler/core/scaleup/equivalence" "k8s.io/autoscaler/cluster-autoscaler/core/scaleup/resource" - "k8s.io/autoscaler/cluster-autoscaler/core/utils" "k8s.io/autoscaler/cluster-autoscaler/estimator" "k8s.io/autoscaler/cluster-autoscaler/expander" "k8s.io/autoscaler/cluster-autoscaler/metrics" @@ -35,6 +34,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroups" "k8s.io/autoscaler/cluster-autoscaler/processors/nodegroupset" "k8s.io/autoscaler/cluster-autoscaler/processors/status" + "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" "k8s.io/autoscaler/cluster-autoscaler/utils/klogx" @@ -527,7 +527,7 @@ func (o *ScaleUpOrchestrator) CreateNodeGroup( // If possible replace candidate node-info with node info based on crated node group. The latter // one should be more in line with nodes which will be created by node group. - mainCreatedNodeInfo, aErr := utils.GetNodeInfoFromTemplate(createNodeGroupResult.MainCreatedNodeGroup, daemonSets, o.taintConfig) + mainCreatedNodeInfo, aErr := simulator.TemplateNodeInfoFromNodeGroupTemplate(createNodeGroupResult.MainCreatedNodeGroup, daemonSets, o.taintConfig) if aErr == nil { nodeInfos[createNodeGroupResult.MainCreatedNodeGroup.Id()] = mainCreatedNodeInfo schedulablePodGroups[createNodeGroupResult.MainCreatedNodeGroup.Id()] = o.SchedulablePodGroups(podEquivalenceGroups, createNodeGroupResult.MainCreatedNodeGroup, mainCreatedNodeInfo) @@ -542,7 +542,7 @@ func (o *ScaleUpOrchestrator) CreateNodeGroup( delete(schedulablePodGroups, oldId) } for _, nodeGroup := range createNodeGroupResult.ExtraCreatedNodeGroups { - nodeInfo, aErr := utils.GetNodeInfoFromTemplate(nodeGroup, daemonSets, o.taintConfig) + nodeInfo, aErr := simulator.TemplateNodeInfoFromNodeGroupTemplate(nodeGroup, daemonSets, o.taintConfig) if aErr != nil { klog.Warningf("Cannot build node info for newly created extra node group %v; balancing similar node groups will not work; err=%v", nodeGroup.Id(), aErr) continue diff --git a/cluster-autoscaler/core/scaleup/orchestrator/orchestrator_test.go b/cluster-autoscaler/core/scaleup/orchestrator/orchestrator_test.go index cb8dcaefdf30..c9786dfbc49a 100644 --- a/cluster-autoscaler/core/scaleup/orchestrator/orchestrator_test.go +++ b/cluster-autoscaler/core/scaleup/orchestrator/orchestrator_test.go @@ -1045,6 +1045,8 @@ func runSimpleScaleUpTest(t *testing.T, config *ScaleUpTestConfig) *ScaleUpTestR // build orchestrator context, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) assert.NoError(t, err) + err = context.ClusterSnapshot.Initialize(nodes, kube_util.ScheduledPods(pods)) + assert.NoError(t, err) nodeInfos, err := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false). Process(&context, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) assert.NoError(t, err) @@ -1126,13 +1128,15 @@ func TestScaleUpUnhealthy(t *testing.T) { SetNodeReadyState(n1, true, someTimeAgo) n2 := BuildTestNode("n2", 1000, 1000) SetNodeReadyState(n2, true, someTimeAgo) + nodes := []*apiv1.Node{n1, n2} p1 := BuildTestPod("p1", 80, 0) p2 := BuildTestPod("p2", 800, 0) p1.Spec.NodeName = "n1" p2.Spec.NodeName = "n2" + pods := []*apiv1.Pod{p1, p2} - podLister := kube_util.NewTestPodLister([]*apiv1.Pod{p1, p2}) + podLister := kube_util.NewTestPodLister(pods) listers := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) provider := testprovider.NewTestCloudProvider(func(nodeGroup string, increase int) error { @@ -1151,8 +1155,8 @@ func TestScaleUpUnhealthy(t *testing.T) { } context, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) assert.NoError(t, err) - - nodes := []*apiv1.Node{n1, n2} + err = context.ClusterSnapshot.Initialize(nodes, pods) + assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&context, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) clusterState := clusterstate.NewClusterStateRegistry(provider, clusterstate.ClusterStateRegistryConfig{}, context.LogRecorder, NewBackoff(), nodegroupconfig.NewDefaultNodeGroupConfigProcessor(config.NodeGroupAutoscalingOptions{MaxNodeProvisionTime: 15 * time.Minute}), asyncnodegroups.NewDefaultAsyncNodeGroupStateChecker()) clusterState.UpdateNodes(nodes, nodeInfos, time.Now()) @@ -1194,7 +1198,8 @@ func TestBinpackingLimiter(t *testing.T) { context, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) assert.NoError(t, err) - + err = context.ClusterSnapshot.Initialize(nodes, nil) + assert.NoError(t, err) nodeInfos, err := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false). Process(&context, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) assert.NoError(t, err) @@ -1229,11 +1234,13 @@ func TestScaleUpNoHelp(t *testing.T) { n1 := BuildTestNode("n1", 100, 1000) now := time.Now() SetNodeReadyState(n1, true, now.Add(-2*time.Minute)) + nodes := []*apiv1.Node{n1} p1 := BuildTestPod("p1", 80, 0) p1.Spec.NodeName = "n1" + pods := []*apiv1.Pod{p1} - podLister := kube_util.NewTestPodLister([]*apiv1.Pod{p1}) + podLister := kube_util.NewTestPodLister(pods) listers := kube_util.NewListerRegistry(nil, nil, podLister, nil, nil, nil, nil, nil, nil) provider := testprovider.NewTestCloudProvider(func(nodeGroup string, increase int) error { @@ -1251,8 +1258,8 @@ func TestScaleUpNoHelp(t *testing.T) { } context, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) assert.NoError(t, err) - - nodes := []*apiv1.Node{n1} + err = context.ClusterSnapshot.Initialize(nodes, pods) + assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&context, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) clusterState := clusterstate.NewClusterStateRegistry(provider, clusterstate.ClusterStateRegistryConfig{}, context.LogRecorder, NewBackoff(), nodegroupconfig.NewDefaultNodeGroupConfigProcessor(config.NodeGroupAutoscalingOptions{MaxNodeProvisionTime: 15 * time.Minute}), asyncnodegroups.NewDefaultAsyncNodeGroupStateChecker()) clusterState.UpdateNodes(nodes, nodeInfos, time.Now()) @@ -1406,7 +1413,8 @@ func TestComputeSimilarNodeGroups(t *testing.T) { listers := kube_util.NewListerRegistry(nil, nil, kube_util.NewTestPodLister(nil), nil, nil, nil, nil, nil, nil) ctx, err := NewScaleTestAutoscalingContext(config.AutoscalingOptions{BalanceSimilarNodeGroups: tc.balancingEnabled}, &fake.Clientset{}, listers, provider, nil, nil) assert.NoError(t, err) - + err = ctx.ClusterSnapshot.Initialize(nodes, nil) + assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&ctx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) clusterState := clusterstate.NewClusterStateRegistry(provider, clusterstate.ClusterStateRegistryConfig{}, ctx.LogRecorder, NewBackoff(), nodegroupconfig.NewDefaultNodeGroupConfigProcessor(config.NodeGroupAutoscalingOptions{MaxNodeProvisionTime: 15 * time.Minute}), asyncnodegroups.NewDefaultAsyncNodeGroupStateChecker()) assert.NoError(t, clusterState.UpdateNodes(nodes, nodeInfos, time.Now())) @@ -1470,7 +1478,8 @@ func TestScaleUpBalanceGroups(t *testing.T) { } context, err := NewScaleTestAutoscalingContext(options, &fake.Clientset{}, listers, provider, nil, nil) assert.NoError(t, err) - + err = context.ClusterSnapshot.Initialize(nodes, podList) + assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&context, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) clusterState := clusterstate.NewClusterStateRegistry(provider, clusterstate.ClusterStateRegistryConfig{}, context.LogRecorder, NewBackoff(), nodegroupconfig.NewDefaultNodeGroupConfigProcessor(config.NodeGroupAutoscalingOptions{MaxNodeProvisionTime: 15 * time.Minute}), asyncnodegroups.NewDefaultAsyncNodeGroupStateChecker()) clusterState.UpdateNodes(nodes, nodeInfos, time.Now()) @@ -1646,6 +1655,8 @@ func TestScaleUpToMeetNodeGroupMinSize(t *testing.T) { assert.NoError(t, err) nodes := []*apiv1.Node{n1, n2} + err = context.ClusterSnapshot.Initialize(nodes, nil) + assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&context, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, time.Now()) processors := NewTestProcessors(&context) clusterState := clusterstate.NewClusterStateRegistry(provider, clusterstate.ClusterStateRegistryConfig{}, context.LogRecorder, NewBackoff(), nodegroupconfig.NewDefaultNodeGroupConfigProcessor(config.NodeGroupAutoscalingOptions{MaxNodeProvisionTime: 15 * time.Minute}), asyncnodegroups.NewDefaultAsyncNodeGroupStateChecker()) diff --git a/cluster-autoscaler/core/scaleup/resource/manager_test.go b/cluster-autoscaler/core/scaleup/resource/manager_test.go index dfe7b9415770..8bcfc4fa2b49 100644 --- a/cluster-autoscaler/core/scaleup/resource/manager_test.go +++ b/cluster-autoscaler/core/scaleup/resource/manager_test.go @@ -69,6 +69,8 @@ func TestDeltaForNode(t *testing.T) { ng := testCase.nodeGroupConfig group, nodes := newNodeGroup(t, cp, ng.Name, ng.Min, ng.Max, ng.Size, ng.CPU, ng.Mem) + err := ctx.ClusterSnapshot.Initialize(nodes, nil) + assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&ctx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, time.Now()) rm := NewManager(processors.CustomResourcesProcessor) @@ -110,6 +112,8 @@ func TestResourcesLeft(t *testing.T) { ng := testCase.nodeGroupConfig _, nodes := newNodeGroup(t, cp, ng.Name, ng.Min, ng.Max, ng.Size, ng.CPU, ng.Mem) + err := ctx.ClusterSnapshot.Initialize(nodes, nil) + assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&ctx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, time.Now()) rm := NewManager(processors.CustomResourcesProcessor) @@ -161,6 +165,8 @@ func TestApplyLimits(t *testing.T) { ng := testCase.nodeGroupConfig group, nodes := newNodeGroup(t, cp, ng.Name, ng.Min, ng.Max, ng.Size, ng.CPU, ng.Mem) + err := ctx.ClusterSnapshot.Initialize(nodes, nil) + assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&ctx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, time.Now()) rm := NewManager(processors.CustomResourcesProcessor) @@ -226,6 +232,8 @@ func TestResourceManagerWithGpuResource(t *testing.T) { assert.NoError(t, err) nodes := []*corev1.Node{n1} + err = context.ClusterSnapshot.Initialize(nodes, nil) + assert.NoError(t, err) nodeInfos, _ := nodeinfosprovider.NewDefaultTemplateNodeInfoProvider(nil, false).Process(&context, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, time.Now()) rm := NewManager(processors.CustomResourcesProcessor) diff --git a/cluster-autoscaler/core/static_autoscaler.go b/cluster-autoscaler/core/static_autoscaler.go index b413d26093b7..3db8fb7967bc 100644 --- a/cluster-autoscaler/core/static_autoscaler.go +++ b/cluster-autoscaler/core/static_autoscaler.go @@ -52,7 +52,6 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/utils/backoff" caerrors "k8s.io/autoscaler/cluster-autoscaler/utils/errors" kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" - scheduler_utils "k8s.io/autoscaler/cluster-autoscaler/utils/scheduler" "k8s.io/autoscaler/cluster-autoscaler/utils/taints" "k8s.io/utils/integer" @@ -1016,7 +1015,7 @@ func getUpcomingNodeInfos(upcomingCounts map[string]int, nodeInfos map[string]*f // Ensure new nodes have different names because nodeName // will be used as a map key. Also deep copy pods (daemonsets & // any pods added by cloud provider on template). - nodes = append(nodes, scheduler_utils.DeepCopyTemplateNode(nodeTemplate, fmt.Sprintf("upcoming-%d", i))) + nodes = append(nodes, simulator.FreshNodeInfoFromTemplateNodeInfo(nodeTemplate, fmt.Sprintf("upcoming-%d", i))) } upcomingNodes[nodeGroup] = nodes } diff --git a/cluster-autoscaler/core/static_autoscaler_test.go b/cluster-autoscaler/core/static_autoscaler_test.go index 16c139bc801f..34e76164a382 100644 --- a/cluster-autoscaler/core/static_autoscaler_test.go +++ b/cluster-autoscaler/core/static_autoscaler_test.go @@ -402,7 +402,7 @@ func TestStaticAutoscalerRunOnce(t *testing.T) { // MaxNodesTotal reached. readyNodeLister.SetNodes([]*apiv1.Node{n1}) allNodeLister.SetNodes([]*apiv1.Node{n1}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() @@ -413,7 +413,7 @@ func TestStaticAutoscalerRunOnce(t *testing.T) { // Scale up. readyNodeLister.SetNodes([]*apiv1.Node{n1}) allNodeLister.SetNodes([]*apiv1.Node{n1}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() onScaleUpMock.On("ScaleUp", "ng1", 1).Return(nil).Once() @@ -427,7 +427,7 @@ func TestStaticAutoscalerRunOnce(t *testing.T) { // Mark unneeded nodes. readyNodeLister.SetNodes([]*apiv1.Node{n1, n2}) allNodeLister.SetNodes([]*apiv1.Node{n1, n2}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{p1}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() @@ -442,7 +442,7 @@ func TestStaticAutoscalerRunOnce(t *testing.T) { // Scale down. readyNodeLister.SetNodes([]*apiv1.Node{n1, n2}) allNodeLister.SetNodes([]*apiv1.Node{n1, n2}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1}, nil).Times(3) + allPodListerMock.On("List").Return([]*apiv1.Pod{p1}, nil).Twice() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() onScaleDownMock.On("ScaleDown", "ng1", "n2").Return(nil).Once() @@ -456,7 +456,7 @@ func TestStaticAutoscalerRunOnce(t *testing.T) { // Mark unregistered nodes. readyNodeLister.SetNodes([]*apiv1.Node{n1, n2}) allNodeLister.SetNodes([]*apiv1.Node{n1, n2}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() @@ -471,7 +471,7 @@ func TestStaticAutoscalerRunOnce(t *testing.T) { // Remove unregistered nodes. readyNodeLister.SetNodes([]*apiv1.Node{n1, n2}) allNodeLister.SetNodes([]*apiv1.Node{n1, n2}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() onScaleDownMock.On("ScaleDown", "ng2", "n3").Return(nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() @@ -485,7 +485,7 @@ func TestStaticAutoscalerRunOnce(t *testing.T) { // Scale up to node group min size. readyNodeLister.SetNodes([]*apiv1.Node{n4}) allNodeLister.SetNodes([]*apiv1.Node{n4}) - allPodListerMock.On("List").Return([]*apiv1.Pod{}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil) podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil) onScaleUpMock.On("ScaleUp", "ng3", 2).Return(nil).Once() // 2 new nodes are supposed to be scaled up. @@ -685,7 +685,7 @@ func TestStaticAutoscalerRunOnceWithScaleDownDelayPerNG(t *testing.T) { // Mark unneeded nodes. readyNodeLister.SetNodes([]*apiv1.Node{n1, n2}) allNodeLister.SetNodes([]*apiv1.Node{n1, n2}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{p1}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() @@ -697,7 +697,7 @@ func TestStaticAutoscalerRunOnceWithScaleDownDelayPerNG(t *testing.T) { // Scale down nodegroup readyNodeLister.SetNodes([]*apiv1.Node{n1, n2}) allNodeLister.SetNodes([]*apiv1.Node{n1, n2}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1}, nil).Times(3) + allPodListerMock.On("List").Return([]*apiv1.Pod{p1}, nil).Twice() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil) onScaleDownMock.On("ScaleDown", tc.expectedScaleDownNG, tc.expectedScaleDownNode).Return(nil).Once() @@ -824,7 +824,7 @@ func TestStaticAutoscalerRunOnceWithAutoprovisionedEnabled(t *testing.T) { // Scale up. readyNodeLister.SetNodes([]*apiv1.Node{n1}) allNodeLister.SetNodes([]*apiv1.Node{n1}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() onNodeGroupCreateMock.On("Create", "autoprovisioned-TN2").Return(nil).Once() @@ -841,7 +841,7 @@ func TestStaticAutoscalerRunOnceWithAutoprovisionedEnabled(t *testing.T) { // Remove autoprovisioned node group and mark unneeded nodes. readyNodeLister.SetNodes([]*apiv1.Node{n1, n2}) allNodeLister.SetNodes([]*apiv1.Node{n1, n2}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{p1}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() onNodeGroupDeleteMock.On("Delete", "autoprovisioned-TN1").Return(nil).Once() @@ -857,7 +857,7 @@ func TestStaticAutoscalerRunOnceWithAutoprovisionedEnabled(t *testing.T) { // Scale down. readyNodeLister.SetNodes([]*apiv1.Node{n1, n2}) allNodeLister.SetNodes([]*apiv1.Node{n1, n2}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1}, nil).Times(3) + allPodListerMock.On("List").Return([]*apiv1.Pod{p1}, nil).Twice() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() onNodeGroupDeleteMock.On("Delete", "autoprovisioned-"+ @@ -974,7 +974,7 @@ func TestStaticAutoscalerRunOnceWithALongUnregisteredNode(t *testing.T) { // Scale up. readyNodeLister.SetNodes([]*apiv1.Node{n1}) allNodeLister.SetNodes([]*apiv1.Node{n1}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() onScaleUpMock.On("ScaleUp", "ng1", 1).Return(nil).Once() @@ -990,7 +990,7 @@ func TestStaticAutoscalerRunOnceWithALongUnregisteredNode(t *testing.T) { readyNodeLister.SetNodes([]*apiv1.Node{n1, n2}) allNodeLister.SetNodes([]*apiv1.Node{n1, n2}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2}, nil).Once() onScaleDownMock.On("ScaleDown", "ng1", "broken").Return(nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() @@ -1123,7 +1123,7 @@ func TestStaticAutoscalerRunOncePodsWithPriorities(t *testing.T) { // Scale up readyNodeLister.SetNodes([]*apiv1.Node{n1, n2, n3}) allNodeLister.SetNodes([]*apiv1.Node{n1, n2, n3}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2, p3, p4, p5, p6}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2, p3, p4, p5, p6}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() onScaleUpMock.On("ScaleUp", "ng2", 1).Return(nil).Once() @@ -1136,7 +1136,7 @@ func TestStaticAutoscalerRunOncePodsWithPriorities(t *testing.T) { // Mark unneeded nodes. readyNodeLister.SetNodes([]*apiv1.Node{n1, n2, n3}) allNodeLister.SetNodes([]*apiv1.Node{n1, n2, n3}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2, p3, p4, p5}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2, p3, p4, p5}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() @@ -1150,7 +1150,7 @@ func TestStaticAutoscalerRunOncePodsWithPriorities(t *testing.T) { // Scale down. readyNodeLister.SetNodes([]*apiv1.Node{n1, n2, n3}) allNodeLister.SetNodes([]*apiv1.Node{n1, n2, n3}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2, p3, p4, p5}, nil).Times(3) + allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2, p3, p4, p5}, nil).Twice() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() onScaleDownMock.On("ScaleDown", "ng1", "n1").Return(nil).Once() @@ -1252,7 +1252,7 @@ func TestStaticAutoscalerRunOnceWithFilteringOnBinPackingEstimator(t *testing.T) // Scale up readyNodeLister.SetNodes([]*apiv1.Node{n1, n2}) allNodeLister.SetNodes([]*apiv1.Node{n1, n2}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p3, p4}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p3, p4}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() @@ -1351,7 +1351,7 @@ func TestStaticAutoscalerRunOnceWithFilteringOnUpcomingNodesEnabledNoScaleUp(t * // Scale up readyNodeLister.SetNodes([]*apiv1.Node{n2, n3}) allNodeLister.SetNodes([]*apiv1.Node{n2, n3}) - allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2, p3}, nil).Twice() + allPodListerMock.On("List").Return([]*apiv1.Pod{p1, p2, p3}, nil).Once() daemonSetListerMock.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() podDisruptionBudgetListerMock.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() @@ -1552,7 +1552,7 @@ func TestStaticAutoscalerRunOnceWithBypassedSchedulers(t *testing.T) { tc.setupConfig.mocks.readyNodeLister.SetNodes([]*apiv1.Node{n1}) tc.setupConfig.mocks.allNodeLister.SetNodes([]*apiv1.Node{n1}) - tc.setupConfig.mocks.allPodLister.On("List").Return(tc.pods, nil).Twice() + tc.setupConfig.mocks.allPodLister.On("List").Return(tc.pods, nil).Once() tc.setupConfig.mocks.daemonSetLister.On("List", labels.Everything()).Return([]*appsv1.DaemonSet{}, nil).Once() tc.setupConfig.mocks.podDisruptionBudgetLister.On("List").Return([]*policyv1.PodDisruptionBudget{}, nil).Once() if tc.expectedScaleUp != nil { diff --git a/cluster-autoscaler/core/utils/utils.go b/cluster-autoscaler/core/utils/utils.go index 5cb1720e8e9e..1b493b783cfc 100644 --- a/cluster-autoscaler/core/utils/utils.go +++ b/cluster-autoscaler/core/utils/utils.go @@ -17,52 +17,17 @@ limitations under the License. package utils import ( - "fmt" - "math/rand" "reflect" "time" - appsv1 "k8s.io/api/apps/v1" apiv1 "k8s.io/api/core/v1" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/autoscaler/cluster-autoscaler/clusterstate" "k8s.io/autoscaler/cluster-autoscaler/metrics" - "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" - "k8s.io/autoscaler/cluster-autoscaler/utils/daemonset" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" - "k8s.io/autoscaler/cluster-autoscaler/utils/labels" - "k8s.io/autoscaler/cluster-autoscaler/utils/taints" ) -// GetNodeInfoFromTemplate returns NodeInfo object built base on TemplateNodeInfo returned by NodeGroup.TemplateNodeInfo(). -func GetNodeInfoFromTemplate(nodeGroup cloudprovider.NodeGroup, daemonsets []*appsv1.DaemonSet, taintConfig taints.TaintConfig) (*framework.NodeInfo, errors.AutoscalerError) { - id := nodeGroup.Id() - baseNodeInfo, err := nodeGroup.TemplateNodeInfo() - if err != nil { - return nil, errors.ToAutoscalerError(errors.CloudProviderError, err) - } - - labels.UpdateDeprecatedLabels(baseNodeInfo.Node().ObjectMeta.Labels) - - sanitizedNode, typedErr := SanitizeNode(baseNodeInfo.Node(), id, taintConfig) - if err != nil { - return nil, typedErr - } - baseNodeInfo.SetNode(sanitizedNode) - - pods, err := daemonset.GetDaemonSetPodsForNode(baseNodeInfo, daemonsets) - if err != nil { - return nil, errors.ToAutoscalerError(errors.InternalError, err) - } - for _, podInfo := range baseNodeInfo.Pods { - pods = append(pods, &framework.PodInfo{Pod: podInfo.Pod}) - } - - sanitizedNodeInfo := framework.NewNodeInfo(sanitizedNode, nil, SanitizePods(pods, sanitizedNode)...) - return sanitizedNodeInfo, nil -} - // isVirtualNode determines if the node is created by virtual kubelet func isVirtualNode(node *apiv1.Node) bool { return node.ObjectMeta.Labels["type"] == "virtual-kubelet" @@ -89,48 +54,6 @@ func FilterOutNodesFromNotAutoscaledGroups(nodes []*apiv1.Node, cloudProvider cl return result, nil } -// DeepCopyNodeInfo clones the provided nodeInfo -func DeepCopyNodeInfo(nodeInfo *framework.NodeInfo) *framework.NodeInfo { - newPods := make([]*framework.PodInfo, 0) - for _, podInfo := range nodeInfo.Pods { - newPods = append(newPods, &framework.PodInfo{Pod: podInfo.Pod.DeepCopy()}) - } - - // Build a new node info. - newNodeInfo := framework.NewNodeInfo(nodeInfo.Node().DeepCopy(), nil, newPods...) - return newNodeInfo -} - -// SanitizeNode cleans up nodes used for node group templates -func SanitizeNode(node *apiv1.Node, nodeGroup string, taintConfig taints.TaintConfig) (*apiv1.Node, errors.AutoscalerError) { - newNode := node.DeepCopy() - nodeName := fmt.Sprintf("template-node-for-%s-%d", nodeGroup, rand.Int63()) - newNode.Labels = make(map[string]string, len(node.Labels)) - for k, v := range node.Labels { - if k != apiv1.LabelHostname { - newNode.Labels[k] = v - } else { - newNode.Labels[k] = nodeName - } - } - newNode.Name = nodeName - newNode.Spec.Taints = taints.SanitizeTaints(newNode.Spec.Taints, taintConfig) - return newNode, nil -} - -// SanitizePods cleans up pods used for node group templates -func SanitizePods(pods []*framework.PodInfo, sanitizedNode *apiv1.Node) []*framework.PodInfo { - // Update node name in pods. - sanitizedPods := make([]*framework.PodInfo, 0) - for _, pod := range pods { - sanitizedPod := pod.Pod.DeepCopy() - sanitizedPod.Spec.NodeName = sanitizedNode.Name - sanitizedPods = append(sanitizedPods, &framework.PodInfo{Pod: sanitizedPod}) - } - - return sanitizedPods -} - func hasHardInterPodAffinity(affinity *apiv1.Affinity) bool { if affinity == nil { return false diff --git a/cluster-autoscaler/core/utils/utils_test.go b/cluster-autoscaler/core/utils/utils_test.go index b63badbcc834..2613b0419a13 100644 --- a/cluster-autoscaler/core/utils/utils_test.go +++ b/cluster-autoscaler/core/utils/utils_test.go @@ -20,8 +20,6 @@ import ( "testing" "time" - "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" - "k8s.io/autoscaler/cluster-autoscaler/utils/taints" . "k8s.io/autoscaler/cluster-autoscaler/utils/test" "github.com/stretchr/testify/assert" @@ -29,33 +27,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestSanitizePods(t *testing.T) { - pod := BuildTestPod("p1", 80, 0) - pod.Spec.NodeName = "n1" - pods := []*framework.PodInfo{{Pod: pod}} - - node := BuildTestNode("node", 1000, 1000) - - resNode, err := SanitizeNode(node, "test-group", taints.TaintConfig{}) - assert.NoError(t, err) - res := SanitizePods(pods, resNode) - assert.Equal(t, 1, len(res)) -} - -func TestSanitizeLabels(t *testing.T) { - oldNode := BuildTestNode("ng1-1", 1000, 1000) - oldNode.Labels = map[string]string{ - apiv1.LabelHostname: "abc", - "x": "y", - } - node, err := SanitizeNode(oldNode, "bzium", taints.TaintConfig{}) - assert.NoError(t, err) - assert.NotEqual(t, node.Labels[apiv1.LabelHostname], "abc", nil) - assert.Equal(t, node.Labels["x"], "y") - assert.NotEqual(t, node.Name, oldNode.Name) - assert.Equal(t, node.Labels[apiv1.LabelHostname], node.Name) -} - func TestGetNodeResource(t *testing.T) { node := BuildTestNode("n1", 1000, 2*MiB) diff --git a/cluster-autoscaler/estimator/binpacking_estimator.go b/cluster-autoscaler/estimator/binpacking_estimator.go index 46470609f50d..4ab6c050ac26 100644 --- a/cluster-autoscaler/estimator/binpacking_estimator.go +++ b/cluster-autoscaler/estimator/binpacking_estimator.go @@ -21,10 +21,10 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + core_utils "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" "k8s.io/autoscaler/cluster-autoscaler/simulator/predicatechecker" - "k8s.io/autoscaler/cluster-autoscaler/utils/scheduler" klog "k8s.io/klog/v2" ) @@ -210,7 +210,7 @@ func (e *BinpackingNodeEstimator) addNewNodeToSnapshot( estimationState *estimationState, template *framework.NodeInfo, ) error { - newNodeInfo := scheduler.DeepCopyTemplateNode(template, fmt.Sprintf("e-%d", estimationState.newNodeNameIndex)) + newNodeInfo := core_utils.FreshNodeInfoFromTemplateNodeInfo(template, fmt.Sprintf("e-%d", estimationState.newNodeNameIndex)) if err := e.clusterSnapshot.AddNodeInfo(newNodeInfo); err != nil { return err } diff --git a/cluster-autoscaler/processors/nodeinfosprovider/mixed_nodeinfos_processor.go b/cluster-autoscaler/processors/nodeinfosprovider/mixed_nodeinfos_processor.go index 036593b7e328..30e871259eab 100644 --- a/cluster-autoscaler/processors/nodeinfosprovider/mixed_nodeinfos_processor.go +++ b/cluster-autoscaler/processors/nodeinfosprovider/mixed_nodeinfos_processor.go @@ -24,7 +24,6 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/autoscaler/cluster-autoscaler/context" - "k8s.io/autoscaler/cluster-autoscaler/core/utils" "k8s.io/autoscaler/cluster-autoscaler/simulator" "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" @@ -78,11 +77,6 @@ func (p *MixedTemplateNodeInfoProvider) Process(ctx *context.AutoscalingContext, result := make(map[string]*framework.NodeInfo) seenGroups := make(map[string]bool) - podsForNodes, err := getPodsForNodes(ctx.ListerRegistry) - if err != nil { - return map[string]*framework.NodeInfo{}, err - } - // processNode returns information whether the nodeTemplate was generated and if there was an error. processNode := func(node *apiv1.Node) (bool, string, errors.AutoscalerError) { nodeGroup, err := ctx.CloudProvider.NodeGroupForNode(node) @@ -94,22 +88,15 @@ func (p *MixedTemplateNodeInfoProvider) Process(ctx *context.AutoscalingContext, } id := nodeGroup.Id() if _, found := result[id]; !found { - // Build nodeInfo. - sanitizedNode, err := utils.SanitizeNode(node, id, taintConfig) + nodeInfo, err := ctx.ClusterSnapshot.GetNodeInfo(node.Name) if err != nil { - return false, "", err + return false, "", errors.NewAutoscalerError(errors.InternalError, "error while retrieving node %s from cluster snapshot - this shouldn't happen: %v", node.Name, err) } - nodeInfo, err := simulator.BuildNodeInfoForNode(sanitizedNode, podsForNodes[node.Name], daemonsets, p.forceDaemonSets) + templateNodeInfo, caErr := simulator.TemplateNodeInfoFromExampleNodeInfo(nodeInfo, id, daemonsets, p.forceDaemonSets, taintConfig) if err != nil { - return false, "", err - } - - var pods []*apiv1.Pod - for _, podInfo := range nodeInfo.Pods { - pods = append(pods, podInfo.Pod) + return false, "", caErr } - sanitizedNodeInfo := framework.NewNodeInfo(sanitizedNode, nil, utils.SanitizePods(nodeInfo.Pods, sanitizedNode)...) - result[id] = sanitizedNodeInfo + result[id] = templateNodeInfo return true, id, nil } return false, "", nil @@ -125,7 +112,7 @@ func (p *MixedTemplateNodeInfoProvider) Process(ctx *context.AutoscalingContext, return map[string]*framework.NodeInfo{}, typedErr } if added && p.nodeInfoCache != nil { - nodeInfoCopy := utils.DeepCopyNodeInfo(result[id]) + nodeInfoCopy := simulator.DeepCopyNodeInfo(result[id]) p.nodeInfoCache[id] = cacheItem{NodeInfo: nodeInfoCopy, added: time.Now()} } } @@ -142,7 +129,7 @@ func (p *MixedTemplateNodeInfoProvider) Process(ctx *context.AutoscalingContext, if p.isCacheItemExpired(cacheItem.added) { delete(p.nodeInfoCache, id) } else { - result[id] = utils.DeepCopyNodeInfo(cacheItem.NodeInfo) + result[id] = simulator.DeepCopyNodeInfo(cacheItem.NodeInfo) continue } } @@ -150,7 +137,7 @@ func (p *MixedTemplateNodeInfoProvider) Process(ctx *context.AutoscalingContext, // No good template, trying to generate one. This is called only if there are no // working nodes in the node groups. By default CA tries to use a real-world example. - nodeInfo, err := utils.GetNodeInfoFromTemplate(nodeGroup, daemonsets, taintConfig) + nodeInfo, err := simulator.TemplateNodeInfoFromNodeGroupTemplate(nodeGroup, daemonsets, taintConfig) if err != nil { if err == cloudprovider.ErrNotImplemented { continue @@ -192,19 +179,6 @@ func (p *MixedTemplateNodeInfoProvider) Process(ctx *context.AutoscalingContext, return result, nil } -func getPodsForNodes(listers kube_util.ListerRegistry) (map[string][]*apiv1.Pod, errors.AutoscalerError) { - pods, err := listers.AllPodLister().List() - if err != nil { - return nil, errors.ToAutoscalerError(errors.ApiCallError, err) - } - scheduledPods := kube_util.ScheduledPods(pods) - podsForNodes := map[string][]*apiv1.Pod{} - for _, p := range scheduledPods { - podsForNodes[p.Spec.NodeName] = append(podsForNodes[p.Spec.NodeName], p) - } - return podsForNodes, nil -} - func isNodeGoodTemplateCandidate(node *apiv1.Node, now time.Time) bool { ready, lastTransitionTime, _ := kube_util.GetReadinessState(node) stable := lastTransitionTime.Add(stabilizationDelay).Before(now) diff --git a/cluster-autoscaler/processors/nodeinfosprovider/mixed_nodeinfos_processor_test.go b/cluster-autoscaler/processors/nodeinfosprovider/mixed_nodeinfos_processor_test.go index f3ebce8a1995..5c89404a6367 100644 --- a/cluster-autoscaler/processors/nodeinfosprovider/mixed_nodeinfos_processor_test.go +++ b/cluster-autoscaler/processors/nodeinfosprovider/mixed_nodeinfos_processor_test.go @@ -22,6 +22,7 @@ import ( testprovider "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/test" "k8s.io/autoscaler/cluster-autoscaler/context" + "k8s.io/autoscaler/cluster-autoscaler/simulator/clustersnapshot" "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" "k8s.io/autoscaler/cluster-autoscaler/simulator/predicatechecker" kube_util "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" @@ -78,14 +79,20 @@ func TestGetNodeInfosForGroups(t *testing.T) { predicateChecker, err := predicatechecker.NewTestPredicateChecker() assert.NoError(t, err) + nodes := []*apiv1.Node{justReady5, unready4, unready3, ready2, ready1} + snapshot := clustersnapshot.NewBasicClusterSnapshot() + err = snapshot.Initialize(nodes, nil) + assert.NoError(t, err) + ctx := context.AutoscalingContext{ CloudProvider: provider1, + ClusterSnapshot: snapshot, PredicateChecker: predicateChecker, AutoscalingKubeClients: context.AutoscalingKubeClients{ ListerRegistry: registry, }, } - res, err := NewMixedTemplateNodeInfoProvider(&cacheTtl, false).Process(&ctx, []*apiv1.Node{justReady5, unready4, unready3, ready2, ready1}, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) + res, err := NewMixedTemplateNodeInfoProvider(&cacheTtl, false).Process(&ctx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) assert.NoError(t, err) assert.Equal(t, 5, len(res)) info, found := res["ng1"] @@ -107,6 +114,7 @@ func TestGetNodeInfosForGroups(t *testing.T) { // Test for a nodegroup without nodes and TemplateNodeInfo not implemented by cloud proivder ctx = context.AutoscalingContext{ CloudProvider: provider2, + ClusterSnapshot: clustersnapshot.NewBasicClusterSnapshot(), PredicateChecker: predicateChecker, AutoscalingKubeClients: context.AutoscalingKubeClients{ ListerRegistry: registry, @@ -162,16 +170,22 @@ func TestGetNodeInfosForGroupsCache(t *testing.T) { predicateChecker, err := predicatechecker.NewTestPredicateChecker() assert.NoError(t, err) + nodes := []*apiv1.Node{unready4, unready3, ready2, ready1} + snapshot := clustersnapshot.NewBasicClusterSnapshot() + err = snapshot.Initialize(nodes, nil) + assert.NoError(t, err) + // Fill cache ctx := context.AutoscalingContext{ CloudProvider: provider1, + ClusterSnapshot: snapshot, PredicateChecker: predicateChecker, AutoscalingKubeClients: context.AutoscalingKubeClients{ ListerRegistry: registry, }, } niProcessor := NewMixedTemplateNodeInfoProvider(&cacheTtl, false) - res, err := niProcessor.Process(&ctx, []*apiv1.Node{unready4, unready3, ready2, ready1}, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) + res, err := niProcessor.Process(&ctx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) assert.NoError(t, err) // Check results assert.Equal(t, 4, len(res)) @@ -205,7 +219,7 @@ func TestGetNodeInfosForGroupsCache(t *testing.T) { assert.Equal(t, "ng3", lastDeletedGroup) // Check cache with all nodes removed - res, err = niProcessor.Process(&ctx, []*apiv1.Node{unready4, unready3, ready2, ready1}, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) + res, err = niProcessor.Process(&ctx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) assert.NoError(t, err) // Check results assert.Equal(t, 2, len(res)) @@ -226,7 +240,7 @@ func TestGetNodeInfosForGroupsCache(t *testing.T) { // Fill cache manually infoNg4Node6 := framework.NewNodeInfo(ready6.DeepCopy(), nil) niProcessor.nodeInfoCache = map[string]cacheItem{"ng4": {NodeInfo: infoNg4Node6, added: now}} - res, err = niProcessor.Process(&ctx, []*apiv1.Node{unready4, unready3, ready2, ready1}, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) + res, err = niProcessor.Process(&ctx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) // Check if cache was used assert.NoError(t, err) assert.Equal(t, 2, len(res)) @@ -250,8 +264,14 @@ func TestGetNodeInfosCacheExpired(t *testing.T) { predicateChecker, err := predicatechecker.NewTestPredicateChecker() assert.NoError(t, err) + nodes := []*apiv1.Node{ready1} + snapshot := clustersnapshot.NewBasicClusterSnapshot() + err = snapshot.Initialize(nodes, nil) + assert.NoError(t, err) + ctx := context.AutoscalingContext{ CloudProvider: provider, + ClusterSnapshot: snapshot, PredicateChecker: predicateChecker, AutoscalingKubeClients: context.AutoscalingKubeClients{ ListerRegistry: registry, @@ -269,7 +289,7 @@ func TestGetNodeInfosCacheExpired(t *testing.T) { provider.AddNode("ng1", ready1) assert.Equal(t, 2, len(niProcessor1.nodeInfoCache)) - _, err = niProcessor1.Process(&ctx, []*apiv1.Node{ready1}, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) + _, err = niProcessor1.Process(&ctx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) assert.NoError(t, err) assert.Equal(t, 1, len(niProcessor1.nodeInfoCache)) @@ -280,7 +300,7 @@ func TestGetNodeInfosCacheExpired(t *testing.T) { "ng2": {NodeInfo: tni, added: now.Add(-2 * time.Second)}, } assert.Equal(t, 2, len(niProcessor2.nodeInfoCache)) - _, err = niProcessor1.Process(&ctx, []*apiv1.Node{ready1}, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) + _, err = niProcessor1.Process(&ctx, nodes, []*appsv1.DaemonSet{}, taints.TaintConfig{}, now) assert.NoError(t, err) assert.Equal(t, 2, len(niProcessor2.nodeInfoCache)) diff --git a/cluster-autoscaler/simulator/node_info.go b/cluster-autoscaler/simulator/node_info.go new file mode 100644 index 000000000000..eaf43a51b52c --- /dev/null +++ b/cluster-autoscaler/simulator/node_info.go @@ -0,0 +1,157 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package simulator + +import ( + "fmt" + "math/rand" + + appsv1 "k8s.io/api/apps/v1" + apiv1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" + "k8s.io/autoscaler/cluster-autoscaler/utils/daemonset" + "k8s.io/autoscaler/cluster-autoscaler/utils/errors" + "k8s.io/autoscaler/cluster-autoscaler/utils/labels" + pod_util "k8s.io/autoscaler/cluster-autoscaler/utils/pod" + "k8s.io/autoscaler/cluster-autoscaler/utils/taints" +) + +// TemplateNodeInfoFromNodeGroupTemplate returns a template NodeInfo object based on NodeGroup.TemplateNodeInfo(). The template is sanitized, and only +// contains the pods that should appear on a new Node from the same node group (e.g. DaemonSet pods). +func TemplateNodeInfoFromNodeGroupTemplate(nodeGroup cloudprovider.NodeGroup, daemonsets []*appsv1.DaemonSet, taintConfig taints.TaintConfig) (*framework.NodeInfo, errors.AutoscalerError) { + id := nodeGroup.Id() + baseNodeInfo, err := nodeGroup.TemplateNodeInfo() + if err != nil { + return nil, errors.ToAutoscalerError(errors.CloudProviderError, err) + } + labels.UpdateDeprecatedLabels(baseNodeInfo.Node().ObjectMeta.Labels) + + return TemplateNodeInfoFromExampleNodeInfo(baseNodeInfo, id, daemonsets, true, taintConfig) +} + +// TemplateNodeInfoFromExampleNodeInfo returns a template NodeInfo object based on a real example NodeInfo from the cluster. The template is sanitized, and only +// contains the pods that should appear on a new Node from the same node group (e.g. DaemonSet pods). +func TemplateNodeInfoFromExampleNodeInfo(example *framework.NodeInfo, nodeGroupId string, daemonsets []*appsv1.DaemonSet, forceDaemonSets bool, taintConfig taints.TaintConfig) (*framework.NodeInfo, errors.AutoscalerError) { + randSuffix := fmt.Sprintf("%d", rand.Int63()) + newNodeNameBase := fmt.Sprintf("template-node-for-%s", nodeGroupId) + + expectedPods, err := podsExpectedOnFreshNode(example, daemonsets, forceDaemonSets) + if err != nil { + return nil, err + } + exampleWithOnlyExpectedPods := framework.NewNodeInfo(example.Node(), nil, expectedPods...) + + return sanitizeNodeInfo(exampleWithOnlyExpectedPods, newNodeNameBase, randSuffix, &taintConfig), nil +} + +// FreshNodeInfoFromTemplateNodeInfo duplicates the provided template NodeInfo, returning a fresh NodeInfo that can be injected into the cluster snapshot. +// The NodeInfo is sanitized (names, UIDs are changed, etc.), so that it can be injected along other copies created from the same template. +func FreshNodeInfoFromTemplateNodeInfo(template *framework.NodeInfo, suffix string) *framework.NodeInfo { + // Template node infos should already have taints and pods filtered, so not setting these parameters. + return sanitizeNodeInfo(template, template.Node().Name, suffix, nil) +} + +// DeepCopyNodeInfo clones the provided NodeInfo +func DeepCopyNodeInfo(nodeInfo *framework.NodeInfo) *framework.NodeInfo { + newPods := make([]*framework.PodInfo, 0) + for _, podInfo := range nodeInfo.Pods { + newPods = append(newPods, &framework.PodInfo{Pod: podInfo.Pod.DeepCopy()}) + } + + // Build a new node info. + newNodeInfo := framework.NewNodeInfo(nodeInfo.Node().DeepCopy(), nil, newPods...) + return newNodeInfo +} + +func sanitizeNodeInfo(nodeInfo *framework.NodeInfo, newNodeNameBase string, namesSuffix string, taintConfig *taints.TaintConfig) *framework.NodeInfo { + freshNodeName := fmt.Sprintf("%s-%s", newNodeNameBase, namesSuffix) + freshNode := sanitizeNode(nodeInfo.Node(), freshNodeName, taintConfig) + result := framework.NewNodeInfo(freshNode, nil) + + for _, podInfo := range nodeInfo.Pods { + freshPod := sanitizePod(podInfo.Pod, freshNode.Name, namesSuffix) + result.AddPod(&framework.PodInfo{Pod: freshPod}) + } + return result +} + +func sanitizeNode(node *apiv1.Node, newName string, taintConfig *taints.TaintConfig) *apiv1.Node { + newNode := node.DeepCopy() + newNode.Labels = make(map[string]string, len(node.Labels)) + for k, v := range node.Labels { + if k != apiv1.LabelHostname { + newNode.Labels[k] = v + } else { + newNode.Labels[k] = newName + } + } + newNode.Name = newName + newNode.UID = uuid.NewUUID() + if taintConfig != nil { + newNode.Spec.Taints = taints.SanitizeTaints(newNode.Spec.Taints, *taintConfig) + } + return newNode +} + +func sanitizePod(pod *apiv1.Pod, nodeName, nameSuffix string) *apiv1.Pod { + sanitizedPod := pod.DeepCopy() + sanitizedPod.UID = uuid.NewUUID() + sanitizedPod.Name = fmt.Sprintf("%s-%s", pod.Name, nameSuffix) + sanitizedPod.Spec.NodeName = nodeName + return sanitizedPod +} + +func podsExpectedOnFreshNode(exampleNodeInfo *framework.NodeInfo, daemonsets []*appsv1.DaemonSet, forceDaemonSets bool) ([]*framework.PodInfo, errors.AutoscalerError) { + var result []*framework.PodInfo + runningDS := make(map[types.UID]bool) + for _, pod := range exampleNodeInfo.Pods { + // Ignore scheduled pods in deletion phase + if pod.DeletionTimestamp != nil { + continue + } + // Add scheduled mirror and DS pods + if pod_util.IsMirrorPod(pod.Pod) || pod_util.IsDaemonSetPod(pod.Pod) { + result = append(result, pod) + } + // Mark DS pods as running + controllerRef := metav1.GetControllerOf(pod) + if controllerRef != nil && controllerRef.Kind == "DaemonSet" { + runningDS[controllerRef.UID] = true + } + } + // Add all pending DS pods if force scheduling DS + if forceDaemonSets { + var pendingDS []*appsv1.DaemonSet + for _, ds := range daemonsets { + if !runningDS[ds.UID] { + pendingDS = append(pendingDS, ds) + } + } + daemonPods, err := daemonset.GetDaemonSetPodsForNode(exampleNodeInfo, pendingDS) + if err != nil { + return nil, errors.ToAutoscalerError(errors.InternalError, err) + } + for _, pod := range daemonPods { + result = append(result, pod) + } + } + return result, nil +} diff --git a/cluster-autoscaler/simulator/nodes_test.go b/cluster-autoscaler/simulator/node_info_test.go similarity index 59% rename from cluster-autoscaler/simulator/nodes_test.go rename to cluster-autoscaler/simulator/node_info_test.go index 4a2be55b13df..1bed798bc7dc 100644 --- a/cluster-autoscaler/simulator/nodes_test.go +++ b/cluster-autoscaler/simulator/node_info_test.go @@ -1,5 +1,5 @@ /* -Copyright 2016 The Kubernetes Authors. +Copyright 2024 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -26,11 +26,38 @@ import ( apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - "k8s.io/autoscaler/cluster-autoscaler/utils/test" + "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" + "k8s.io/autoscaler/cluster-autoscaler/utils/taints" + . "k8s.io/autoscaler/cluster-autoscaler/utils/test" "k8s.io/kubernetes/pkg/controller/daemon" ) -func TestBuildNodeInfoForNode(t *testing.T) { +func TestSanitizePod(t *testing.T) { + pod := BuildTestPod("p1", 80, 0) + pod.Spec.NodeName = "n1" + + node := BuildTestNode("node", 1000, 1000) + + resNode := sanitizeNode(node, "test-group", nil) + res := sanitizePod(pod, resNode.Name, "abc") + assert.Equal(t, res.Spec.NodeName, resNode.Name) + assert.Equal(t, res.Name, "p1-abc") +} + +func TestSanitizeLabels(t *testing.T) { + oldNode := BuildTestNode("ng1-1", 1000, 1000) + oldNode.Labels = map[string]string{ + apiv1.LabelHostname: "abc", + "x": "y", + } + node := sanitizeNode(oldNode, "bzium", nil) + assert.NotEqual(t, node.Labels[apiv1.LabelHostname], "abc", nil) + assert.Equal(t, node.Labels["x"], "y") + assert.NotEqual(t, node.Name, oldNode.Name) + assert.Equal(t, node.Labels[apiv1.LabelHostname], node.Name) +} + +func TestTemplateNodeInfoFromExampleNodeInfo(t *testing.T) { ds1 := &appsv1.DaemonSet{ ObjectMeta: metav1.ObjectMeta{ Name: "ds1", @@ -74,40 +101,40 @@ func TestBuildNodeInfoForNode(t *testing.T) { }{ { name: "node without any pods", - node: test.BuildTestNode("n", 1000, 10), + node: BuildTestNode("n", 1000, 10), }, { name: "node with non-DS/mirror pods", - node: test.BuildTestNode("n", 1000, 10), + node: BuildTestNode("n", 1000, 10), pods: []*apiv1.Pod{ - test.BuildScheduledTestPod("p1", 100, 1, "n"), - test.BuildScheduledTestPod("p2", 100, 1, "n"), + BuildScheduledTestPod("p1", 100, 1, "n"), + BuildScheduledTestPod("p2", 100, 1, "n"), }, }, { name: "node with a mirror pod", - node: test.BuildTestNode("n", 1000, 10), + node: BuildTestNode("n", 1000, 10), pods: []*apiv1.Pod{ - test.SetMirrorPodSpec(test.BuildScheduledTestPod("p1", 100, 1, "n")), + SetMirrorPodSpec(BuildScheduledTestPod("p1", 100, 1, "n")), }, wantPods: []*apiv1.Pod{ - test.SetMirrorPodSpec(test.BuildScheduledTestPod("p1", 100, 1, "n")), + SetMirrorPodSpec(BuildScheduledTestPod("p1", 100, 1, "n")), }, }, { name: "node with a deleted mirror pod", - node: test.BuildTestNode("n", 1000, 10), + node: BuildTestNode("n", 1000, 10), pods: []*apiv1.Pod{ - test.SetMirrorPodSpec(test.BuildScheduledTestPod("p1", 100, 1, "n")), - setDeletionTimestamp(test.SetMirrorPodSpec(test.BuildScheduledTestPod("p2", 100, 1, "n"))), + SetMirrorPodSpec(BuildScheduledTestPod("p1", 100, 1, "n")), + setDeletionTimestamp(SetMirrorPodSpec(BuildScheduledTestPod("p2", 100, 1, "n"))), }, wantPods: []*apiv1.Pod{ - test.SetMirrorPodSpec(test.BuildScheduledTestPod("p1", 100, 1, "n")), + SetMirrorPodSpec(BuildScheduledTestPod("p1", 100, 1, "n")), }, }, { name: "node with DS pods [forceDS=false, no daemon sets]", - node: test.BuildTestNode("n", 1000, 10), + node: BuildTestNode("n", 1000, 10), pods: []*apiv1.Pod{ buildDSPod(ds1, "n"), setDeletionTimestamp(buildDSPod(ds2, "n")), @@ -118,7 +145,7 @@ func TestBuildNodeInfoForNode(t *testing.T) { }, { name: "node with DS pods [forceDS=false, some daemon sets]", - node: test.BuildTestNode("n", 1000, 10), + node: BuildTestNode("n", 1000, 10), pods: []*apiv1.Pod{ buildDSPod(ds1, "n"), setDeletionTimestamp(buildDSPod(ds2, "n")), @@ -130,7 +157,7 @@ func TestBuildNodeInfoForNode(t *testing.T) { }, { name: "node with a DS pod [forceDS=true, no daemon sets]", - node: test.BuildTestNode("n", 1000, 10), + node: BuildTestNode("n", 1000, 10), pods: []*apiv1.Pod{ buildDSPod(ds1, "n"), setDeletionTimestamp(buildDSPod(ds2, "n")), @@ -142,7 +169,7 @@ func TestBuildNodeInfoForNode(t *testing.T) { }, { name: "node with a DS pod [forceDS=true, some daemon sets]", - node: test.BuildTestNode("n", 1000, 10), + node: BuildTestNode("n", 1000, 10), pods: []*apiv1.Pod{ buildDSPod(ds1, "n"), setDeletionTimestamp(buildDSPod(ds2, "n")), @@ -156,36 +183,36 @@ func TestBuildNodeInfoForNode(t *testing.T) { }, { name: "everything together [forceDS=false]", - node: test.BuildTestNode("n", 1000, 10), + node: BuildTestNode("n", 1000, 10), pods: []*apiv1.Pod{ - test.BuildScheduledTestPod("p1", 100, 1, "n"), - test.BuildScheduledTestPod("p2", 100, 1, "n"), - test.SetMirrorPodSpec(test.BuildScheduledTestPod("p3", 100, 1, "n")), - setDeletionTimestamp(test.SetMirrorPodSpec(test.BuildScheduledTestPod("p4", 100, 1, "n"))), + BuildScheduledTestPod("p1", 100, 1, "n"), + BuildScheduledTestPod("p2", 100, 1, "n"), + SetMirrorPodSpec(BuildScheduledTestPod("p3", 100, 1, "n")), + setDeletionTimestamp(SetMirrorPodSpec(BuildScheduledTestPod("p4", 100, 1, "n"))), buildDSPod(ds1, "n"), setDeletionTimestamp(buildDSPod(ds2, "n")), }, daemonSets: []*appsv1.DaemonSet{ds1, ds2, ds3}, wantPods: []*apiv1.Pod{ - test.SetMirrorPodSpec(test.BuildScheduledTestPod("p3", 100, 1, "n")), + SetMirrorPodSpec(BuildScheduledTestPod("p3", 100, 1, "n")), buildDSPod(ds1, "n"), }, }, { name: "everything together [forceDS=true]", - node: test.BuildTestNode("n", 1000, 10), + node: BuildTestNode("n", 1000, 10), pods: []*apiv1.Pod{ - test.BuildScheduledTestPod("p1", 100, 1, "n"), - test.BuildScheduledTestPod("p2", 100, 1, "n"), - test.SetMirrorPodSpec(test.BuildScheduledTestPod("p3", 100, 1, "n")), - setDeletionTimestamp(test.SetMirrorPodSpec(test.BuildScheduledTestPod("p4", 100, 1, "n"))), + BuildScheduledTestPod("p1", 100, 1, "n"), + BuildScheduledTestPod("p2", 100, 1, "n"), + SetMirrorPodSpec(BuildScheduledTestPod("p3", 100, 1, "n")), + setDeletionTimestamp(SetMirrorPodSpec(BuildScheduledTestPod("p4", 100, 1, "n"))), buildDSPod(ds1, "n"), setDeletionTimestamp(buildDSPod(ds2, "n")), }, daemonSets: []*appsv1.DaemonSet{ds1, ds2, ds3}, forceDS: true, wantPods: []*apiv1.Pod{ - test.SetMirrorPodSpec(test.BuildScheduledTestPod("p3", 100, 1, "n")), + SetMirrorPodSpec(BuildScheduledTestPod("p3", 100, 1, "n")), buildDSPod(ds1, "n"), buildDSPod(ds2, "n"), }, @@ -194,13 +221,17 @@ func TestBuildNodeInfoForNode(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - nodeInfo, err := BuildNodeInfoForNode(tc.node, tc.pods, tc.daemonSets, tc.forceDS) + exampleNodeInfo := framework.NewNodeInfo(tc.node, nil) + for _, pod := range tc.pods { + exampleNodeInfo.AddPod(&framework.PodInfo{Pod: pod}) + } + nodeInfo, err := TemplateNodeInfoFromExampleNodeInfo(exampleNodeInfo, "nodeGroupId", tc.daemonSets, tc.forceDS, taints.TaintConfig{}) if tc.wantError { assert.Error(t, err) } else { assert.NoError(t, err) - assert.Equal(t, nodeInfo.Node(), tc.node) + assert.Equal(t, cleanNodeMetadata(nodeInfo.Node()), cleanNodeMetadata(tc.node)) // clean pod metadata for comparison purposes var wantPods, pods []*apiv1.Pod @@ -216,12 +247,32 @@ func TestBuildNodeInfoForNode(t *testing.T) { } } +func TestTemplateNodeInfoFromNodeGroupTemplate(t *testing.T) { + // TODO(DRA): Write. +} + +func TestFreshNodeInfoFromTemplateNodeInfo(t *testing.T) { + // TODO(DRA): Write. +} + +func TestDeepCopyNodeInfo(t *testing.T) { + // TODO(DRA): Write. +} + func cleanPodMetadata(pod *apiv1.Pod) *apiv1.Pod { pod.Name = strings.Split(pod.Name, "-")[0] + pod.UID = "" pod.OwnerReferences = nil + pod.Spec.NodeName = "" return pod } +func cleanNodeMetadata(node *apiv1.Node) *apiv1.Node { + node.UID = "" + node.Name = "" + return node +} + func buildDSPod(ds *appsv1.DaemonSet, nodeName string) *apiv1.Pod { pod := daemon.NewPod(ds, nodeName) pod.Name = ds.Name diff --git a/cluster-autoscaler/simulator/nodes.go b/cluster-autoscaler/simulator/nodes.go deleted file mode 100644 index b1c4cdfd4be6..000000000000 --- a/cluster-autoscaler/simulator/nodes.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package simulator - -import ( - appsv1 "k8s.io/api/apps/v1" - apiv1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" - "k8s.io/autoscaler/cluster-autoscaler/utils/daemonset" - "k8s.io/autoscaler/cluster-autoscaler/utils/errors" - - pod_util "k8s.io/autoscaler/cluster-autoscaler/utils/pod" -) - -// BuildNodeInfoForNode build a NodeInfo structure for the given node as if the node was just created. -func BuildNodeInfoForNode(node *apiv1.Node, scheduledPods []*apiv1.Pod, daemonsets []*appsv1.DaemonSet, forceDaemonSets bool) (*framework.NodeInfo, errors.AutoscalerError) { - nodeInfo := framework.NewNodeInfo(node, nil) - nodeInfo.SetNode(node) - return addExpectedPods(nodeInfo, scheduledPods, daemonsets, forceDaemonSets) -} - -func addExpectedPods(nodeInfo *framework.NodeInfo, scheduledPods []*apiv1.Pod, daemonsets []*appsv1.DaemonSet, forceDaemonSets bool) (*framework.NodeInfo, errors.AutoscalerError) { - runningDS := make(map[types.UID]bool) - for _, pod := range scheduledPods { - // Ignore scheduled pods in deletion phase - if pod.DeletionTimestamp != nil { - continue - } - // Add scheduled mirror and DS pods - if pod_util.IsMirrorPod(pod) || pod_util.IsDaemonSetPod(pod) { - nodeInfo.AddPod(&framework.PodInfo{Pod: pod}) - } - // Mark DS pods as running - controllerRef := metav1.GetControllerOf(pod) - if controllerRef != nil && controllerRef.Kind == "DaemonSet" { - runningDS[controllerRef.UID] = true - } - } - // Add all pending DS pods if force scheduling DS - if forceDaemonSets { - var pendingDS []*appsv1.DaemonSet - for _, ds := range daemonsets { - if !runningDS[ds.UID] { - pendingDS = append(pendingDS, ds) - } - } - daemonPods, err := daemonset.GetDaemonSetPodsForNode(nodeInfo, pendingDS) - if err != nil { - return nil, errors.ToAutoscalerError(errors.InternalError, err) - } - for _, pod := range daemonPods { - nodeInfo.AddPod(pod) - } - } - return nodeInfo, nil -} diff --git a/cluster-autoscaler/utils/scheduler/scheduler.go b/cluster-autoscaler/utils/scheduler/scheduler.go index 87be5ee2ece2..c63da3cbf437 100644 --- a/cluster-autoscaler/utils/scheduler/scheduler.go +++ b/cluster-autoscaler/utils/scheduler/scheduler.go @@ -23,7 +23,6 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/autoscaler/cluster-autoscaler/simulator/framework" scheduler_config "k8s.io/kubernetes/pkg/scheduler/apis/config" scheduler_scheme "k8s.io/kubernetes/pkg/scheduler/apis/config/scheme" @@ -79,27 +78,6 @@ func isHugePageResourceName(name apiv1.ResourceName) bool { return strings.HasPrefix(string(name), apiv1.ResourceHugePagesPrefix) } -// DeepCopyTemplateNode copies NodeInfo object used as a template. It changes -// names of UIDs of both node and pods running on it, so that copies can be used -// to represent multiple nodes. -func DeepCopyTemplateNode(nodeTemplate *framework.NodeInfo, suffix string) *framework.NodeInfo { - node := nodeTemplate.Node().DeepCopy() - node.Name = fmt.Sprintf("%s-%s", node.Name, suffix) - node.UID = uuid.NewUUID() - if node.Labels == nil { - node.Labels = make(map[string]string) - } - node.Labels["kubernetes.io/hostname"] = node.Name - nodeInfo := framework.NewNodeInfo(node, nil) - for _, podInfo := range nodeTemplate.Pods { - pod := podInfo.Pod.DeepCopy() - pod.Name = fmt.Sprintf("%s-%s", podInfo.Pod.Name, suffix) - pod.UID = uuid.NewUUID() - nodeInfo.AddPod(&framework.PodInfo{Pod: pod}) - } - return nodeInfo -} - // ResourceToResourceList returns a resource list of the resource. func ResourceToResourceList(r *schedulerframework.Resource) apiv1.ResourceList { result := apiv1.ResourceList{