fix(clusterstate): invalidate instance cache when scaling down

cluster-autoscaler/cloudprovider/kwok/kwok_provider.go

@@ -71,7 +71,20 @@ func (kwok *KwokCloudProvider) NodeGroupForNode(node *apiv1.Node) (cloudprovider
 // HasInstance returns whether a given node has a corresponding instance in this cloud provider
 // Since there is no underlying cloud provider instance, return true
 func (kwok *KwokCloudProvider) HasInstance(node *apiv1.Node) (bool, error) {
-	return true, nil
+	for _, nodeGroup := range kwok.nodeGroups {
+		nodeList, err := nodeGroup.lister.List()
+		if err != nil {
+			return false, err
+		}
+
+		for _, no := range nodeList {
+			if no.GetName() == node.GetName() {
+				return true, nil
+			}
+		}
+	}
+
+	return false, nil
 }
 
 // Pricing returns pricing model for this cloud provider or error if not available.

cluster-autoscaler/clusterstate/clusterstate.go

@@ -65,8 +65,8 @@ type ScaleUpRequest struct {
 
 // ScaleDownRequest contains information about the requested node deletion.
 type ScaleDownRequest struct {
-	// NodeName is the name of the node to be deleted.
-	NodeName string
+	// Node is the node to be deleted.
+	Node *apiv1.Node
 	// NodeGroup is the node group of the deleted node.
 	NodeGroup cloudprovider.NodeGroup
 	// Time is the time when the node deletion was requested.
@@ -283,6 +283,12 @@ func (csr *ClusterStateRegistry) updateScaleRequests(currentTime time.Time) {
 
 	newScaleDownRequests := make([]*ScaleDownRequest, 0)
 	for _, scaleDownRequest := range csr.scaleDownRequests {
+		// delete scaleDownRequest if there's no instance in cloud provider side
+		// otherwise we check the delete time
+		hasInstance, err := csr.cloudProvider.HasInstance(scaleDownRequest.Node)
+		if err == nil && !hasInstance {
+			continue
+		}
 		if scaleDownRequest.ExpectedDeleteTime.After(currentTime) {
 			newScaleDownRequests = append(newScaleDownRequests, scaleDownRequest)
 		}
@@ -501,6 +507,16 @@ func (csr *ClusterStateRegistry) IsNodeGroupScalingUp(nodeGroupName string) bool
 	return found
 }
 
+// IsNodeGroupScalingDown returns true if the node group is currently scaling down.
+func (csr *ClusterStateRegistry) IsNodeGroupScalingDown(nodeGroupName string) bool {
+	for _, scaleDownRequest := range csr.scaleDownRequests {
+		if scaleDownRequest.NodeGroup.Id() == nodeGroupName {
+			return true
+		}
+	}
+	return false
+}
+
 // HasNodeGroupStartedScaleUp returns true if the node group has started scale up regardless
 // of whether there are any upcoming nodes. This is useful in the case when the node group's
 // size reverts back to its previous size before the next UpdatesCall and we want to know
@@ -991,7 +1007,7 @@ func (csr *ClusterStateRegistry) GetUpcomingNodes() (upcomingCounts map[string]i
 // as returned by NodeGroup.Nodes().
 func (csr *ClusterStateRegistry) getCloudProviderNodeInstances() (map[string][]cloudprovider.Instance, error) {
 	for _, nodeGroup := range csr.cloudProvider.NodeGroups() {
-		if csr.IsNodeGroupScalingUp(nodeGroup.Id()) {
+		if csr.IsNodeGroupScalingUp(nodeGroup.Id()) || csr.IsNodeGroupScalingDown(nodeGroup.Id()) {
 			csr.cloudProviderNodeInstancesCache.InvalidateCacheEntry(nodeGroup)
 		}
 	}

cluster-autoscaler/clusterstate/clusterstate_test.go

@@ -498,7 +498,7 @@ func TestRegisterScaleDown(t *testing.T) {
 
 	clusterstate.RegisterScaleDown(&ScaleDownRequest{
 		NodeGroup:          provider.GetNodeGroup("ng1"),
-		NodeName:           "ng1-1",
+		Node:               ng1_1,
 		ExpectedDeleteTime: now.Add(time.Minute),
 		Time:               now,
 	})

cluster-autoscaler/core/scaledown/actuation/delete_in_batch.go

@@ -197,7 +197,7 @@ func RegisterAndRecordSuccessfulScaleDownEvent(ctx *context.AutoscalingContext,
 	ctx.Recorder.Eventf(node, apiv1.EventTypeNormal, "ScaleDown", "nodes removed by cluster autoscaler")
 	csr.RegisterScaleDown(&clusterstate.ScaleDownRequest{
 		NodeGroup:          nodeGroup,
-		NodeName:           node.Name,
+		Node:               node,
 		Time:               time.Now(),
 		ExpectedDeleteTime: time.Now().Add(MaxCloudProviderNodeDeletionTime),
 	})