From 4c03cb771fc183dcdc24a97eaf7df43a3f74c335 Mon Sep 17 00:00:00 2001 From: Jason Deal Date: Thu, 13 Jun 2024 17:34:26 -0700 Subject: [PATCH] test: update node termination model --- .../node/termination/suite_test.go | 227 ++++++++++++------ 1 file changed, 154 insertions(+), 73 deletions(-) diff --git a/pkg/controllers/node/termination/suite_test.go b/pkg/controllers/node/termination/suite_test.go index d26b0a58e6..3bb5f806b2 100644 --- a/pkg/controllers/node/termination/suite_test.go +++ b/pkg/controllers/node/termination/suite_test.go @@ -27,6 +27,7 @@ import ( "github.com/samber/lo" clock "k8s.io/utils/clock/testing" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/karpenter/pkg/apis" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" @@ -38,6 +39,7 @@ import ( "sigs.k8s.io/karpenter/pkg/test" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -99,25 +101,37 @@ var _ = Describe("Termination", func() { Context("Reconciliation", func() { It("should delete nodes", func() { - ExpectApplied(ctx, env.Client, node) + ExpectApplied(ctx, env.Client, node, nodeClaim) Expect(env.Client.Delete(ctx, node)).To(Succeed()) node = ExpectNodeExists(ctx, env.Client, node.Name) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - ExpectNotFound(ctx, env.Client, node) + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should delete nodeclaims associated with nodes", func() { ExpectApplied(ctx, env.Client, node, nodeClaim) Expect(env.Client.Delete(ctx, node)).To(Succeed()) node = ExpectNodeExists(ctx, env.Client, node.Name) + + // The first reconciliation should trigger the Delete, and set the terminating status condition ExpectObjectReconciled(ctx, env.Client, terminationController, node) - ExpectExists(ctx, env.Client, nodeClaim) + nc := ExpectExists(ctx, env.Client, nodeClaim) + Expect(nc.StatusConditions().Get(v1beta1.ConditionTypeTerminating).IsTrue()).To(BeTrue()) + ExpectNodeExists(ctx, env.Client, node.Name) + + // The second reconciliation should call get, see the "instance" is terminated, and remove the node. + // We should have deleted the NodeClaim from the node termination controller, so removing it's finalizer should result in it being removed. ExpectObjectReconciled(ctx, env.Client, terminationController, node) ExpectFinalizersRemoved(ctx, env.Client, nodeClaim) ExpectNotFound(ctx, env.Client, node, nodeClaim) }) It("should not race if deleting nodes in parallel", func() { + const numNodes = 10 var nodes []*v1.Node - for i := 0; i < 10; i++ { + for i := 0; i < numNodes; i++ { node = test.Node(test.NodeOptions{ ObjectMeta: metav1.ObjectMeta{ Finalizers: []string{v1beta1.TerminationFinalizer}, @@ -129,34 +143,43 @@ var _ = Describe("Termination", func() { nodes = append(nodes, node) } - var wg sync.WaitGroup - // this is enough to trip the race detector - for i := 0; i < 10; i++ { - wg.Add(1) - go func(node *v1.Node) { - defer GinkgoRecover() - defer wg.Done() - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - }(nodes[i]) - } - wg.Wait() - ExpectNotFound(ctx, env.Client, lo.Map(nodes, func(n *v1.Node, _ int) client.Object { return n })...) + Eventually(func(g Gomega) { + var wg sync.WaitGroup + // this is enough to trip the race detector + for i := 0; i < numNodes; i++ { + wg.Add(1) + go func(node *v1.Node) { + defer GinkgoRecover() + defer wg.Done() + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + }(nodes[i]) + } + wg.Wait() + for _, n := range nodes { + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(n), tmpNode))).To(BeTrue()) + } + ExpectNotFound(ctx, env.Client, lo.Map(nodes, func(n *v1.Node, _ int) client.Object { return n })...) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should exclude nodes from load balancers when terminating", func() { - // This is a kludge to prevent the node from being deleted before we can - // inspect its labels - podNoEvict := test.Pod(test.PodOptions{ - NodeName: node.Name, + labels := map[string]string{"foo": "bar"} + pod := test.Pod(test.PodOptions{ ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{v1beta1.DoNotDisruptAnnotationKey: "true"}, OwnerReferences: defaultOwnerRefs, + Labels: labels, }, }) + // Create a fully blocking PDB to prevent the node from being deleted before we can observe its labels + pdb := test.PodDisruptionBudget(test.PDBOptions{ + Labels: labels, + MaxUnavailable: lo.ToPtr(intstr.FromInt(0)), + }) - ExpectApplied(ctx, env.Client, node, podNoEvict) - + ExpectApplied(ctx, env.Client, node, nodeClaim, pod, pdb) + ExpectManualBinding(ctx, env.Client, pod, node) Expect(env.Client.Delete(ctx, node)).To(Succeed()) - node = ExpectNodeExists(ctx, env.Client, node.Name) ExpectObjectReconciled(ctx, env.Client, terminationController, node) node = ExpectNodeExists(ctx, env.Client, node.Name) Expect(node.Labels[v1.LabelNodeExcludeBalancers]).Should(Equal("karpenter")) @@ -168,7 +191,7 @@ var _ = Describe("Termination", func() { Tolerations: []v1.Toleration{{Key: v1beta1.DisruptionTaintKey, Operator: v1.TolerationOpEqual, Effect: v1beta1.DisruptionNoScheduleTaint.Effect, Value: v1beta1.DisruptionNoScheduleTaint.Value}}, ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}, }) - ExpectApplied(ctx, env.Client, node, podEvict, podSkip) + ExpectApplied(ctx, env.Client, node, nodeClaim, podEvict, podSkip) // Trigger Termination Controller Expect(env.Client.Delete(ctx, node)).To(Succeed()) @@ -186,8 +209,12 @@ var _ = Describe("Termination", func() { // Reconcile to delete node node = ExpectNodeExists(ctx, env.Client, node.Name) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - ExpectNotFound(ctx, env.Client, node) + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should not evict pods that tolerate karpenter disruption taint with exists operator", func() { podEvict := test.Pod(test.PodOptions{NodeName: node.Name, ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}}) @@ -196,7 +223,7 @@ var _ = Describe("Termination", func() { Tolerations: []v1.Toleration{{Key: v1beta1.DisruptionTaintKey, Operator: v1.TolerationOpExists, Effect: v1beta1.DisruptionNoScheduleTaint.Effect}}, ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}, }) - ExpectApplied(ctx, env.Client, node, podEvict, podSkip) + ExpectApplied(ctx, env.Client, node, nodeClaim, podEvict, podSkip) // Trigger Termination Controller Expect(env.Client.Delete(ctx, node)).To(Succeed()) @@ -216,8 +243,12 @@ var _ = Describe("Termination", func() { // Reconcile to delete node node = ExpectNodeExists(ctx, env.Client, node.Name) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - ExpectNotFound(ctx, env.Client, node) + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should evict pods that tolerate the node.kubernetes.io/unschedulable taint", func() { podEvict := test.Pod(test.PodOptions{ @@ -225,7 +256,7 @@ var _ = Describe("Termination", func() { Tolerations: []v1.Toleration{{Key: v1.TaintNodeUnschedulable, Operator: v1.TolerationOpExists, Effect: v1.TaintEffectNoSchedule}}, ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}, }) - ExpectApplied(ctx, env.Client, node, podEvict) + ExpectApplied(ctx, env.Client, node, nodeClaim, podEvict) // Trigger Termination Controller Expect(env.Client.Delete(ctx, node)).To(Succeed()) @@ -242,8 +273,12 @@ var _ = Describe("Termination", func() { // Reconcile to delete node node = ExpectNodeExists(ctx, env.Client, node.Name) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - ExpectNotFound(ctx, env.Client, node) + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should delete nodes that have pods without an ownerRef", func() { pod := test.Pod(test.PodOptions{ @@ -253,7 +288,7 @@ var _ = Describe("Termination", func() { }, }) - ExpectApplied(ctx, env.Client, node, pod) + ExpectApplied(ctx, env.Client, node, nodeClaim, pod) Expect(env.Client.Delete(ctx, node)).To(Succeed()) node = ExpectNodeExists(ctx, env.Client, node.Name) ExpectObjectReconciled(ctx, env.Client, terminationController, node) @@ -268,12 +303,14 @@ var _ = Describe("Termination", func() { // Delete no owner refs pod to simulate successful eviction ExpectDeleted(ctx, env.Client, pod) - // Reconcile node to evict pod + // Reconcile node to evict pod and delete node node = ExpectNodeExists(ctx, env.Client, node.Name) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - - // Reconcile to delete node - ExpectNotFound(ctx, env.Client, node) + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should delete nodes with terminal pods", func() { podEvictPhaseSucceeded := test.Pod(test.PodOptions{ @@ -285,12 +322,16 @@ var _ = Describe("Termination", func() { Phase: v1.PodFailed, }) - ExpectApplied(ctx, env.Client, node, podEvictPhaseSucceeded, podEvictPhaseFailed) + ExpectApplied(ctx, env.Client, node, nodeClaim, podEvictPhaseSucceeded, podEvictPhaseFailed) Expect(env.Client.Delete(ctx, node)).To(Succeed()) node = ExpectNodeExists(ctx, env.Client, node.Name) // Trigger Termination Controller, which should ignore these pods and delete the node - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - ExpectNotFound(ctx, env.Client, node) + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should fail to evict pods that violate a PDB", func() { minAvailable := intstr.FromInt32(1) @@ -309,7 +350,7 @@ var _ = Describe("Termination", func() { Phase: v1.PodRunning, }) - ExpectApplied(ctx, env.Client, node, podNoEvict, pdb) + ExpectApplied(ctx, env.Client, node, nodeClaim, podNoEvict, pdb) // Trigger Termination Controller Expect(env.Client.Delete(ctx, node)).To(Succeed()) @@ -331,14 +372,18 @@ var _ = Describe("Termination", func() { // Reconcile to delete node node = ExpectNodeExists(ctx, env.Client, node.Name) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - ExpectNotFound(ctx, env.Client, node) + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should evict pods in order", func() { daemonEvict := test.DaemonSet() daemonNodeCritical := test.DaemonSet(test.DaemonSetOptions{PodOptions: test.PodOptions{PriorityClassName: "system-node-critical"}}) daemonClusterCritical := test.DaemonSet(test.DaemonSetOptions{PodOptions: test.PodOptions{PriorityClassName: "system-cluster-critical"}}) - ExpectApplied(ctx, env.Client, node, daemonEvict, daemonNodeCritical, daemonClusterCritical) + ExpectApplied(ctx, env.Client, node, nodeClaim, daemonEvict, daemonNodeCritical, daemonClusterCritical) podEvict := test.Pod(test.PodOptions{NodeName: node.Name, ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}}) podDaemonEvict := test.Pod(test.PodOptions{NodeName: node.Name, ObjectMeta: metav1.ObjectMeta{OwnerReferences: []metav1.OwnerReference{{ @@ -368,7 +413,7 @@ var _ = Describe("Termination", func() { BlockOwnerDeletion: lo.ToPtr(true), }}}}) - ExpectApplied(ctx, env.Client, node, podEvict, podNodeCritical, podClusterCritical, podDaemonEvict, podDaemonNodeCritical, podDaemonClusterCritical) + ExpectApplied(ctx, env.Client, node, nodeClaim, nodeClaim, podEvict, podNodeCritical, podClusterCritical, podDaemonEvict, podDaemonNodeCritical, podDaemonClusterCritical) // Trigger Termination Controller Expect(env.Client.Delete(ctx, node)).To(Succeed()) @@ -417,15 +462,19 @@ var _ = Describe("Termination", func() { // Reconcile to delete node node = ExpectNodeExists(ctx, env.Client, node.Name) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - ExpectNotFound(ctx, env.Client, node) + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should evict non-critical pods first", func() { podEvict := test.Pod(test.PodOptions{NodeName: node.Name, ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}}) podNodeCritical := test.Pod(test.PodOptions{NodeName: node.Name, PriorityClassName: "system-node-critical", ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}}) podClusterCritical := test.Pod(test.PodOptions{NodeName: node.Name, PriorityClassName: "system-cluster-critical", ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}}) - ExpectApplied(ctx, env.Client, node, podEvict, podNodeCritical, podClusterCritical) + ExpectApplied(ctx, env.Client, node, nodeClaim, podEvict, podNodeCritical, podClusterCritical) // Trigger Termination Controller Expect(env.Client.Delete(ctx, node)).To(Succeed()) @@ -452,13 +501,16 @@ var _ = Describe("Termination", func() { // Reconcile to delete node node = ExpectNodeExists(ctx, env.Client, node.Name) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - ExpectNotFound(ctx, env.Client, node) + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should not evict static pods", func() { - ExpectApplied(ctx, env.Client, node) podEvict := test.Pod(test.PodOptions{NodeName: node.Name, ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}}) - ExpectApplied(ctx, env.Client, node, podEvict) + ExpectApplied(ctx, env.Client, node, nodeClaim, podEvict) podNoEvict := test.Pod(test.PodOptions{ NodeName: node.Name, @@ -496,13 +548,16 @@ var _ = Describe("Termination", func() { // Reconcile to delete node node = ExpectNodeExists(ctx, env.Client, node.Name) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - ExpectNotFound(ctx, env.Client, node) - + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should not delete nodes until all pods are deleted", func() { pods := test.Pods(2, test.PodOptions{NodeName: node.Name, ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}}) - ExpectApplied(ctx, env.Client, node, pods[0], pods[1]) + ExpectApplied(ctx, env.Client, node, nodeClaim, pods[0], pods[1]) // Trigger Termination Controller Expect(env.Client.Delete(ctx, node)).To(Succeed()) @@ -530,12 +585,16 @@ var _ = Describe("Termination", func() { // Reconcile to delete node node = ExpectNodeExists(ctx, env.Client, node.Name) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - ExpectNotFound(ctx, env.Client, node) + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should delete nodes with no underlying instance even if not fully drained", func() { pods := test.Pods(2, test.PodOptions{NodeName: node.Name, ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}}) - ExpectApplied(ctx, env.Client, node, pods[0], pods[1]) + ExpectApplied(ctx, env.Client, node, nodeClaim, pods[0], pods[1]) // Make Node NotReady since it's automatically marked as Ready on first deploy ExpectMakeNodesNotReady(ctx, env.Client, node) @@ -563,12 +622,16 @@ var _ = Describe("Termination", func() { // Reconcile to delete node node = ExpectNodeExists(ctx, env.Client, node.Name) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - ExpectNotFound(ctx, env.Client, node) + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should not delete nodes with no underlying instance if the node is still Ready", func() { pods := test.Pods(2, test.PodOptions{NodeName: node.Name, ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}}) - ExpectApplied(ctx, env.Client, node, pods[0], pods[1]) + ExpectApplied(ctx, env.Client, node, nodeClaim, pods[0], pods[1]) // Trigger Termination Controller Expect(env.Client.Delete(ctx, node)).To(Succeed()) @@ -594,13 +657,17 @@ var _ = Describe("Termination", func() { // Reconcile to try to delete the node, but don't succeed because the readiness condition // of the node still won't let us delete it node = ExpectNodeExists(ctx, env.Client, node.Name) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - ExpectNodeExists(ctx, env.Client, node.Name) + Consistently(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeFalse()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should wait for pods to terminate", func() { pod := test.Pod(test.PodOptions{NodeName: node.Name, ObjectMeta: metav1.ObjectMeta{OwnerReferences: defaultOwnerRefs}}) fakeClock.SetTime(time.Now()) // make our fake clock match the pod creation time - ExpectApplied(ctx, env.Client, node, pod) + ExpectApplied(ctx, env.Client, node, nodeClaim, pod) // Before grace period, node should not delete Expect(env.Client.Delete(ctx, node)).To(Succeed()) @@ -613,8 +680,12 @@ var _ = Describe("Termination", func() { // to eliminate test-flakiness we reset the time to current time + 90 seconds instead of just advancing // the clock by 90 seconds. fakeClock.SetTime(time.Now().Add(90 * time.Second)) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) - ExpectNotFound(ctx, env.Client, node) + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) }) It("should not evict a new pod with the same name using the old pod's eviction queue key", func() { pod := test.Pod(test.PodOptions{ @@ -624,7 +695,7 @@ var _ = Describe("Termination", func() { OwnerReferences: defaultOwnerRefs, }, }) - ExpectApplied(ctx, env.Client, node, pod) + ExpectApplied(ctx, env.Client, node, nodeClaim, pod) // Trigger Termination Controller Expect(env.Client.Delete(ctx, node)).To(Succeed()) @@ -675,7 +746,12 @@ var _ = Describe("Termination", func() { ExpectApplied(ctx, env.Client, node) Expect(env.Client.Delete(ctx, node)).To(Succeed()) node = ExpectNodeExists(ctx, env.Client, node.Name) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) m, ok := FindMetricWithLabelValues("karpenter_nodes_termination_time_seconds", map[string]string{"nodepool": node.Labels[v1beta1.NodePoolLabelKey]}) Expect(ok).To(BeTrue()) @@ -685,7 +761,12 @@ var _ = Describe("Termination", func() { ExpectApplied(ctx, env.Client, node) Expect(env.Client.Delete(ctx, node)).To(Succeed()) node = ExpectNodeExists(ctx, env.Client, node.Name) - ExpectObjectReconciled(ctx, env.Client, terminationController, node) + Eventually(func(g Gomega) { + _, err := reconcile.AsReconciler(env.Client, terminationController).Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(node)}) + g.Expect(err).ToNot(HaveOccurred()) + tmpNode := &v1.Node{} + g.Expect(errors.IsNotFound(env.Client.Get(ctx, client.ObjectKeyFromObject(node), tmpNode))).To(BeTrue()) + }, ReconcilerPropagationTime, RequestInterval).Should(Succeed()) m, ok := FindMetricWithLabelValues("karpenter_nodes_terminated", map[string]string{"nodepool": node.Labels[v1beta1.NodePoolLabelKey]}) Expect(ok).To(BeTrue())