From 38c727b39701be6079e5c7cc4615aeb5e099533a Mon Sep 17 00:00:00 2001
From: Antonin Stefanutti <antonin@stefanutti.fr>
Date: Wed, 31 Jan 2024 15:24:05 +0100
Subject: [PATCH] Make sure pending workloads are removed from the cache

---
 pkg/controller/core/workload_controller.go    |  5 +-
 .../core/clusterqueue_controller_test.go      | 57 +++++++++++++++++++
 2 files changed, 59 insertions(+), 3 deletions(-)

diff --git a/pkg/controller/core/workload_controller.go b/pkg/controller/core/workload_controller.go
index f8425838d7..7e9eefaac2 100644
--- a/pkg/controller/core/workload_controller.go
+++ b/pkg/controller/core/workload_controller.go
@@ -403,9 +403,8 @@ func (r *WorkloadReconciler) Delete(e event.DeleteEvent) bool {
 
 	// Even if the state is unknown, the last cached state tells us whether the
 	// workload was in the queues and should be cleared from them.
-	if workload.HasQuotaReservation(wl) {
-		r.queues.DeleteWorkload(wl)
-	}
+	r.queues.DeleteWorkload(wl)
+
 	return true
 }
 
diff --git a/test/integration/controller/core/clusterqueue_controller_test.go b/test/integration/controller/core/clusterqueue_controller_test.go
index d7a215f7ec..b2d1d11532 100644
--- a/test/integration/controller/core/clusterqueue_controller_test.go
+++ b/test/integration/controller/core/clusterqueue_controller_test.go
@@ -362,6 +362,63 @@ var _ = ginkgo.Describe("ClusterQueue controller", ginkgo.Ordered, ginkgo.Contin
 			util.ExpectReservingActiveWorkloadsMetric(clusterQueue, 0)
 		})
 
+		ginkgo.It("Should update status and report metrics when a pending workload is deleted", func() {
+			workload := testing.MakeWorkload("one", ns.Name).Queue(localQueue.Name).
+				Request(corev1.ResourceCPU, "5").Obj()
+
+			ginkgo.By("Creating a workload", func() {
+				gomega.Expect(k8sClient.Create(ctx, workload)).To(gomega.Succeed())
+			})
+
+			// Pending workloads count is incremented as the workload is inadmissible
+			// because ResourceFlavors don't exist.
+			gomega.Eventually(func() kueue.ClusterQueueStatus {
+				var updatedCq kueue.ClusterQueue
+				gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), &updatedCq)).To(gomega.Succeed())
+				return updatedCq.Status
+			}, util.Timeout, util.Interval).Should(gomega.BeComparableTo(kueue.ClusterQueueStatus{
+				PendingWorkloads:   1,
+				FlavorsReservation: emptyUsedFlavors,
+				FlavorsUsage:       emptyUsedFlavors,
+				Conditions: []metav1.Condition{
+					{
+						Type:    kueue.ClusterQueueActive,
+						Status:  metav1.ConditionFalse,
+						Reason:  "FlavorNotFound",
+						Message: "Can't admit new workloads: FlavorNotFound",
+					},
+				},
+			}, ignoreConditionTimestamps, ignorePendingWorkloadsStatus))
+
+			util.ExpectPendingWorkloadsMetric(clusterQueue, 0, 1)
+			util.ExpectReservingActiveWorkloadsMetric(clusterQueue, 0)
+
+			ginkgo.By("Deleting the pending workload", func() {
+				gomega.Expect(k8sClient.Delete(ctx, workload)).To(gomega.Succeed())
+			})
+
+			// Pending workloads count is decrement as the deleted workload has been removed from the queue.
+			gomega.Eventually(func() kueue.ClusterQueueStatus {
+				var updatedCq kueue.ClusterQueue
+				gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(clusterQueue), &updatedCq)).To(gomega.Succeed())
+				return updatedCq.Status
+			}, util.Timeout, util.Interval).Should(gomega.BeComparableTo(kueue.ClusterQueueStatus{
+				PendingWorkloads:   0,
+				FlavorsReservation: emptyUsedFlavors,
+				FlavorsUsage:       emptyUsedFlavors,
+				Conditions: []metav1.Condition{
+					{
+						Type:    kueue.ClusterQueueActive,
+						Status:  metav1.ConditionFalse,
+						Reason:  "FlavorNotFound",
+						Message: "Can't admit new workloads: FlavorNotFound",
+					},
+				},
+			}, ignoreConditionTimestamps, ignorePendingWorkloadsStatus))
+			util.ExpectPendingWorkloadsMetric(clusterQueue, 0, 0)
+			util.ExpectReservingActiveWorkloadsMetric(clusterQueue, 0)
+		})
+
 		ginkgo.It("Should update status when workloads have reclaimable pods", func() {
 
 			ginkgo.By("Creating ResourceFlavors", func() {