diff --git a/pkg/scheduler/preemption/preemption.go b/pkg/scheduler/preemption/preemption.go index 0c3c00e4e9..44ee5b43bc 100644 --- a/pkg/scheduler/preemption/preemption.go +++ b/pkg/scheduler/preemption/preemption.go @@ -333,12 +333,20 @@ func workloadFits(wlReq cache.FlavorResourceQuantities, cq *cache.ClusterQueue, cohortResRequestable = cq.Cohort.RequestableResources[flvQuotas.Name] } for rName, rReq := range flvReq { - limit := flvQuotas.Resources[rName].Nominal - if flvQuotas.Resources[rName].BorrowingLimit != nil && allowBorrowing { - limit += *flvQuotas.Resources[rName].BorrowingLimit - } - if cqResUsage[rName]+rReq > limit { - return false + resource := flvQuotas.Resources[rName] + + if cq.Cohort == nil || !allowBorrowing { + if cqResUsage[rName]+rReq > resource.Nominal { + return false + } + } else { + // When resource.BorrowingLimit == nil there is no borrowing + // limit, so we can skip the check. + if resource.BorrowingLimit != nil { + if cqResUsage[rName]+rReq > resource.Nominal+*resource.BorrowingLimit { + return false + } + } } if cq.Cohort != nil && cohortResUsage[rName]+rReq > cohortResRequestable[rName] { return false diff --git a/pkg/scheduler/preemption/preemption_test.go b/pkg/scheduler/preemption/preemption_test.go index a0ef325b4f..5f5aac2bb4 100644 --- a/pkg/scheduler/preemption/preemption_test.go +++ b/pkg/scheduler/preemption/preemption_test.go @@ -101,6 +101,30 @@ func TestPreemption(t *testing.T) { ReclaimWithinCohort: kueue.PreemptionPolicyAny, }). Obj(), + utiltesting.MakeClusterQueue("d1"). + Cohort("cohort-no-limits"). + ResourceGroup(*utiltesting.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "6"). + Resource(corev1.ResourceMemory, "3Gi"). + Obj(), + ). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyLowerPriority, + ReclaimWithinCohort: kueue.PreemptionPolicyLowerPriority, + }). + Obj(), + utiltesting.MakeClusterQueue("d2"). + Cohort("cohort-no-limits"). + ResourceGroup(*utiltesting.MakeFlavorQuotas("default"). + Resource(corev1.ResourceCPU, "6"). + Resource(corev1.ResourceMemory, "3Gi"). + Obj(), + ). + Preemption(kueue.ClusterQueuePreemption{ + WithinClusterQueue: kueue.PreemptionPolicyNever, + ReclaimWithinCohort: kueue.PreemptionPolicyAny, + }). + Obj(), utiltesting.MakeClusterQueue("l1"). Cohort("legion"). ResourceGroup(*utiltesting.MakeFlavorQuotas("default"). @@ -485,6 +509,37 @@ func TestPreemption(t *testing.T) { }), wantPreempted: sets.New("/c1-low"), }, + "preempting locally and borrowing same resource in cohort; no borrowing limit in the cohort": { + admitted: []kueue.Workload{ + *utiltesting.MakeWorkload("d1-med", ""). + Priority(0). + Request(corev1.ResourceCPU, "4"). + ReserveQuota(utiltesting.MakeAdmission("d1").Assignment(corev1.ResourceCPU, "default", "4").Obj()). + Obj(), + *utiltesting.MakeWorkload("d1-low", ""). + Priority(-1). + Request(corev1.ResourceCPU, "4"). + ReserveQuota(utiltesting.MakeAdmission("d1").Assignment(corev1.ResourceCPU, "default", "4").Obj()). + Obj(), + *utiltesting.MakeWorkload("d2-low-1", ""). + Priority(-1). + Request(corev1.ResourceCPU, "4"). + ReserveQuota(utiltesting.MakeAdmission("d2").Assignment(corev1.ResourceCPU, "default", "4").Obj()). + Obj(), + }, + incoming: utiltesting.MakeWorkload("in", ""). + Priority(1). + Request(corev1.ResourceCPU, "4"). + Obj(), + targetCQ: "d1", + assignment: singlePodSetAssignment(flavorassigner.ResourceAssignment{ + corev1.ResourceCPU: &flavorassigner.FlavorAssignment{ + Name: "default", + Mode: flavorassigner.Preempt, + }, + }), + wantPreempted: sets.New("/d1-low"), + }, "preempting locally and borrowing other resources in cohort, with cohort candidates": { admitted: []kueue.Workload{ *utiltesting.MakeWorkload("c1-med", "").