Merge pull request #1030 from k8s-infra-cherrypick-robot/cherry-pick-…

…1024-to-release-0.4 [release-0.4] Consider priority across ClusterQueues in a cohort
kubernetes-sigs · Aug 1, 2023 · 1c326de · 1c326de
2 parents 8730018 + 0703a40
commit 1c326de
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 11 deletions.
diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go
@@ -45,6 +45,7 @@ import (
 	"sigs.k8s.io/kueue/pkg/scheduler/preemption"
 	"sigs.k8s.io/kueue/pkg/util/api"
 	"sigs.k8s.io/kueue/pkg/util/limitrange"
+	"sigs.k8s.io/kueue/pkg/util/priority"
 	"sigs.k8s.io/kueue/pkg/util/resource"
 	"sigs.k8s.io/kueue/pkg/util/routine"
 	"sigs.k8s.io/kueue/pkg/workload"
@@ -430,17 +431,27 @@ func (e entryOrdering) Swap(i, j int) {
 }
 
 // Less is the ordering criteria:
-// 1. request under min quota before borrowing.
-// 2. FIFO on creation timestamp.
+// 1. request under nominal quota before borrowing.
+// 2. higher priority first.
+// 3. FIFO on eviction or creation timestamp.
 func (e entryOrdering) Less(i, j int) bool {
 	a := e[i]
 	b := e[j]
-	// 1. Request under min quota.
+
+	// 1. Request under nominal quota.
 	aBorrows := a.assignment.Borrows()
 	bBorrows := b.assignment.Borrows()
 	if aBorrows != bBorrows {
 		return !aBorrows
 	}
+
+	// 2. Higher priority first.
+	p1 := priority.Priority(a.Obj)
+	p2 := priority.Priority(b.Obj)
+	if p1 != p2 {
+		return p1 > p2
+	}
+
 	// 2. FIFO.
 	aComparisonTimestamp := workload.GetQueueOrderTimestamp(a.Obj)
 	bComparisonTimestamp := workload.GetQueueOrderTimestamp(b.Obj)

diff --git a/pkg/scheduler/scheduler_test.go b/pkg/scheduler/scheduler_test.go
@@ -985,7 +985,7 @@ func TestEntryOrdering(t *testing.T) {
 		{
 			Info: workload.Info{
 				Obj: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{
-					Name:              "alpha",
+					Name:              "old_borrowing",
 					CreationTimestamp: metav1.NewTime(now),
 				}},
 			},
@@ -998,23 +998,48 @@ func TestEntryOrdering(t *testing.T) {
 		{
 			Info: workload.Info{
 				Obj: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{
-					Name:              "beta",
+					Name:              "old",
 					CreationTimestamp: metav1.NewTime(now.Add(time.Second)),
 				}},
 			},
 		},
 		{
 			Info: workload.Info{
 				Obj: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{
-					Name:              "gamma",
+					Name:              "new",
 					CreationTimestamp: metav1.NewTime(now.Add(3 * time.Second)),
 				}},
 			},
 		},
 		{
 			Info: workload.Info{
 				Obj: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{
-					Name:              "delta",
+					Name:              "high_pri_borrowing",
+					CreationTimestamp: metav1.NewTime(now.Add(3 * time.Second)),
+				}, Spec: kueue.WorkloadSpec{
+					Priority: pointer.Int32(1),
+				}},
+			},
+			assignment: flavorassigner.Assignment{
+				TotalBorrow: cache.FlavorResourceQuantities{
+					"flavor": {},
+				},
+			},
+		},
+		{
+			Info: workload.Info{
+				Obj: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{
+					Name:              "new_high_pri",
+					CreationTimestamp: metav1.NewTime(now.Add(3 * time.Second)),
+				}, Spec: kueue.WorkloadSpec{
+					Priority: pointer.Int32(1),
+				}},
+			},
+		},
+		{
+			Info: workload.Info{
+				Obj: &kueue.Workload{ObjectMeta: metav1.ObjectMeta{
+					Name:              "new_borrowing",
 					CreationTimestamp: metav1.NewTime(now.Add(3 * time.Second)),
 				}},
 			},
@@ -1028,7 +1053,7 @@ func TestEntryOrdering(t *testing.T) {
 			Info: workload.Info{
 				Obj: &kueue.Workload{
 					ObjectMeta: metav1.ObjectMeta{
-						Name:              "epsilon",
+						Name:              "evicted_borrowing",
 						CreationTimestamp: metav1.NewTime(now),
 					},
 					Status: kueue.WorkloadStatus{
@@ -1053,8 +1078,8 @@ func TestEntryOrdering(t *testing.T) {
 			Info: workload.Info{
 				Obj: &kueue.Workload{
 					ObjectMeta: metav1.ObjectMeta{
-						Name:              "zeta",
-						CreationTimestamp: metav1.NewTime(now.Add(2 * time.Second)),
+						Name:              "recently_evicted",
+						CreationTimestamp: metav1.NewTime(now),
 					},
 					Status: kueue.WorkloadStatus{
 						Conditions: []metav1.Condition{
@@ -1075,7 +1100,7 @@ func TestEntryOrdering(t *testing.T) {
 	for i, e := range input {
 		order[i] = e.Obj.Name
 	}
-	wantOrder := []string{"beta", "zeta", "gamma", "alpha", "epsilon", "delta"}
+	wantOrder := []string{"new_high_pri", "old", "recently_evicted", "new", "high_pri_borrowing", "old_borrowing", "evicted_borrowing", "new_borrowing"}
 	if diff := cmp.Diff(wantOrder, order); diff != "" {
 		t.Errorf("Unexpected order (-want,+got):\n%s", diff)
 	}