feat: add weights to provisioners and order provisioner scheduling by…

… weights (#2221)

* Sort the provisioners, ordered by weight/priority

* Add functional test support for weighted provisioners

* Update to Provisioner API docs

* Make weight a pointer

* Favor ptr.Int32Value

charts/karpenter/crds/karpenter.sh_provisioners.yaml

@@ -205,6 +205,16 @@ spec:
                   is not set."
                 format: int64
                 type: integer
+              weight:
+                description: Weight is the priority given to the provisioner during
+                  scheduling. A higher numerical weight indicates that this provisioner
+                  will be ordered ahead of other provisioners with lower weights.
+                  A provisioner with no weight will be treated as if it is a provisioner
+                  with a weight of 0.
+                format: int32
+                maximum: 100
+                minimum: 1
+                type: integer
             type: object
           status:
             description: ProvisionerStatus defines the observed state of Provisioner

pkg/apis/provisioning/v1alpha5/provisioner.go

@@ -15,6 +15,10 @@ limitations under the License.
 package v1alpha5
 
 import (
+	"sort"
+
+	"knative.dev/pkg/ptr"
+
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -70,6 +74,14 @@ type ProvisionerSpec struct {
 	TTLSecondsUntilExpired *int64 `json:"ttlSecondsUntilExpired,omitempty"`
 	// Limits define a set of bounds for provisioning capacity.
 	Limits *Limits `json:"limits,omitempty"`
+	// Weight is the priority given to the provisioner during scheduling. A higher
+	// numerical weight indicates that this provisioner will be ordered
+	// ahead of other provisioners with lower weights. A provisioner with no weight
+	// will be treated as if it is a provisioner with a weight of 0.
+	// +kubebuilder:validation:Minimum:=1
+	// +kubebuilder:validation:Maximum:=100
+	// +optional
+	Weight *int32 `json:"weight,omitempty"`
 }
 
 // +kubebuilder:object:generate=false
@@ -119,3 +131,11 @@ type ProvisionerList struct {
 	metav1.ListMeta `json:"metadata,omitempty"`
 	Items           []Provisioner `json:"items"`
 }
+
+// OrderByWeight orders the provisioners in the ProvisionerList
+// by their priority weight in-place
+func (pl *ProvisionerList) OrderByWeight() {
+	sort.Slice(pl.Items, func(a, b int) bool {
+		return ptr.Int32Value(pl.Items[a].Spec.Weight) > ptr.Int32Value(pl.Items[b].Spec.Weight)
+	})
+}

pkg/controllers/provisioning/provisioner.go

@@ -186,6 +186,12 @@ func (p *Provisioner) schedule(ctx context.Context, pods []*v1.Pod) ([]*schedule
 	if err := p.kubeClient.List(ctx, &provisionerList); err != nil {
 		return nil, fmt.Errorf("listing provisioners, %w", err)
 	}
+
+	// nodeTemplates generated from provisioners are ordered by weight
+	// since they are stored within a slice and scheduling
+	// will always attempt to schedule on the first nodeTemplate
+	provisionerList.OrderByWeight()
+
 	for i := range provisionerList.Items {
 		provisioner := &provisionerList.Items[i]
 		if !provisioner.DeletionTimestamp.IsZero() {

pkg/controllers/provisioning/suite_test.go

@@ -16,6 +16,8 @@ package provisioning_test
 
 import (
 	"context"
+	"knative.dev/pkg/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"testing"
 	"time"
 
@@ -700,7 +702,6 @@ var _ = Describe("Multiple Provisioners", func() {
 	It("should schedule to a provisioner by labels", func() {
 		provisioner := test.Provisioner(test.ProvisionerOptions{Labels: map[string]string{"foo": "bar"}})
 		ExpectApplied(ctx, env.Client, provisioner, test.Provisioner())
-		ExpectProvisioned(ctx, env.Client, controller)
 		pod := ExpectProvisioned(ctx, env.Client, controller,
 			test.UnschedulablePod(test.PodOptions{NodeSelector: provisioner.Spec.Labels}),
 		)[0]
@@ -714,4 +715,33 @@ var _ = Describe("Multiple Provisioners", func() {
 		node := ExpectScheduled(ctx, env.Client, pod)
 		Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).ToNot(Equal(provisioner.Name))
 	})
+	Describe("Weighted Provisioners", func() {
+		It("should schedule to the provisioner with the highest priority always", func() {
+			provisioners := []client.Object{
+				test.Provisioner(),
+				test.Provisioner(test.ProvisionerOptions{Weight: ptr.Int32(20)}),
+				test.Provisioner(test.ProvisionerOptions{Weight: ptr.Int32(100)}),
+			}
+			ExpectApplied(ctx, env.Client, provisioners...)
+			pods := ExpectProvisioned(ctx, env.Client, controller, test.UnschedulablePod(), test.UnschedulablePod(), test.UnschedulablePod())
+			for _, pod := range pods {
+				node := ExpectScheduled(ctx, env.Client, pod)
+				Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(provisioners[2].GetName()))
+			}
+		})
+		It("should schedule to explicitly selected provisioner even if other provisioners are higher priority", func() {
+			targetedProvisioner := test.Provisioner()
+			provisioners := []client.Object{
+				targetedProvisioner,
+				test.Provisioner(test.ProvisionerOptions{Weight: ptr.Int32(20)}),
+				test.Provisioner(test.ProvisionerOptions{Weight: ptr.Int32(100)}),
+			}
+			ExpectApplied(ctx, env.Client, provisioners...)
+			pod := ExpectProvisioned(ctx, env.Client, controller,
+				test.UnschedulablePod(test.PodOptions{NodeSelector: map[string]string{v1alpha5.ProvisionerNameLabelKey: targetedProvisioner.Name}}),
+			)[0]
+			node := ExpectScheduled(ctx, env.Client, pod)
+			Expect(node.Labels[v1alpha5.ProvisionerNameLabelKey]).To(Equal(targetedProvisioner.Name))
+		})
+	})
 })

pkg/test/provisioner.go

@@ -43,6 +43,7 @@ type ProvisionerOptions struct {
 	Status                 v1alpha5.ProvisionerStatus
 	TTLSecondsAfterEmpty   *int64
 	TTLSecondsUntilExpired *int64
+	Weight                 *int32
 }
 
 // Provisioner creates a test provisioner with defaults that can be overridden by ProvisionerOptions.
@@ -73,6 +74,7 @@ func Provisioner(overrides ...ProvisionerOptions) *v1alpha5.Provisioner {
 			Limits:                 &v1alpha5.Limits{Resources: options.Limits},
 			TTLSecondsAfterEmpty:   options.TTLSecondsAfterEmpty,
 			TTLSecondsUntilExpired: options.TTLSecondsUntilExpired,
+			Weight:                 options.Weight,
 		},
 		Status: options.Status,
 	}

website/content/en/preview/provisioner.md

@@ -21,6 +21,11 @@ spec:
 
   # If omitted, the feature is disabled, nodes will never scale down due to low utilization
   ttlSecondsAfterEmpty: 30
+
+  # Priority given to the provisioner when the scheduler considers which provisioner
+  # to select. Higher weights indicate higher priority when comparing provisioners.
+  # Specifying no weight is equivalent to specifying a weight of 0.
+  weight: 10
 
   # Provisioned nodes will have these taints
   # Taints may prevent pods from scheduling if they are not tolerated by the pod.