create constraints on offerings

kwok/cloudprovider/cloudprovider.go

@@ -207,8 +207,8 @@ func addInstanceLabels(labels map[string]string, instanceType *cloudprovider.Ins
 	// Kwok has some scalability limitations.
 	// Randomly add each new node to one of the pre-created kwokPartitions.
 	ret[kwokPartitionLabelKey] = lo.Sample(KwokPartitions)
-	ret[v1beta1.CapacityTypeLabelKey] = offering.CapacityType
-	ret[v1.LabelTopologyZone] = offering.Zone
+	ret[v1beta1.CapacityTypeLabelKey] = offering.Requirements.Get(v1beta1.CapacityTypeLabelKey).Values()[0]
+	ret[v1.LabelTopologyZone] = offering.Requirements.Get(v1.LabelTopologyZone).Values()[0]
 	ret[v1.LabelHostname] = nodeClaim.Name
 
 	ret[kwokLabelKey] = kwokLabelValue

kwok/cloudprovider/helpers.go

@@ -140,8 +140,12 @@ func newInstanceType(options InstanceTypeOptions) *cloudprovider.InstanceType {
 		scheduling.NewRequirement(v1.LabelInstanceTypeStable, v1.NodeSelectorOpIn, options.Name),
 		scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, options.Architecture),
 		scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, osNames...),
-		scheduling.NewRequirement(v1.LabelTopologyZone, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.Zone })...),
-		scheduling.NewRequirement(v1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.CapacityType })...),
+		scheduling.NewRequirement(v1.LabelTopologyZone, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string {
+			return o.Requirements.Get(v1.LabelTopologyZone).Values()[0]
+		})...),
+		scheduling.NewRequirement(v1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string {
+			return o.Requirements.Get(v1beta1.CapacityTypeLabelKey).Values()[0]
+		})...),
 		scheduling.NewRequirement(InstanceSizeLabelKey, v1.NodeSelectorOpIn, options.instanceTypeLabels[InstanceSizeLabelKey]),
 		scheduling.NewRequirement(InstanceFamilyLabelKey, v1.NodeSelectorOpIn, options.instanceTypeLabels[InstanceFamilyLabelKey]),
 		scheduling.NewRequirement(InstanceCPULabelKey, v1.NodeSelectorOpIn, options.instanceTypeLabels[InstanceCPULabelKey]),

kwok/tools/gen_instance_types.go

@@ -28,6 +28,7 @@ import (
 	kwok "sigs.k8s.io/karpenter/kwok/cloudprovider"
 	"sigs.k8s.io/karpenter/pkg/apis/v1beta1"
 	"sigs.k8s.io/karpenter/pkg/cloudprovider"
+	"sigs.k8s.io/karpenter/pkg/scheduling"
 )
 
 var (
@@ -93,10 +94,12 @@ func constructGenericInstanceTypes() []kwok.InstanceTypeOptions {
 					for _, zone := range KwokZones {
 						for _, ct := range []string{v1beta1.CapacityTypeSpot, v1beta1.CapacityTypeOnDemand} {
 							opts.Offerings = append(opts.Offerings, cloudprovider.Offering{
-								CapacityType: ct,
-								Zone:         zone,
-								Price:        lo.Ternary(ct == v1beta1.CapacityTypeSpot, price*.7, price),
-								Available:    true,
+								Requirements: scheduling.NewLabelRequirements(map[string]string{
+									v1beta1.CapacityTypeLabelKey: ct,
+									v1.LabelTopologyZone:         zone,
+								}),
+								Price:     lo.Ternary(ct == v1beta1.CapacityTypeSpot, price*.7, price),
+								Available: true,
 							})
 						}
 					}

pkg/cloudprovider/fake/cloudprovider.go

@@ -108,7 +108,7 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *v1beta1.NodeClaim
 	np := &v1beta1.NodePool{ObjectMeta: metav1.ObjectMeta{Name: nodeClaim.Labels[v1beta1.NodePoolLabelKey]}}
 	instanceTypes := lo.Filter(lo.Must(c.GetInstanceTypes(ctx, np)), func(i *cloudprovider.InstanceType, _ int) bool {
 		return reqs.Compatible(i.Requirements, scheduling.AllowUndefinedWellKnownLabels) == nil &&
-			len(i.Offerings.Compatible(reqs).Available()) > 0 &&
+			len(i.Offerings.Available().Compatible(reqs)) > 0 &&
 			resources.Fits(nodeClaim.Spec.Resources.Requests, i.Allocatable())
 	})
 	// Order instance types so that we get the cheapest instance types of the available offerings
@@ -127,12 +127,9 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *v1beta1.NodeClaim
 	}
 	// Find Offering
 	for _, o := range instanceType.Offerings.Available() {
-		if reqs.Compatible(scheduling.NewRequirements(
-			scheduling.NewRequirement(v1.LabelTopologyZone, v1.NodeSelectorOpIn, o.Zone),
-			scheduling.NewRequirement(v1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, o.CapacityType),
-		), scheduling.AllowUndefinedWellKnownLabels) == nil {
-			labels[v1.LabelTopologyZone] = o.Zone
-			labels[v1beta1.CapacityTypeLabelKey] = o.CapacityType
+		if reqs.Compatible(o.Requirements, scheduling.AllowUndefinedWellKnownLabels) == nil {
+			labels[v1.LabelTopologyZone] = o.Zone()
+			labels[v1beta1.CapacityTypeLabelKey] = o.CapacityType()
 			break
 		}
 	}

pkg/cloudprovider/fake/instancetype.go

@@ -66,11 +66,26 @@ func NewInstanceTypeWithCustomRequirement(options InstanceTypeOptions, customReq
 	}
 	if len(options.Offerings) == 0 {
 		options.Offerings = []cloudprovider.Offering{
-			{CapacityType: "spot", Zone: "test-zone-1", Price: PriceFromResources(options.Resources), Available: true},
-			{CapacityType: "spot", Zone: "test-zone-2", Price: PriceFromResources(options.Resources), Available: true},
-			{CapacityType: "on-demand", Zone: "test-zone-1", Price: PriceFromResources(options.Resources), Available: true},
-			{CapacityType: "on-demand", Zone: "test-zone-2", Price: PriceFromResources(options.Resources), Available: true},
-			{CapacityType: "on-demand", Zone: "test-zone-3", Price: PriceFromResources(options.Resources), Available: true},
+			{Requirements: scheduling.NewLabelRequirements(map[string]string{
+				v1beta1.CapacityTypeLabelKey: "spot",
+				v1.LabelTopologyZone:         "test-zone-1",
+			}), Price: PriceFromResources(options.Resources), Available: true},
+			{Requirements: scheduling.NewLabelRequirements(map[string]string{
+				v1beta1.CapacityTypeLabelKey: "spot",
+				v1.LabelTopologyZone:         "test-zone-2",
+			}), Price: PriceFromResources(options.Resources), Available: true},
+			{Requirements: scheduling.NewLabelRequirements(map[string]string{
+				v1beta1.CapacityTypeLabelKey: "on-demand",
+				v1.LabelTopologyZone:         "test-zone-1",
+			}), Price: PriceFromResources(options.Resources), Available: true},
+			{Requirements: scheduling.NewLabelRequirements(map[string]string{
+				v1beta1.CapacityTypeLabelKey: "on-demand",
+				v1.LabelTopologyZone:         "test-zone-2",
+			}), Price: PriceFromResources(options.Resources), Available: true},
+			{Requirements: scheduling.NewLabelRequirements(map[string]string{
+				v1beta1.CapacityTypeLabelKey: "on-demand",
+				v1.LabelTopologyZone:         "test-zone-3",
+			}), Price: PriceFromResources(options.Resources), Available: true},
 		}
 	}
 	if len(options.Architecture) == 0 {
@@ -83,8 +98,12 @@ func NewInstanceTypeWithCustomRequirement(options InstanceTypeOptions, customReq
 		scheduling.NewRequirement(v1.LabelInstanceTypeStable, v1.NodeSelectorOpIn, options.Name),
 		scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, options.Architecture),
 		scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, sets.List(options.OperatingSystems)...),
-		scheduling.NewRequirement(v1.LabelTopologyZone, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.Zone })...),
-		scheduling.NewRequirement(v1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.CapacityType })...),
+		scheduling.NewRequirement(v1.LabelTopologyZone, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string {
+			return o.Zone()
+		})...),
+		scheduling.NewRequirement(v1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string {
+			return o.CapacityType()
+		})...),
 		scheduling.NewRequirement(LabelInstanceSize, v1.NodeSelectorOpDoesNotExist),
 		scheduling.NewRequirement(ExoticInstanceLabelKey, v1.NodeSelectorOpDoesNotExist),
 		scheduling.NewRequirement(IntegerInstanceLabelKey, v1.NodeSelectorOpIn, fmt.Sprint(options.Resources.Cpu().Value())),
@@ -135,10 +154,12 @@ func InstanceTypesAssorted() []*cloudprovider.InstanceType {
 							price := PriceFromResources(opts.Resources)
 							opts.Offerings = []cloudprovider.Offering{
 								{
-									CapacityType: ct,
-									Zone:         zone,
-									Price:        price,
-									Available:    true,
+									Requirements: scheduling.NewLabelRequirements(map[string]string{
+										v1beta1.CapacityTypeLabelKey: ct,
+										v1.LabelTopologyZone:         zone,
+									}),
+									Price:     price,
+									Available: true,
 								},
 							}
 							instanceTypes = append(instanceTypes, NewInstanceType(opts))

pkg/cloudprovider/types.go

@@ -215,10 +215,10 @@ func (i InstanceTypeOverhead) Total() v1.ResourceList {
 }
 
 // An Offering describes where an InstanceType is available to be used, with the expectation that its properties
-// may be tightly coupled (e.g. the availability of an instance type in some zone is scoped to a capacity type)
+// may be tightly coupled (e.g. the availability of an instance type in some zone is scoped to a capacity type) and
+// these properties are captured with labels in the Constraints map. v1beta1.CapacityTypeLabelKey is required.
 type Offering struct {
-	CapacityType string
-	Zone         string
+	Requirements scheduling.Requirements
 	Price        float64
 	// Available is added so that Offerings can return all offerings that have ever existed for an instance type,
 	// so we can get historical pricing data for calculating savings in consolidation
@@ -228,10 +228,10 @@ type Offering struct {
 type Offerings []Offering
 
 // Get gets the offering from an offering slice that matches the
-// passed zone and capacity type
-func (ofs Offerings) Get(ct, zone string) (Offering, bool) {
+// passed zone, capacityType, and other constraints
+func (ofs Offerings) Get(reqs scheduling.Requirements) (Offering, bool) {
 	return lo.Find(ofs, func(of Offering) bool {
-		return of.CapacityType == ct && of.Zone == zone
+		return reqs.Compatible(of.Requirements, scheduling.AllowUndefinedWellKnownLabels) == nil
 	})
 }
 
@@ -245,8 +245,8 @@ func (ofs Offerings) Available() Offerings {
 // Compatible returns the offerings based on the passed requirements
 func (ofs Offerings) Compatible(reqs scheduling.Requirements) Offerings {
 	return lo.Filter(ofs, func(offering Offering, _ int) bool {
-		return (!reqs.Has(v1.LabelTopologyZone) || reqs.Get(v1.LabelTopologyZone).Has(offering.Zone)) &&
-			(!reqs.Has(v1beta1.CapacityTypeLabelKey) || reqs.Get(v1beta1.CapacityTypeLabelKey).Has(offering.CapacityType))
+		// possible add allow undefined labels
+		return reqs.Compatible(offering.Requirements, scheduling.AllowUndefinedWellKnownLabels) == nil
 	})
 }
 
@@ -257,6 +257,39 @@ func (ofs Offerings) Cheapest() Offering {
 	})
 }
 
+// MostExpensive returns the most expensive offering from the return offerings
+func (ofs Offerings) MostExpensive() Offering {
+	return lo.MaxBy(ofs, func(a, b Offering) bool {
+		return a.Price > b.Price
+	})
+}
+
+func (of Offering) CapacityType() string {
+	req, ok := of.Requirements[v1beta1.CapacityTypeLabelKey]
+	if !ok {
+		panic("failed to get capacity type for offering, capacity type must be definied on all offerings")
+	}
+	return req.Values()[0]
+}
+
+func (of Offering) Zone() string {
+	req, ok := of.Requirements[v1.LabelTopologyZone]
+	if !ok {
+		panic("failed to get zone for offering, zone must be definied on all offerings")
+	}
+	return req.Values()[0]
+}
+
+/*
+func (ofs Offerings) CapacityType() string {
+    ct, ok := ofs.Constraints[v1beta1.CapacityTypeLabelKey]
+    if !ok {
+        panic("failed to get capacity type for offering, capacity type must be defined on all offerings")
+    }
+    return ct
+}
+*/
+
 // NodeClaimNotFoundError is an error type returned by CloudProviders when the reason for failure is NotFound
 type NodeClaimNotFoundError struct {
 	error

pkg/controllers/disruption/consolidation.go

@@ -291,7 +291,7 @@ func (c *consolidation) computeSpotToSpotConsolidation(ctx context.Context, cand
 func getCandidatePrices(candidates []*Candidate) (float64, error) {
 	var price float64
 	for _, c := range candidates {
-		offering, ok := c.instanceType.Offerings.Get(c.capacityType, c.zone)
+		offering, ok := c.instanceType.Offerings.Get(scheduling.NewLabelRequirements(c.StateNode.Labels()))
 		if !ok {
 			return 0.0, fmt.Errorf("unable to determine offering for %s/%s/%s", c.instanceType.Name, c.capacityType, c.zone)
 		}