create constraints on offerings

kwok/cloudprovider/cloudprovider.go

@@ -207,8 +207,8 @@ func addInstanceLabels(labels map[string]string, instanceType *cloudprovider.Ins
 	// Kwok has some scalability limitations.
 	// Randomly add each new node to one of the pre-created kwokPartitions.
 	ret[kwokPartitionLabelKey] = lo.Sample(KwokPartitions)
-	ret[v1beta1.CapacityTypeLabelKey] = offering.CapacityType
-	ret[v1.LabelTopologyZone] = offering.Zone
+	ret[v1beta1.CapacityTypeLabelKey] = offering.Constraints[v1beta1.CapacityTypeLabelKey]
+	ret[v1.LabelTopologyZone] = offering.Constraints[v1.LabelTopologyZone]
 	ret[v1.LabelHostname] = nodeClaim.Name
 
 	ret[kwokLabelKey] = kwokLabelValue

kwok/cloudprovider/helpers.go

@@ -140,8 +140,12 @@ func newInstanceType(options InstanceTypeOptions) *cloudprovider.InstanceType {
 		scheduling.NewRequirement(v1.LabelInstanceTypeStable, v1.NodeSelectorOpIn, options.Name),
 		scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, options.Architecture),
 		scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, osNames...),
-		scheduling.NewRequirement(v1.LabelTopologyZone, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.Zone })...),
-		scheduling.NewRequirement(v1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.CapacityType })...),
+		scheduling.NewRequirement(v1.LabelTopologyZone, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string {
+			return o.Constraints[v1.LabelTopologyZone]
+		})...),
+		scheduling.NewRequirement(v1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string {
+			return o.Constraints[v1beta1.CapacityTypeLabelKey]
+		})...),
 		scheduling.NewRequirement(InstanceSizeLabelKey, v1.NodeSelectorOpIn, options.instanceTypeLabels[InstanceSizeLabelKey]),
 		scheduling.NewRequirement(InstanceFamilyLabelKey, v1.NodeSelectorOpIn, options.instanceTypeLabels[InstanceFamilyLabelKey]),
 		scheduling.NewRequirement(InstanceCPULabelKey, v1.NodeSelectorOpIn, options.instanceTypeLabels[InstanceCPULabelKey]),

kwok/tools/gen_instance_types.go

@@ -93,10 +93,12 @@ func constructGenericInstanceTypes() []kwok.InstanceTypeOptions {
 					for _, zone := range KwokZones {
 						for _, ct := range []string{v1beta1.CapacityTypeSpot, v1beta1.CapacityTypeOnDemand} {
 							opts.Offerings = append(opts.Offerings, cloudprovider.Offering{
-								CapacityType: ct,
-								Zone:         zone,
-								Price:        lo.Ternary(ct == v1beta1.CapacityTypeSpot, price*.7, price),
-								Available:    true,
+								Constraints: map[string]string{
+									v1beta1.CapacityTypeLabelKey: ct,
+									v1.LabelTopologyZone:         zone,
+								},
+								Price:     lo.Ternary(ct == v1beta1.CapacityTypeSpot, price*.7, price),
+								Available: true,
 							})
 						}
 					}

pkg/cloudprovider/fake/cloudprovider.go

@@ -128,11 +128,11 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *v1beta1.NodeClaim
 	// Find Offering
 	for _, o := range instanceType.Offerings.Available() {
 		if reqs.Compatible(scheduling.NewRequirements(
-			scheduling.NewRequirement(v1.LabelTopologyZone, v1.NodeSelectorOpIn, o.Zone),
-			scheduling.NewRequirement(v1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, o.CapacityType),
+			scheduling.NewRequirement(v1.LabelTopologyZone, v1.NodeSelectorOpIn, o.Constraints[v1.LabelTopologyZone]),
+			scheduling.NewRequirement(v1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, o.Constraints[v1beta1.CapacityTypeLabelKey]),
 		), scheduling.AllowUndefinedWellKnownLabels) == nil {
-			labels[v1.LabelTopologyZone] = o.Zone
-			labels[v1beta1.CapacityTypeLabelKey] = o.CapacityType
+			labels[v1.LabelTopologyZone] = o.Constraints[v1.LabelTopologyZone]
+			labels[v1beta1.CapacityTypeLabelKey] = o.Constraints[v1beta1.CapacityTypeLabelKey]
 			break
 		}
 	}

pkg/cloudprovider/fake/instancetype.go

@@ -29,6 +29,7 @@ import (
 	"sigs.k8s.io/karpenter/pkg/apis/v1beta1"
 	"sigs.k8s.io/karpenter/pkg/cloudprovider"
 	"sigs.k8s.io/karpenter/pkg/scheduling"
+	"sigs.k8s.io/karpenter/pkg/test"
 )
 
 const (
@@ -66,11 +67,11 @@ func NewInstanceTypeWithCustomRequirement(options InstanceTypeOptions, customReq
 	}
 	if len(options.Offerings) == 0 {
 		options.Offerings = []cloudprovider.Offering{
-			{CapacityType: "spot", Zone: "test-zone-1", Price: PriceFromResources(options.Resources), Available: true},
-			{CapacityType: "spot", Zone: "test-zone-2", Price: PriceFromResources(options.Resources), Available: true},
-			{CapacityType: "on-demand", Zone: "test-zone-1", Price: PriceFromResources(options.Resources), Available: true},
-			{CapacityType: "on-demand", Zone: "test-zone-2", Price: PriceFromResources(options.Resources), Available: true},
-			{CapacityType: "on-demand", Zone: "test-zone-3", Price: PriceFromResources(options.Resources), Available: true},
+			{Constraints: test.Constraints("spot", "test-zone-1"), Price: PriceFromResources(options.Resources), Available: true},
+			{Constraints: test.Constraints("spot", "test-zone-2"), Price: PriceFromResources(options.Resources), Available: true},
+			{Constraints: test.Constraints("on-demand", "test-zone-1"), Price: PriceFromResources(options.Resources), Available: true},
+			{Constraints: test.Constraints("on-demand", "test-zone-2"), Price: PriceFromResources(options.Resources), Available: true},
+			{Constraints: test.Constraints("on-demand", "test-zone-3"), Price: PriceFromResources(options.Resources), Available: true},
 		}
 	}
 	if len(options.Architecture) == 0 {
@@ -83,8 +84,12 @@ func NewInstanceTypeWithCustomRequirement(options InstanceTypeOptions, customReq
 		scheduling.NewRequirement(v1.LabelInstanceTypeStable, v1.NodeSelectorOpIn, options.Name),
 		scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, options.Architecture),
 		scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, sets.List(options.OperatingSystems)...),
-		scheduling.NewRequirement(v1.LabelTopologyZone, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.Zone })...),
-		scheduling.NewRequirement(v1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.CapacityType })...),
+		scheduling.NewRequirement(v1.LabelTopologyZone, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string {
+			return o.Constraints[v1.LabelTopologyZone]
+		})...),
+		scheduling.NewRequirement(v1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, lo.Map(options.Offerings.Available(), func(o cloudprovider.Offering, _ int) string {
+			return o.Constraints[v1beta1.CapacityTypeLabelKey]
+		})...),
 		scheduling.NewRequirement(LabelInstanceSize, v1.NodeSelectorOpDoesNotExist),
 		scheduling.NewRequirement(ExoticInstanceLabelKey, v1.NodeSelectorOpDoesNotExist),
 		scheduling.NewRequirement(IntegerInstanceLabelKey, v1.NodeSelectorOpIn, fmt.Sprint(options.Resources.Cpu().Value())),
@@ -135,10 +140,9 @@ func InstanceTypesAssorted() []*cloudprovider.InstanceType {
 							price := PriceFromResources(opts.Resources)
 							opts.Offerings = []cloudprovider.Offering{
 								{
-									CapacityType: ct,
-									Zone:         zone,
-									Price:        price,
-									Available:    true,
+									Constraints: test.Constraints(ct, zone),
+									Price:       price,
+									Available:   true,
 								},
 							}
 							instanceTypes = append(instanceTypes, NewInstanceType(opts))

pkg/cloudprovider/types.go

@@ -215,11 +215,11 @@ func (i InstanceTypeOverhead) Total() v1.ResourceList {
 }
 
 // An Offering describes where an InstanceType is available to be used, with the expectation that its properties
-// may be tightly coupled (e.g. the availability of an instance type in some zone is scoped to a capacity type)
+// may be tightly coupled (e.g. the availability of an instance type in some zone is scoped to a capacity type) and
+// these properties are captured with labels in the Constraints map. v1beta1.CapacityTypeLabelKey is required.
 type Offering struct {
-	CapacityType string
-	Zone         string
-	Price        float64
+	Constraints map[string]string
+	Price       float64
 	// Available is added so that Offerings can return all offerings that have ever existed for an instance type,
 	// so we can get historical pricing data for calculating savings in consolidation
 	Available bool
@@ -228,10 +228,18 @@ type Offering struct {
 type Offerings []Offering
 
 // Get gets the offering from an offering slice that matches the
-// passed zone and capacity type
-func (ofs Offerings) Get(ct, zone string) (Offering, bool) {
+// passed zone, capacityType, and other constraints
+func (ofs Offerings) Get(constraints map[string]string) (Offering, bool) {
 	return lo.Find(ofs, func(of Offering) bool {
-		return of.CapacityType == ct && of.Zone == zone
+		match := false
+		for label, v := range of.Constraints {
+			fmt.Println("value: ", v, " and constraint label: ", label, " and constraint value: ", constraints[label])
+			match = lo.Ternary(v == constraints[label], true, false)
+		}
+		fmt.Println("")
+		fmt.Println("match: ", match)
+		fmt.Println("")
+		return match
 	})
 }
 
@@ -245,8 +253,15 @@ func (ofs Offerings) Available() Offerings {
 // Compatible returns the offerings based on the passed requirements
 func (ofs Offerings) Compatible(reqs scheduling.Requirements) Offerings {
 	return lo.Filter(ofs, func(offering Offering, _ int) bool {
-		return (!reqs.Has(v1.LabelTopologyZone) || reqs.Get(v1.LabelTopologyZone).Has(offering.Zone)) &&
-			(!reqs.Has(v1beta1.CapacityTypeLabelKey) || reqs.Get(v1beta1.CapacityTypeLabelKey).Has(offering.CapacityType))
+		match := true
+		for label, v := range offering.Constraints {
+			match = (!reqs.Has(label) || reqs.Get(label).Has(v)) && match
+		}
+		fmt.Println("")
+		fmt.Println("")
+		fmt.Println("")
+		fmt.Println("match: ", match, " this of: ", offering, " with these reqs: ", reqs)
+		return match
 	})
 }
 

pkg/controllers/disruption/consolidation.go

@@ -291,7 +291,7 @@ func (c *consolidation) computeSpotToSpotConsolidation(ctx context.Context, cand
 func getCandidatePrices(candidates []*Candidate) (float64, error) {
 	var price float64
 	for _, c := range candidates {
-		offering, ok := c.instanceType.Offerings.Get(c.capacityType, c.zone)
+		offering, ok := c.instanceType.Offerings.Get(c.StateNode.Labels())
 		if !ok {
 			return 0.0, fmt.Errorf("unable to determine offering for %s/%s/%s", c.instanceType.Name, c.capacityType, c.zone)
 		}