feat: Adding capacityType discount as a percentage

designs/discount-pricing.md

@@ -0,0 +1,55 @@
+# Discounted Pricing Support
+
+## Overview
+
+Karpenter is currently unaware of any discounted pricing, such as volume discounts or reserved instances/savings plans, which can lead to more expensive instances being chosen. For example an on demand instance with a savings plan discount may cost less than a spot instance of the same type. This pricing is apparently only available from the "payer" account, not any other child account API's for pricing.
+
+This was made explicit recently - the price of spot instances rose sharply at the beginning of Q2 2023. For users on default pricing this may not be noticeable, however if there is any discount for on-demand instances in an account, it could begin to become cheaper to use on-demand instances instead of spot.
+
+## User Stories
+
+* Karpenter will automatically prioritise the cheapest node capacity type in an account based on personal modifications to EC2 pricing
+* Karpenter will allow me to configure discounted pricing at the account level
+
+## Background
+
+[Conversation on Slack](https://kubernetes.slack.com/archives/C02SFFZSA2K/p1684246928553159)
+
+## How Will Karpenter Handle Discounted Pricing
+
+A multiplier will be applied to the price to allow any discounts to be applied and determine the real cost of an EC2 instance.
+For example, a multiplier of 0.9 would apply a 10% discount
+
+Separate multiplier values for Spot and On Demand pricing will be allowed to allow for accounts which have different pricing discounts for each type.
+
+The multiplier will default to a value of 1 so no discount will be applied unless explicitly enabled. 
+
+### Spot pricing
+
+The Spot price will be multiplied with the SpotPriceMultiplier to determine the real cost for Spot instances.
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: karpenter-global-settings
+  namespace: karpenter
+data:
+  # Spot Price Multiplier for including volume discounts etc. for spot prices. The spot price will be multiplied with the spotPriceMultiplier to determine the real cost
+  aws.spotPriceMultiplier: "0.95"
+```
+
+### On Demand pricing
+
+The On Demand price will be multiplied with the OnDemandPriceMultiplier to determine the real cost for On Demand instances.
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: karpenter-global-settings
+  namespace: karpenter
+data:
+  # On Demand Multiplier for including volume discounts etc. to ensure choosing the cheapest available instance. The ondemand price will be multiplied with the onDemandPriceMultiplier to determine the real cost
+  aws.onDemandPriceMultiplier: "0.60"
+```

pkg/apis/settings/settings.go

@@ -42,6 +42,8 @@ var defaultSettings = &Settings{
 	InterruptionQueueName:      "",
 	Tags:                       map[string]string{},
 	ReservedENIs:               0,
+	SpotPriceMultiplier:        1,
+	OnDemandPriceMultiplier:    1,
 }
 
 // +k8s:deepcopy-gen=true
@@ -59,6 +61,8 @@ type Settings struct {
 	InterruptionQueueName      string
 	Tags                       map[string]string
 	ReservedENIs               int
+	SpotPriceMultiplier        float64
+	OnDemandPriceMultiplier    float64
 }
 
 func (*Settings) ConfigMap() string {
@@ -83,6 +87,8 @@ func (*Settings) Inject(ctx context.Context, cm *v1.ConfigMap) (context.Context,
 		configmap.AsString("aws.interruptionQueueName", &s.InterruptionQueueName),
 		AsStringMap("aws.tags", &s.Tags),
 		configmap.AsInt("aws.reservedENIs", &s.ReservedENIs),
+		configmap.AsFloat64("aws.spotPriceMultiplier", &s.SpotPriceMultiplier),
+		configmap.AsFloat64("aws.onDemandPriceMultiplier", &s.OnDemandPriceMultiplier),
 	); err != nil {
 		return ctx, fmt.Errorf("parsing settings, %w", err)
 	}

pkg/apis/settings/settings_validation.go

@@ -32,6 +32,7 @@ func (s Settings) Validate() (errs *apis.FieldError) {
 		s.validateVMMemoryOverheadPercent(),
 		s.validateReservedENIs(),
 		s.validateAssumeRoleDuration(),
+		s.validatePriceMultiplier(),
 	).ViaField("aws")
 }
 
@@ -86,3 +87,13 @@ func (s Settings) validateReservedENIs() (errs *apis.FieldError) {
 	}
 	return nil
 }
+
+func (s Settings) validatePriceMultiplier() (errs *apis.FieldError) {
+	if s.SpotPriceMultiplier <= 0 {
+		return errs.Also(apis.ErrInvalidValue("cannot be zero or negative", "SpotPriceMultiplier"))
+	}
+	if s.OnDemandPriceMultiplier <= 0 {
+		return errs.Also(apis.ErrInvalidValue("cannot be zero or negative", "OnDemandPriceMultiplier"))
+	}
+	return nil
+}

pkg/apis/settings/suite_test.go

@@ -56,6 +56,8 @@ var _ = Describe("Validation", func() {
 		Expect(s.VMMemoryOverheadPercent).To(Equal(0.075))
 		Expect(len(s.Tags)).To(BeZero())
 		Expect(s.ReservedENIs).To(Equal(0))
+		Expect(s.SpotPriceMultiplier).To(Equal(1.0))
+		Expect(s.OnDemandPriceMultiplier).To(Equal(1.0))
 	})
 	It("should succeed to set custom values", func() {
 		cm := &v1.ConfigMap{
@@ -72,6 +74,8 @@ var _ = Describe("Validation", func() {
 				"aws.vmMemoryOverheadPercent":    "0.1",
 				"aws.tags":                       `{"tag1": "value1", "tag2": "value2", "example.com/tag": "my-value"}`,
 				"aws.reservedENIs":               "1",
+				"aws.spotPriceMultiplier":        "2",
+				"aws.onDemandPriceMultiplier":    "0.5",
 			},
 		}
 		ctx, err := (&settings.Settings{}).Inject(ctx, cm)
@@ -90,6 +94,8 @@ var _ = Describe("Validation", func() {
 		Expect(s.Tags).To(HaveKeyWithValue("tag2", "value2"))
 		Expect(s.Tags).To(HaveKeyWithValue("example.com/tag", "my-value"))
 		Expect(s.ReservedENIs).To(Equal(1))
+		Expect(s.SpotPriceMultiplier).To(Equal(2.0))
+		Expect(s.OnDemandPriceMultiplier).To(Equal(0.5))
 	})
 	It("should succeed when setting values that no longer exist (backwards compatibility)", func() {
 		cm := &v1.ConfigMap{
@@ -217,4 +223,22 @@ var _ = Describe("Validation", func() {
 		_, err := (&settings.Settings{}).Inject(ctx, cm)
 		Expect(err).To(HaveOccurred())
 	})
+	It("should fail validation with spotPriceMultiplier is negative", func() {
+		cm := &v1.ConfigMap{
+			Data: map[string]string{
+				"aws.spotPriceMultiplier": "-1",
+			},
+		}
+		_, err := (&settings.Settings{}).Inject(ctx, cm)
+		Expect(err).To(HaveOccurred())
+	})
+	It("should fail validation with OnDemandPriceMultiplier is negative", func() {
+		cm := &v1.ConfigMap{
+			Data: map[string]string{
+				"aws.OnDemandPriceMultiplier": "-0.1",
+			},
+		}
+		_, err := (&settings.Settings{}).Inject(ctx, cm)
+		Expect(err).To(HaveOccurred())
+	})
 })

pkg/providers/pricing/pricing.go

@@ -37,6 +37,7 @@ import (
 	"knative.dev/pkg/logging"
 
 	"github.com/aws/karpenter-core/pkg/utils/pretty"
+	"github.com/aws/karpenter/pkg/apis/settings"
 )
 
 // Provider provides actual pricing data to the AWS cloud provider to allow it to make more informed decisions
@@ -264,7 +265,7 @@ func (p *Provider) fetchOnDemandPricing(ctx context.Context, additionalFilters .
 		additionalFilters...)
 	if err := p.pricing.GetProductsPagesWithContext(ctx, &pricing.GetProductsInput{
 		Filters:     filters,
-		ServiceCode: aws.String("AmazonEC2")}, p.onDemandPage(prices)); err != nil {
+		ServiceCode: aws.String("AmazonEC2")}, p.onDemandPage(ctx, prices)); err != nil {
 		return nil, err
 	}
 	return prices, nil
@@ -273,7 +274,7 @@ func (p *Provider) fetchOnDemandPricing(ctx context.Context, additionalFilters .
 // turning off cyclo here, it measures as a 12 due to all of the type checks of the pricing data which returns a deeply
 // nested map[string]interface{}
 // nolint: gocyclo
-func (p *Provider) onDemandPage(prices map[string]float64) func(output *pricing.GetProductsOutput, b bool) bool {
+func (p *Provider) onDemandPage(ctx context.Context, prices map[string]float64) func(output *pricing.GetProductsOutput, b bool) bool {
 	// this isn't the full pricing struct, just the portions we care about
 	type priceItem struct {
 		Product struct {
@@ -290,6 +291,8 @@ func (p *Provider) onDemandPage(prices map[string]float64) func(output *pricing.
 		}
 	}
 
+	onDemandPriceMultiplier := settings.FromContext(ctx).OnDemandPriceMultiplier
+
 	return func(output *pricing.GetProductsOutput, b bool) bool {
 		currency := "USD"
 		if p.region == "cn-north-1" {
@@ -315,7 +318,7 @@ func (p *Provider) onDemandPage(prices map[string]float64) func(output *pricing.
 					if err != nil || price == 0 {
 						continue
 					}
-					prices[pItem.Product.Attributes.InstanceType] = price
+					prices[pItem.Product.Attributes.InstanceType] = price * onDemandPriceMultiplier
 				}
 			}
 		}
@@ -328,6 +331,8 @@ func (p *Provider) UpdateSpotPricing(ctx context.Context) error {
 	totalOfferings := 0
 
 	prices := map[string]map[string]float64{}
+	spotPriceMultiplier := settings.FromContext(ctx).SpotPriceMultiplier
+
 	err := p.ec2.DescribeSpotPriceHistoryPagesWithContext(ctx, &ec2.DescribeSpotPriceHistoryInput{
 		ProductDescriptions: []*string{aws.String("Linux/UNIX"), aws.String("Linux/UNIX (Amazon VPC)")},
 		// get the latest spot price for each instance type
@@ -344,6 +349,7 @@ func (p *Provider) UpdateSpotPricing(ctx context.Context) error {
 			if sph.Timestamp == nil {
 				continue
 			}
+			spotPrice *= spotPriceMultiplier
 			instanceType := aws.StringValue(sph.InstanceType)
 			az := aws.StringValue(sph.AvailabilityZone)
 			_, ok := prices[instanceType]

pkg/providers/pricing/suite_test.go

@@ -120,6 +120,35 @@ var _ = Describe("Pricing", func() {
 		Expect(price).To(BeNumerically("==", 1.23))
 		Expect(getPricingEstimateMetricValue("c99.large", ec2.UsageClassTypeOnDemand, "")).To(BeNumerically("==", 1.23))
 	})
+	It("should update on-demand pricing with response from the pricing API modified by specified multiplier", func() {
+		ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
+			OnDemandPriceMultiplier: lo.ToPtr(0.5),
+		}))
+		c98ExpectedPrice := 1.20 * settings.FromContext(ctx).OnDemandPriceMultiplier
+		c99ExpectedPrice := 1.23 * settings.FromContext(ctx).OnDemandPriceMultiplier
+
+		// modify our API before creating the pricing provider as it performs an initial update on creation. The pricing
+		// API provides on-demand prices, the ec2 API provides spot prices
+		awsEnv.PricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{
+			PriceList: []aws.JSONValue{
+				fake.NewOnDemandPrice("c98.large", 1.20),
+				fake.NewOnDemandPrice("c99.large", 1.23),
+			},
+		})
+		updateStart := time.Now()
+		ExpectReconcileFailed(ctx, controller, types.NamespacedName{})
+		Eventually(func() bool { return awsEnv.PricingProvider.OnDemandLastUpdated().After(updateStart) }).Should(BeTrue())
+
+		price, ok := awsEnv.PricingProvider.OnDemandPrice("c98.large")
+		Expect(ok).To(BeTrue())
+		Expect(price).To(BeNumerically("==", c98ExpectedPrice))
+		Expect(getPricingEstimateMetricValue("c98.large", ec2.UsageClassTypeOnDemand, "")).To(BeNumerically("==", c98ExpectedPrice))
+
+		price, ok = awsEnv.PricingProvider.OnDemandPrice("c99.large")
+		Expect(ok).To(BeTrue())
+		Expect(price).To(BeNumerically("==", c99ExpectedPrice))
+		Expect(getPricingEstimateMetricValue("c99.large", ec2.UsageClassTypeOnDemand, "")).To(BeNumerically("==", c99ExpectedPrice))
+	})
 	It("should update spot pricing with response from the pricing API", func() {
 		now := time.Now()
 		awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{
@@ -170,6 +199,74 @@ var _ = Describe("Pricing", func() {
 		Expect(price).To(BeNumerically("==", 1.23))
 		Expect(getPricingEstimateMetricValue("c99.large", ec2.UsageClassTypeSpot, "test-zone-1a")).To(BeNumerically("==", 1.23))
 	})
+	It("should update spot pricing with response from the pricing API modified by specified multiplier", func() {
+		ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{
+			SpotPriceMultiplier: lo.ToPtr(0.7),
+		}))
+		c99Zone1aExpectedPrice := 1.23 * settings.FromContext(ctx).SpotPriceMultiplier
+		c99Zone1bExpectedPrice := 1.50 * settings.FromContext(ctx).SpotPriceMultiplier
+		c98Zone1aExpectedPrice := 1.20 * settings.FromContext(ctx).SpotPriceMultiplier
+		c98Zone1bExpectedPrice := 1.10 * settings.FromContext(ctx).SpotPriceMultiplier
+
+		now := time.Now()
+		awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{
+			SpotPriceHistory: []*ec2.SpotPrice{
+				{
+					AvailabilityZone: aws.String("test-zone-1a"),
+					InstanceType:     aws.String("c99.large"),
+					SpotPrice:        aws.String("1.23"),
+					Timestamp:        &now,
+				},
+				{
+					AvailabilityZone: aws.String("test-zone-1a"),
+					InstanceType:     aws.String("c98.large"),
+					SpotPrice:        aws.String("1.20"),
+					Timestamp:        &now,
+				},
+				{
+					AvailabilityZone: aws.String("test-zone-1b"),
+					InstanceType:     aws.String("c99.large"),
+					SpotPrice:        aws.String("1.50"),
+					Timestamp:        &now,
+				},
+				{
+					AvailabilityZone: aws.String("test-zone-1b"),
+					InstanceType:     aws.String("c98.large"),
+					SpotPrice:        aws.String("1.10"),
+					Timestamp:        &now,
+				},
+			},
+		})
+		awsEnv.PricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{
+			PriceList: []aws.JSONValue{
+				fake.NewOnDemandPrice("c98.large", 1.20),
+				fake.NewOnDemandPrice("c99.large", 1.23),
+			},
+		})
+		updateStart := time.Now()
+		ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{})
+		Eventually(func() bool { return awsEnv.PricingProvider.SpotLastUpdated().After(updateStart) }).Should(BeTrue())
+
+		price, ok := awsEnv.PricingProvider.SpotPrice("c98.large", "test-zone-1a")
+		Expect(ok).To(BeTrue())
+		Expect(price).To(BeNumerically("==", c98Zone1aExpectedPrice))
+		Expect(getPricingEstimateMetricValue("c98.large", ec2.UsageClassTypeSpot, "test-zone-1a")).To(BeNumerically("==", c98Zone1aExpectedPrice))
+
+		price, ok = awsEnv.PricingProvider.SpotPrice("c98.large", "test-zone-1b")
+		Expect(ok).To(BeTrue())
+		Expect(price).To(BeNumerically("==", c98Zone1bExpectedPrice))
+		Expect(getPricingEstimateMetricValue("c98.large", ec2.UsageClassTypeSpot, "test-zone-1b")).To(BeNumerically("==", c98Zone1bExpectedPrice))
+
+		price, ok = awsEnv.PricingProvider.SpotPrice("c99.large", "test-zone-1a")
+		Expect(ok).To(BeTrue())
+		Expect(price).To(BeNumerically("==", c99Zone1aExpectedPrice))
+		Expect(getPricingEstimateMetricValue("c99.large", ec2.UsageClassTypeSpot, "test-zone-1a")).To(BeNumerically("==", c99Zone1aExpectedPrice))
+
+		price, ok = awsEnv.PricingProvider.SpotPrice("c99.large", "test-zone-1b")
+		Expect(ok).To(BeTrue())
+		Expect(price).To(BeNumerically("==", c99Zone1bExpectedPrice))
+		Expect(getPricingEstimateMetricValue("c99.large", ec2.UsageClassTypeSpot, "test-zone-1b")).To(BeNumerically("==", c99Zone1bExpectedPrice))
+	})
 	It("should update zonal pricing with data from the spot pricing API", func() {
 		now := time.Now()
 		awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{

pkg/test/settings.go

@@ -34,6 +34,8 @@ type SettingOptions struct {
 	InterruptionQueueName      *string
 	Tags                       map[string]string
 	ReservedENIs               *int
+	SpotPriceMultiplier        *float64
+	OnDemandPriceMultiplier    *float64
 }
 
 func Settings(overrides ...SettingOptions) *awssettings.Settings {
@@ -54,5 +56,7 @@ func Settings(overrides ...SettingOptions) *awssettings.Settings {
 		InterruptionQueueName:      lo.FromPtrOr(options.InterruptionQueueName, ""),
 		Tags:                       options.Tags,
 		ReservedENIs:               lo.FromPtrOr(options.ReservedENIs, 0),
+		SpotPriceMultiplier:        lo.FromPtrOr(options.SpotPriceMultiplier, 1.0),
+		OnDemandPriceMultiplier:    lo.FromPtrOr(options.OnDemandPriceMultiplier, 1.0),
 	}
 }

website/content/en/preview/concepts/settings.md

@@ -75,6 +75,10 @@ data:
   # Reserved ENIs are not included in the calculations for max-pods or kube-reserved
   # This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html
   aws.reservedENIs: "1"
+  # Spot Price Multiplier for including volume discounts etc. for spot prices. The spot price will be multiplied with the spotPriceMultiplier to determine the real cost
+  aws.spotPriceMultiplier: "0.95"
+  # On Demand Multiplier for including volume discounts etc. to ensure choosing the cheapest available instance. The ondemand price will be multiplied with the onDemandPriceMultiplier to determine the real cost
+  aws.onDemandPriceMultiplier: "0.60"
 ```
 
 ### Feature Gates