From fabd0e07c4a0bba2bfb53949bca03da81bd25c1a Mon Sep 17 00:00:00 2001 From: Ross Kirk Date: Thu, 14 Dec 2023 12:17:55 +0000 Subject: [PATCH] Add spot and on demand price multipliers to allow customers with discounts to apply them via price modifications. Co-authored-by: Ryan Williams Co-authored-by: Adam Robinson <21spock@gmail.com> --- designs/discount-pricing.md | 55 +++++++++++ pkg/apis/settings/settings.go | 6 ++ pkg/apis/settings/settings_validation.go | 11 +++ pkg/apis/settings/suite_test.go | 24 +++++ pkg/providers/pricing/pricing.go | 12 ++- pkg/providers/pricing/suite_test.go | 97 +++++++++++++++++++ pkg/test/settings.go | 4 + .../content/en/preview/concepts/settings.md | 4 + 8 files changed, 210 insertions(+), 3 deletions(-) create mode 100644 designs/discount-pricing.md diff --git a/designs/discount-pricing.md b/designs/discount-pricing.md new file mode 100644 index 000000000000..758f474f7fe2 --- /dev/null +++ b/designs/discount-pricing.md @@ -0,0 +1,55 @@ +# Discounted Pricing Support + +## Overview + +Karpenter is currently unaware of any discounted pricing, such as volume discounts or reserved instances/savings plans, which can lead to more expensive instances being chosen. For example an on demand instance with a savings plan discount may cost less than a spot instance of the same type. This pricing is apparently only available from the "payer" account, not any other child account API's for pricing. + +This was made explicit recently - the price of spot instances rose sharply at the beginning of Q2 2023. For users on default pricing this may not be noticeable, however if there is any discount for on-demand instances in an account, it could begin to become cheaper to use on-demand instances instead of spot. + +## User Stories + +* Karpenter will automatically prioritise the cheapest node capacity type in an account based on personal modifications to EC2 pricing +* Karpenter will allow me to configure discounted pricing at the account level + +## Background + +[Conversation on Slack](https://kubernetes.slack.com/archives/C02SFFZSA2K/p1684246928553159) + +## How Will Karpenter Handle Discounted Pricing + +A multiplier will be applied to the price to allow any discounts to be applied and determine the real cost of an EC2 instance. +For example, a multiplier of 0.9 would apply a 10% discount + +Separate multiplier values for Spot and On Demand pricing will be allowed to allow for accounts which have different pricing discounts for each type. + +The multiplier will default to a value of 1 so no discount will be applied unless explicitly enabled. + +### Spot pricing + +The Spot price will be multiplied with the SpotPriceMultiplier to determine the real cost for Spot instances. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: karpenter-global-settings + namespace: karpenter +data: + # Spot Price Multiplier for including volume discounts etc. for spot prices. The spot price will be multiplied with the spotPriceMultiplier to determine the real cost + aws.spotPriceMultiplier: "0.95" +``` + +### On Demand pricing + +The On Demand price will be multiplied with the OnDemandPriceMultiplier to determine the real cost for On Demand instances. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: karpenter-global-settings + namespace: karpenter +data: + # On Demand Multiplier for including volume discounts etc. to ensure choosing the cheapest available instance. The ondemand price will be multiplied with the onDemandPriceMultiplier to determine the real cost + aws.onDemandPriceMultiplier: "0.60" +``` diff --git a/pkg/apis/settings/settings.go b/pkg/apis/settings/settings.go index 5a3d6467b225..08064b87ed7a 100644 --- a/pkg/apis/settings/settings.go +++ b/pkg/apis/settings/settings.go @@ -42,6 +42,8 @@ var defaultSettings = &Settings{ InterruptionQueueName: "", Tags: map[string]string{}, ReservedENIs: 0, + SpotPriceMultiplier: 1, + OnDemandPriceMultiplier: 1, } // +k8s:deepcopy-gen=true @@ -59,6 +61,8 @@ type Settings struct { InterruptionQueueName string Tags map[string]string ReservedENIs int + SpotPriceMultiplier float64 + OnDemandPriceMultiplier float64 } func (*Settings) ConfigMap() string { @@ -83,6 +87,8 @@ func (*Settings) Inject(ctx context.Context, cm *v1.ConfigMap) (context.Context, configmap.AsString("aws.interruptionQueueName", &s.InterruptionQueueName), AsStringMap("aws.tags", &s.Tags), configmap.AsInt("aws.reservedENIs", &s.ReservedENIs), + configmap.AsFloat64("aws.spotPriceMultiplier", &s.SpotPriceMultiplier), + configmap.AsFloat64("aws.onDemandPriceMultiplier", &s.OnDemandPriceMultiplier), ); err != nil { return ctx, fmt.Errorf("parsing settings, %w", err) } diff --git a/pkg/apis/settings/settings_validation.go b/pkg/apis/settings/settings_validation.go index e8c3e6f9dbe3..77698b079c33 100644 --- a/pkg/apis/settings/settings_validation.go +++ b/pkg/apis/settings/settings_validation.go @@ -32,6 +32,7 @@ func (s Settings) Validate() (errs *apis.FieldError) { s.validateVMMemoryOverheadPercent(), s.validateReservedENIs(), s.validateAssumeRoleDuration(), + s.validatePriceMultiplier(), ).ViaField("aws") } @@ -86,3 +87,13 @@ func (s Settings) validateReservedENIs() (errs *apis.FieldError) { } return nil } + +func (s Settings) validatePriceMultiplier() (errs *apis.FieldError) { + if s.SpotPriceMultiplier <= 0 { + return errs.Also(apis.ErrInvalidValue("cannot be zero or negative", "SpotPriceMultiplier")) + } + if s.OnDemandPriceMultiplier <= 0 { + return errs.Also(apis.ErrInvalidValue("cannot be zero or negative", "OnDemandPriceMultiplier")) + } + return nil +} diff --git a/pkg/apis/settings/suite_test.go b/pkg/apis/settings/suite_test.go index 6bf00189afdb..a1c3506289ee 100644 --- a/pkg/apis/settings/suite_test.go +++ b/pkg/apis/settings/suite_test.go @@ -56,6 +56,8 @@ var _ = Describe("Validation", func() { Expect(s.VMMemoryOverheadPercent).To(Equal(0.075)) Expect(len(s.Tags)).To(BeZero()) Expect(s.ReservedENIs).To(Equal(0)) + Expect(s.SpotPriceMultiplier).To(Equal(1.0)) + Expect(s.OnDemandPriceMultiplier).To(Equal(1.0)) }) It("should succeed to set custom values", func() { cm := &v1.ConfigMap{ @@ -72,6 +74,8 @@ var _ = Describe("Validation", func() { "aws.vmMemoryOverheadPercent": "0.1", "aws.tags": `{"tag1": "value1", "tag2": "value2", "example.com/tag": "my-value"}`, "aws.reservedENIs": "1", + "aws.spotPriceMultiplier": "2", + "aws.onDemandPriceMultiplier": "0.5", }, } ctx, err := (&settings.Settings{}).Inject(ctx, cm) @@ -90,6 +94,8 @@ var _ = Describe("Validation", func() { Expect(s.Tags).To(HaveKeyWithValue("tag2", "value2")) Expect(s.Tags).To(HaveKeyWithValue("example.com/tag", "my-value")) Expect(s.ReservedENIs).To(Equal(1)) + Expect(s.SpotPriceMultiplier).To(Equal(2.0)) + Expect(s.OnDemandPriceMultiplier).To(Equal(0.5)) }) It("should succeed when setting values that no longer exist (backwards compatibility)", func() { cm := &v1.ConfigMap{ @@ -217,4 +223,22 @@ var _ = Describe("Validation", func() { _, err := (&settings.Settings{}).Inject(ctx, cm) Expect(err).To(HaveOccurred()) }) + It("should fail validation with spotPriceMultiplier is negative", func() { + cm := &v1.ConfigMap{ + Data: map[string]string{ + "aws.spotPriceMultiplier": "-1", + }, + } + _, err := (&settings.Settings{}).Inject(ctx, cm) + Expect(err).To(HaveOccurred()) + }) + It("should fail validation with OnDemandPriceMultiplier is negative", func() { + cm := &v1.ConfigMap{ + Data: map[string]string{ + "aws.OnDemandPriceMultiplier": "-0.1", + }, + } + _, err := (&settings.Settings{}).Inject(ctx, cm) + Expect(err).To(HaveOccurred()) + }) }) diff --git a/pkg/providers/pricing/pricing.go b/pkg/providers/pricing/pricing.go index 3faa5b3c8226..47489959ca82 100644 --- a/pkg/providers/pricing/pricing.go +++ b/pkg/providers/pricing/pricing.go @@ -37,6 +37,7 @@ import ( "knative.dev/pkg/logging" "github.com/aws/karpenter-core/pkg/utils/pretty" + "github.com/aws/karpenter/pkg/apis/settings" ) // Provider provides actual pricing data to the AWS cloud provider to allow it to make more informed decisions @@ -264,7 +265,7 @@ func (p *Provider) fetchOnDemandPricing(ctx context.Context, additionalFilters . additionalFilters...) if err := p.pricing.GetProductsPagesWithContext(ctx, &pricing.GetProductsInput{ Filters: filters, - ServiceCode: aws.String("AmazonEC2")}, p.onDemandPage(prices)); err != nil { + ServiceCode: aws.String("AmazonEC2")}, p.onDemandPage(ctx, prices)); err != nil { return nil, err } return prices, nil @@ -273,7 +274,7 @@ func (p *Provider) fetchOnDemandPricing(ctx context.Context, additionalFilters . // turning off cyclo here, it measures as a 12 due to all of the type checks of the pricing data which returns a deeply // nested map[string]interface{} // nolint: gocyclo -func (p *Provider) onDemandPage(prices map[string]float64) func(output *pricing.GetProductsOutput, b bool) bool { +func (p *Provider) onDemandPage(ctx context.Context, prices map[string]float64) func(output *pricing.GetProductsOutput, b bool) bool { // this isn't the full pricing struct, just the portions we care about type priceItem struct { Product struct { @@ -290,6 +291,8 @@ func (p *Provider) onDemandPage(prices map[string]float64) func(output *pricing. } } + onDemandPriceMultiplier := settings.FromContext(ctx).OnDemandPriceMultiplier + return func(output *pricing.GetProductsOutput, b bool) bool { currency := "USD" if p.region == "cn-north-1" { @@ -315,7 +318,7 @@ func (p *Provider) onDemandPage(prices map[string]float64) func(output *pricing. if err != nil || price == 0 { continue } - prices[pItem.Product.Attributes.InstanceType] = price + prices[pItem.Product.Attributes.InstanceType] = price * onDemandPriceMultiplier } } } @@ -328,6 +331,8 @@ func (p *Provider) UpdateSpotPricing(ctx context.Context) error { totalOfferings := 0 prices := map[string]map[string]float64{} + spotPriceMultiplier := settings.FromContext(ctx).SpotPriceMultiplier + err := p.ec2.DescribeSpotPriceHistoryPagesWithContext(ctx, &ec2.DescribeSpotPriceHistoryInput{ ProductDescriptions: []*string{aws.String("Linux/UNIX"), aws.String("Linux/UNIX (Amazon VPC)")}, // get the latest spot price for each instance type @@ -344,6 +349,7 @@ func (p *Provider) UpdateSpotPricing(ctx context.Context) error { if sph.Timestamp == nil { continue } + spotPrice *= spotPriceMultiplier instanceType := aws.StringValue(sph.InstanceType) az := aws.StringValue(sph.AvailabilityZone) _, ok := prices[instanceType] diff --git a/pkg/providers/pricing/suite_test.go b/pkg/providers/pricing/suite_test.go index d6df237c5b51..e08e82e12782 100644 --- a/pkg/providers/pricing/suite_test.go +++ b/pkg/providers/pricing/suite_test.go @@ -120,6 +120,35 @@ var _ = Describe("Pricing", func() { Expect(price).To(BeNumerically("==", 1.23)) Expect(getPricingEstimateMetricValue("c99.large", ec2.UsageClassTypeOnDemand, "")).To(BeNumerically("==", 1.23)) }) + It("should update on-demand pricing with response from the pricing API modified by specified multiplier", func() { + ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + OnDemandPriceMultiplier: lo.ToPtr(0.5), + })) + c98ExpectedPrice := 1.20 * settings.FromContext(ctx).OnDemandPriceMultiplier + c99ExpectedPrice := 1.23 * settings.FromContext(ctx).OnDemandPriceMultiplier + + // modify our API before creating the pricing provider as it performs an initial update on creation. The pricing + // API provides on-demand prices, the ec2 API provides spot prices + awsEnv.PricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{ + PriceList: []aws.JSONValue{ + fake.NewOnDemandPrice("c98.large", 1.20), + fake.NewOnDemandPrice("c99.large", 1.23), + }, + }) + updateStart := time.Now() + ExpectReconcileFailed(ctx, controller, types.NamespacedName{}) + Eventually(func() bool { return awsEnv.PricingProvider.OnDemandLastUpdated().After(updateStart) }).Should(BeTrue()) + + price, ok := awsEnv.PricingProvider.OnDemandPrice("c98.large") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically("==", c98ExpectedPrice)) + Expect(getPricingEstimateMetricValue("c98.large", ec2.UsageClassTypeOnDemand, "")).To(BeNumerically("==", c98ExpectedPrice)) + + price, ok = awsEnv.PricingProvider.OnDemandPrice("c99.large") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically("==", c99ExpectedPrice)) + Expect(getPricingEstimateMetricValue("c99.large", ec2.UsageClassTypeOnDemand, "")).To(BeNumerically("==", c99ExpectedPrice)) + }) It("should update spot pricing with response from the pricing API", func() { now := time.Now() awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ @@ -170,6 +199,74 @@ var _ = Describe("Pricing", func() { Expect(price).To(BeNumerically("==", 1.23)) Expect(getPricingEstimateMetricValue("c99.large", ec2.UsageClassTypeSpot, "test-zone-1a")).To(BeNumerically("==", 1.23)) }) + It("should update spot pricing with response from the pricing API modified by specified multiplier", func() { + ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + SpotPriceMultiplier: lo.ToPtr(0.7), + })) + c99Zone1aExpectedPrice := 1.23 * settings.FromContext(ctx).SpotPriceMultiplier + c99Zone1bExpectedPrice := 1.50 * settings.FromContext(ctx).SpotPriceMultiplier + c98Zone1aExpectedPrice := 1.20 * settings.FromContext(ctx).SpotPriceMultiplier + c98Zone1bExpectedPrice := 1.10 * settings.FromContext(ctx).SpotPriceMultiplier + + now := time.Now() + awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ + SpotPriceHistory: []*ec2.SpotPrice{ + { + AvailabilityZone: aws.String("test-zone-1a"), + InstanceType: aws.String("c99.large"), + SpotPrice: aws.String("1.23"), + Timestamp: &now, + }, + { + AvailabilityZone: aws.String("test-zone-1a"), + InstanceType: aws.String("c98.large"), + SpotPrice: aws.String("1.20"), + Timestamp: &now, + }, + { + AvailabilityZone: aws.String("test-zone-1b"), + InstanceType: aws.String("c99.large"), + SpotPrice: aws.String("1.50"), + Timestamp: &now, + }, + { + AvailabilityZone: aws.String("test-zone-1b"), + InstanceType: aws.String("c98.large"), + SpotPrice: aws.String("1.10"), + Timestamp: &now, + }, + }, + }) + awsEnv.PricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{ + PriceList: []aws.JSONValue{ + fake.NewOnDemandPrice("c98.large", 1.20), + fake.NewOnDemandPrice("c99.large", 1.23), + }, + }) + updateStart := time.Now() + ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + Eventually(func() bool { return awsEnv.PricingProvider.SpotLastUpdated().After(updateStart) }).Should(BeTrue()) + + price, ok := awsEnv.PricingProvider.SpotPrice("c98.large", "test-zone-1a") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically("==", c98Zone1aExpectedPrice)) + Expect(getPricingEstimateMetricValue("c98.large", ec2.UsageClassTypeSpot, "test-zone-1a")).To(BeNumerically("==", c98Zone1aExpectedPrice)) + + price, ok = awsEnv.PricingProvider.SpotPrice("c98.large", "test-zone-1b") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically("==", c98Zone1bExpectedPrice)) + Expect(getPricingEstimateMetricValue("c98.large", ec2.UsageClassTypeSpot, "test-zone-1b")).To(BeNumerically("==", c98Zone1bExpectedPrice)) + + price, ok = awsEnv.PricingProvider.SpotPrice("c99.large", "test-zone-1a") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically("==", c99Zone1aExpectedPrice)) + Expect(getPricingEstimateMetricValue("c99.large", ec2.UsageClassTypeSpot, "test-zone-1a")).To(BeNumerically("==", c99Zone1aExpectedPrice)) + + price, ok = awsEnv.PricingProvider.SpotPrice("c99.large", "test-zone-1b") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically("==", c99Zone1bExpectedPrice)) + Expect(getPricingEstimateMetricValue("c99.large", ec2.UsageClassTypeSpot, "test-zone-1b")).To(BeNumerically("==", c99Zone1bExpectedPrice)) + }) It("should update zonal pricing with data from the spot pricing API", func() { now := time.Now() awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ diff --git a/pkg/test/settings.go b/pkg/test/settings.go index baff303ebf2c..d629f2a73bba 100644 --- a/pkg/test/settings.go +++ b/pkg/test/settings.go @@ -34,6 +34,8 @@ type SettingOptions struct { InterruptionQueueName *string Tags map[string]string ReservedENIs *int + SpotPriceMultiplier *float64 + OnDemandPriceMultiplier *float64 } func Settings(overrides ...SettingOptions) *awssettings.Settings { @@ -54,5 +56,7 @@ func Settings(overrides ...SettingOptions) *awssettings.Settings { InterruptionQueueName: lo.FromPtrOr(options.InterruptionQueueName, ""), Tags: options.Tags, ReservedENIs: lo.FromPtrOr(options.ReservedENIs, 0), + SpotPriceMultiplier: lo.FromPtrOr(options.SpotPriceMultiplier, 1.0), + OnDemandPriceMultiplier: lo.FromPtrOr(options.OnDemandPriceMultiplier, 1.0), } } diff --git a/website/content/en/preview/concepts/settings.md b/website/content/en/preview/concepts/settings.md index cfe4416f652d..61aad7385094 100644 --- a/website/content/en/preview/concepts/settings.md +++ b/website/content/en/preview/concepts/settings.md @@ -75,6 +75,10 @@ data: # Reserved ENIs are not included in the calculations for max-pods or kube-reserved # This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html aws.reservedENIs: "1" + # Spot Price Multiplier for including volume discounts etc. for spot prices. The spot price will be multiplied with the spotPriceMultiplier to determine the real cost + aws.spotPriceMultiplier: "0.95" + # On Demand Multiplier for including volume discounts etc. to ensure choosing the cheapest available instance. The ondemand price will be multiplied with the onDemandPriceMultiplier to determine the real cost + aws.onDemandPriceMultiplier: "0.60" ``` ### Feature Gates