diff --git a/cluster-autoscaler/FAQ.md b/cluster-autoscaler/FAQ.md index ff00b21e0515..2b127d4db7af 100644 --- a/cluster-autoscaler/FAQ.md +++ b/cluster-autoscaler/FAQ.md @@ -629,6 +629,12 @@ would match the cluster size. This expander is described in more details * `priority` - selects the node group that has the highest priority assigned by the user. It's configuration is described in more details [here](expander/priority/readme.md) + +Multiple expanders may be passed, i.e. +`.cluster-autoscaler --expander=priority,least-waste` + +This will cause the `least-waste` expander to be used as a fallback in the event that the priority expander selects multiple node groups. In general, a list of expanders can be used, where the output of one is passed to the next and the final decision by randomly selecting one. An expander must not appear in the list more than once. + ### Does CA respect node affinity when selecting node groups to scale up? CA respects `nodeSelector` and `requiredDuringSchedulingIgnoredDuringExecution` in nodeAffinity given that you have labelled your node groups accordingly. If there is a pod that cannot be scheduled with either `nodeSelector` or `requiredDuringSchedulingIgnoredDuringExecution` specified, CA will only consider node groups that satisfy those requirements for expansion. diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_manager.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_manager.go index 2b124cde5f4e..429df4e81f9a 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alicloud_manager.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_manager.go @@ -27,7 +27,6 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/alicloud/alibaba-cloud-sdk-go/services/ess" klog "k8s.io/klog/v2" - kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "math/rand" "time" ) @@ -237,13 +236,13 @@ func (m *AliCloudManager) buildNodeFromTemplate(sg *Asg, template *sgTemplate) ( func buildGenericLabels(template *sgTemplate, nodeName string) map[string]string { result := make(map[string]string) - result[kubeletapis.LabelArch] = cloudprovider.DefaultArch - result[kubeletapis.LabelOS] = cloudprovider.DefaultOS + result[apiv1.LabelArchStable] = cloudprovider.DefaultArch + result[apiv1.LabelOSStable] = cloudprovider.DefaultOS - result[apiv1.LabelInstanceType] = template.InstanceType.instanceTypeID + result[apiv1.LabelInstanceTypeStable] = template.InstanceType.instanceTypeID - result[apiv1.LabelZoneRegion] = template.Region - result[apiv1.LabelZoneFailureDomain] = template.Zone + result[apiv1.LabelTopologyRegion] = template.Region + result[apiv1.LabelTopologyZone] = template.Zone result[apiv1.LabelHostname] = nodeName // append custom node labels diff --git a/cluster-autoscaler/cloudprovider/alicloud/alicloud_manager_test.go b/cluster-autoscaler/cloudprovider/alicloud/alicloud_manager_test.go index 5f15427d2109..fe5c06141394 100644 --- a/cluster-autoscaler/cloudprovider/alicloud/alicloud_manager_test.go +++ b/cluster-autoscaler/cloudprovider/alicloud/alicloud_manager_test.go @@ -36,7 +36,7 @@ func TestBuildGenericLabels(t *testing.T) { } nodeName := "virtual-node" labels := buildGenericLabels(template, nodeName) - assert.Equal(t, labels[apiv1.LabelInstanceType], template.InstanceType.instanceTypeID) + assert.Equal(t, labels[apiv1.LabelInstanceTypeStable], template.InstanceType.instanceTypeID) } func TestExtractLabelsFromAsg(t *testing.T) { diff --git a/cluster-autoscaler/cloudprovider/aws/CA_with_AWS_IAM_OIDC.md b/cluster-autoscaler/cloudprovider/aws/CA_with_AWS_IAM_OIDC.md index 217bd3d505eb..53db71aa6cc5 100644 --- a/cluster-autoscaler/cloudprovider/aws/CA_with_AWS_IAM_OIDC.md +++ b/cluster-autoscaler/cloudprovider/aws/CA_with_AWS_IAM_OIDC.md @@ -65,6 +65,7 @@ Note: The keys for the tags that you entered don't have values. Cluster Autoscal "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeAutoScalingInstances", "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeScalingActivities", "autoscaling:DescribeTags", "autoscaling:SetDesiredCapacity", "autoscaling:TerminateInstanceInAutoScalingGroup" diff --git a/cluster-autoscaler/cloudprovider/aws/README.md b/cluster-autoscaler/cloudprovider/aws/README.md index e4e6e2684ece..7fe439468b83 100644 --- a/cluster-autoscaler/cloudprovider/aws/README.md +++ b/cluster-autoscaler/cloudprovider/aws/README.md @@ -27,6 +27,7 @@ The following policy provides the minimum privileges necessary for Cluster Autos "autoscaling:DescribeAutoScalingGroups", "autoscaling:DescribeAutoScalingInstances", "autoscaling:DescribeLaunchConfigurations", + "autoscaling:DescribeScalingActivities", "autoscaling:SetDesiredCapacity", "autoscaling:TerminateInstanceInAutoScalingGroup" ], diff --git a/cluster-autoscaler/cloudprovider/aws/auto_scaling.go b/cluster-autoscaler/cloudprovider/aws/auto_scaling.go deleted file mode 100644 index 81a0d4e45ae5..000000000000 --- a/cluster-autoscaler/cloudprovider/aws/auto_scaling.go +++ /dev/null @@ -1,263 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package aws - -import ( - "fmt" - "sync" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/autoscaling" - "k8s.io/apimachinery/pkg/util/clock" - "k8s.io/apimachinery/pkg/util/rand" - "k8s.io/client-go/tools/cache" - klog "k8s.io/klog/v2" -) - -const ( - launchConfigurationCachedTTL = time.Minute * 20 - cacheMinTTL = 120 - cacheMaxTTL = 600 -) - -// autoScaling is the interface represents a specific aspect of the auto-scaling service provided by AWS SDK for use in CA -type autoScaling interface { - DescribeAutoScalingGroupsPages(input *autoscaling.DescribeAutoScalingGroupsInput, fn func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) error - DescribeLaunchConfigurations(*autoscaling.DescribeLaunchConfigurationsInput) (*autoscaling.DescribeLaunchConfigurationsOutput, error) - DescribeTagsPages(input *autoscaling.DescribeTagsInput, fn func(*autoscaling.DescribeTagsOutput, bool) bool) error - SetDesiredCapacity(input *autoscaling.SetDesiredCapacityInput) (*autoscaling.SetDesiredCapacityOutput, error) - TerminateInstanceInAutoScalingGroup(input *autoscaling.TerminateInstanceInAutoScalingGroupInput) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error) -} - -// autoScalingWrapper provides several utility methods over the auto-scaling service provided by AWS SDK -type autoScalingWrapper struct { - autoScaling - launchConfigurationInstanceTypeCache *expirationStore -} - -// expirationStore cache the launch configuration with their instance type. -// The store expires its keys based on a TTL. This TTL can have a jitter applied to it. -// This allows to get a better repartition of the AWS queries. -type expirationStore struct { - cache.Store - jitterClock *jitterClock -} - -type instanceTypeCachedObject struct { - name string - instanceType string -} - -type jitterClock struct { - clock.Clock - - jitter bool - sync.RWMutex -} - -func newLaunchConfigurationInstanceTypeCache() *expirationStore { - jc := &jitterClock{} - return &expirationStore{ - cache.NewExpirationStore(func(obj interface{}) (s string, e error) { - return obj.(instanceTypeCachedObject).name, nil - }, &cache.TTLPolicy{ - TTL: launchConfigurationCachedTTL, - Clock: jc, - }), - jc, - } -} - -func (c *jitterClock) Since(ts time.Time) time.Duration { - since := time.Since(ts) - c.RLock() - defer c.RUnlock() - if c.jitter { - return since + (time.Second * time.Duration(rand.IntnRange(cacheMinTTL, cacheMaxTTL))) - } - return since -} - -func (m autoScalingWrapper) getInstanceTypeByLCNames(launchConfigToQuery []*string) ([]*autoscaling.LaunchConfiguration, error) { - var launchConfigurations []*autoscaling.LaunchConfiguration - - for i := 0; i < len(launchConfigToQuery); i += 50 { - end := i + 50 - - if end > len(launchConfigToQuery) { - end = len(launchConfigToQuery) - } - params := &autoscaling.DescribeLaunchConfigurationsInput{ - LaunchConfigurationNames: launchConfigToQuery[i:end], - MaxRecords: aws.Int64(50), - } - r, err := m.DescribeLaunchConfigurations(params) - if err != nil { - return nil, err - } - launchConfigurations = append(launchConfigurations, r.LaunchConfigurations...) - for _, lc := range r.LaunchConfigurations { - _ = m.launchConfigurationInstanceTypeCache.Add(instanceTypeCachedObject{ - name: *lc.LaunchConfigurationName, - instanceType: *lc.InstanceType, - }) - } - } - return launchConfigurations, nil -} - -func (m autoScalingWrapper) getInstanceTypeByLCName(name string) (string, error) { - if obj, found, _ := m.launchConfigurationInstanceTypeCache.GetByKey(name); found { - return obj.(instanceTypeCachedObject).instanceType, nil - } - - launchConfigs, err := m.getInstanceTypeByLCNames([]*string{aws.String(name)}) - if err != nil { - klog.Errorf("Failed to query the launch configuration %s to get the instance type: %v", name, err) - return "", err - } - if len(launchConfigs) < 1 || launchConfigs[0].InstanceType == nil { - return "", fmt.Errorf("unable to get first LaunchConfiguration for %s", name) - } - return *launchConfigs[0].InstanceType, nil -} - -func (m *autoScalingWrapper) getAutoscalingGroupsByNames(names []string) ([]*autoscaling.Group, error) { - if len(names) == 0 { - return nil, nil - } - - asgs := make([]*autoscaling.Group, 0) - - // AWS only accepts up to 50 ASG names as input, describe them in batches - for i := 0; i < len(names); i += maxAsgNamesPerDescribe { - end := i + maxAsgNamesPerDescribe - - if end > len(names) { - end = len(names) - } - - input := &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice(names[i:end]), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), - } - if err := m.DescribeAutoScalingGroupsPages(input, func(output *autoscaling.DescribeAutoScalingGroupsOutput, _ bool) bool { - asgs = append(asgs, output.AutoScalingGroups...) - // We return true while we want to be called with the next page of - // results, if any. - return true - }); err != nil { - return nil, err - } - } - - return asgs, nil -} - -func (m autoScalingWrapper) populateLaunchConfigurationInstanceTypeCache(autoscalingGroups []*autoscaling.Group) error { - var launchConfigToQuery []*string - - m.launchConfigurationInstanceTypeCache.jitterClock.Lock() - m.launchConfigurationInstanceTypeCache.jitterClock.jitter = true - m.launchConfigurationInstanceTypeCache.jitterClock.Unlock() - for _, asg := range autoscalingGroups { - if asg == nil { - continue - } - if asg.LaunchConfigurationName == nil { - continue - } - _, found, _ := m.launchConfigurationInstanceTypeCache.GetByKey(*asg.LaunchConfigurationName) - if found { - continue - } - launchConfigToQuery = append(launchConfigToQuery, asg.LaunchConfigurationName) - } - m.launchConfigurationInstanceTypeCache.jitterClock.Lock() - m.launchConfigurationInstanceTypeCache.jitterClock.jitter = false - m.launchConfigurationInstanceTypeCache.jitterClock.Unlock() - - // List expire old entries - _ = m.launchConfigurationInstanceTypeCache.List() - - if len(launchConfigToQuery) == 0 { - klog.V(4).Infof("%d launch configurations already in cache", len(autoscalingGroups)) - return nil - } - klog.V(4).Infof("%d launch configurations to query", len(launchConfigToQuery)) - - _, err := m.getInstanceTypeByLCNames(launchConfigToQuery) - if err != nil { - klog.Errorf("Failed to query %d launch configurations", len(launchConfigToQuery)) - return err - } - - klog.V(4).Infof("Successfully query %d launch configurations", len(launchConfigToQuery)) - return nil -} - -func (m *autoScalingWrapper) getAutoscalingGroupNamesByTags(kvs map[string]string) ([]string, error) { - // DescribeTags does an OR query when multiple filters on different tags are - // specified. In other words, DescribeTags returns [asg1, asg1] for keys - // [t1, t2] when there's only one asg tagged both t1 and t2. - filters := []*autoscaling.Filter{} - for key, value := range kvs { - filter := &autoscaling.Filter{ - Name: aws.String("key"), - Values: []*string{aws.String(key)}, - } - filters = append(filters, filter) - if value != "" { - filters = append(filters, &autoscaling.Filter{ - Name: aws.String("value"), - Values: []*string{aws.String(value)}, - }) - } - } - - tags := []*autoscaling.TagDescription{} - input := &autoscaling.DescribeTagsInput{ - Filters: filters, - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), - } - if err := m.DescribeTagsPages(input, func(out *autoscaling.DescribeTagsOutput, _ bool) bool { - tags = append(tags, out.Tags...) - // We return true while we want to be called with the next page of - // results, if any. - return true - }); err != nil { - return nil, err - } - - // According to how DescribeTags API works, the result contains ASGs which - // not all but only subset of tags are associated. Explicitly select ASGs to - // which all the tags are associated so that we won't end up calling - // DescribeAutoScalingGroups API multiple times on an ASG. - asgNames := []string{} - asgNameOccurrences := make(map[string]int) - for _, t := range tags { - asgName := aws.StringValue(t.ResourceId) - occurrences := asgNameOccurrences[asgName] + 1 - if occurrences >= len(kvs) { - asgNames = append(asgNames, asgName) - } - asgNameOccurrences[asgName] = occurrences - } - - return asgNames, nil -} diff --git a/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups.go b/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups.go index 4915fe1b27aa..cae3a35f1955 100644 --- a/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups.go +++ b/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups.go @@ -21,6 +21,7 @@ import ( "reflect" "strings" "sync" + "time" "k8s.io/autoscaler/cluster-autoscaler/config/dynamic" @@ -30,17 +31,20 @@ import ( ) const ( - scaleToZeroSupported = true - placeholderInstanceNamePrefix = "i-placeholder" + scaleToZeroSupported = true + placeholderInstanceNamePrefix = "i-placeholder" + placeholderUnfulfillableStatus = "placeholder-cannot-be-fulfilled" ) type asgCache struct { - registeredAsgs []*asg - asgToInstances map[AwsRef][]AwsInstanceRef - instanceToAsg map[AwsInstanceRef]*asg - mutex sync.Mutex - service autoScalingWrapper - interrupt chan struct{} + registeredAsgs map[AwsRef]*asg + asgToInstances map[AwsRef][]AwsInstanceRef + instanceToAsg map[AwsInstanceRef]*asg + instanceStatus map[AwsInstanceRef]*string + asgInstanceTypeCache *instanceTypeExpirationStore + mutex sync.Mutex + awsService *awsWrapper + interrupt chan struct{} asgAutoDiscoverySpecs []asgAutoDiscoveryConfig explicitlyConfigured map[AwsRef]bool @@ -59,9 +63,10 @@ type mixedInstancesPolicy struct { type asg struct { AwsRef - minSize int - maxSize int - curSize int + minSize int + maxSize int + curSize int + lastUpdateTime time.Time AvailabilityZones []string LaunchConfigurationName string @@ -70,12 +75,14 @@ type asg struct { Tags []*autoscaling.TagDescription } -func newASGCache(service autoScalingWrapper, explicitSpecs []string, autoDiscoverySpecs []asgAutoDiscoveryConfig) (*asgCache, error) { +func newASGCache(awsService *awsWrapper, explicitSpecs []string, autoDiscoverySpecs []asgAutoDiscoveryConfig) (*asgCache, error) { registry := &asgCache{ - registeredAsgs: make([]*asg, 0), - service: service, + registeredAsgs: make(map[AwsRef]*asg, 0), + awsService: awsService, asgToInstances: make(map[AwsRef][]AwsInstanceRef), instanceToAsg: make(map[AwsInstanceRef]*asg), + instanceStatus: make(map[AwsInstanceRef]*string), + asgInstanceTypeCache: newAsgInstanceTypeCache(awsService), interrupt: make(chan struct{}), asgAutoDiscoverySpecs: autoDiscoverySpecs, explicitlyConfigured: make(map[AwsRef]bool), @@ -88,6 +95,17 @@ func newASGCache(service autoScalingWrapper, explicitSpecs []string, autoDiscove return registry, nil } +// Use a function variable for ease of testing +var getInstanceTypeForAsg = func(m *asgCache, group *asg) (string, error) { + if obj, found, _ := m.asgInstanceTypeCache.GetByKey(group.AwsRef.Name); found { + return obj.(instanceTypeCachedObject).instanceType, nil + } else if result, err := m.awsService.getInstanceTypesForAsgs([]*asg{group}); err == nil { + return result[group.AwsRef.Name], nil + } + + return "", fmt.Errorf("Could not find instance type for %s", group.AwsRef.Name) +} + // Fetch explicitly configured ASGs. These ASGs should never be unregistered // during refreshes, even if they no longer exist in AWS. func (m *asgCache) parseExplicitAsgs(specs []string) error { @@ -105,53 +123,44 @@ func (m *asgCache) parseExplicitAsgs(specs []string) error { // Register ASG. Returns the registered ASG. func (m *asgCache) register(asg *asg) *asg { - for i := range m.registeredAsgs { - if existing := m.registeredAsgs[i]; existing.AwsRef == asg.AwsRef { - if reflect.DeepEqual(existing, asg) { - return existing - } + if existing, asgExists := m.registeredAsgs[asg.AwsRef]; asgExists { + if reflect.DeepEqual(existing, asg) { + return existing + } - klog.V(4).Infof("Updating ASG %s", asg.AwsRef.Name) + klog.V(4).Infof("Updating ASG %s", asg.AwsRef.Name) - // Explicit registered groups should always use the manually provided min/max - // values and the not the ones returned by the API - if !m.explicitlyConfigured[asg.AwsRef] { - existing.minSize = asg.minSize - existing.maxSize = asg.maxSize - } + // Explicit registered groups should always use the manually provided min/max + // values and the not the ones returned by the API + if !m.explicitlyConfigured[asg.AwsRef] { + existing.minSize = asg.minSize + existing.maxSize = asg.maxSize + } - existing.curSize = asg.curSize + existing.curSize = asg.curSize - // Those information are mainly required to create templates when scaling - // from zero - existing.AvailabilityZones = asg.AvailabilityZones - existing.LaunchConfigurationName = asg.LaunchConfigurationName - existing.LaunchTemplate = asg.LaunchTemplate - existing.MixedInstancesPolicy = asg.MixedInstancesPolicy - existing.Tags = asg.Tags + // Those information are mainly required to create templates when scaling + // from zero + existing.AvailabilityZones = asg.AvailabilityZones + existing.LaunchConfigurationName = asg.LaunchConfigurationName + existing.LaunchTemplate = asg.LaunchTemplate + existing.MixedInstancesPolicy = asg.MixedInstancesPolicy + existing.Tags = asg.Tags - return existing - } + return existing } klog.V(1).Infof("Registering ASG %s", asg.AwsRef.Name) - m.registeredAsgs = append(m.registeredAsgs, asg) + m.registeredAsgs[asg.AwsRef] = asg return asg } // Unregister ASG. Returns the unregistered ASG. func (m *asgCache) unregister(a *asg) *asg { - updated := make([]*asg, 0, len(m.registeredAsgs)) - var changed *asg - for _, existing := range m.registeredAsgs { - if existing.AwsRef == a.AwsRef { - klog.V(1).Infof("Unregistered ASG %s", a.AwsRef.Name) - changed = a - continue - } - updated = append(updated, existing) + if _, asgExists := m.registeredAsgs[a.AwsRef]; asgExists { + klog.V(1).Infof("Unregistered ASG %s", a.AwsRef.Name) + delete(m.registeredAsgs, a.AwsRef) } - m.registeredAsgs = updated - return changed + return a } func (m *asgCache) buildAsgFromSpec(spec string) (*asg, error) { @@ -168,7 +177,7 @@ func (m *asgCache) buildAsgFromSpec(spec string) (*asg, error) { } // Get returns the currently registered ASGs -func (m *asgCache) Get() []*asg { +func (m *asgCache) Get() map[AwsRef]*asg { m.mutex.Lock() defer m.mutex.Unlock() @@ -203,6 +212,17 @@ func (m *asgCache) InstancesByAsg(ref AwsRef) ([]AwsInstanceRef, error) { return nil, fmt.Errorf("error while looking for instances of ASG: %s", ref) } +func (m *asgCache) InstanceStatus(ref AwsInstanceRef) (*string, error) { + m.mutex.Lock() + defer m.mutex.Unlock() + + if status, found := m.instanceStatus[ref]; found { + return status, nil + } + + return nil, fmt.Errorf("could not find instance %v", ref) +} + func (m *asgCache) SetAsgSize(asg *asg, size int) error { m.mutex.Lock() defer m.mutex.Unlock() @@ -217,12 +237,15 @@ func (m *asgCache) setAsgSizeNoLock(asg *asg, size int) error { HonorCooldown: aws.Bool(false), } klog.V(0).Infof("Setting asg %s size to %d", asg.Name, size) - _, err := m.service.SetDesiredCapacity(params) + start := time.Now() + _, err := m.awsService.SetDesiredCapacity(params) + observeAWSRequest("SetDesiredCapacity", err, start) if err != nil { return err } // Proactively set the ASG size so autoscaler makes better decisions + asg.lastUpdateTime = start asg.curSize = size return nil @@ -270,7 +293,9 @@ func (m *asgCache) DeleteInstances(instances []*AwsInstanceRef) error { InstanceId: aws.String(instance.Name), ShouldDecrementDesiredCapacity: aws.Bool(true), } - resp, err := m.service.TerminateInstanceInAutoScalingGroup(params) + start := time.Now() + resp, err := m.awsService.TerminateInstanceInAutoScalingGroup(params) + observeAWSRequest("TerminateInstanceInAutoScalingGroup", err, start) if err != nil { return err } @@ -294,7 +319,7 @@ func (m *asgCache) fetchAutoAsgNames() ([]string, error) { groupNames := make([]string, 0) for _, spec := range m.asgAutoDiscoverySpecs { - names, err := m.service.getAutoscalingGroupNamesByTags(spec.Tags) + names, err := m.awsService.getAutoscalingGroupNamesByTags(spec.Tags) if err != nil { return nil, fmt.Errorf("cannot autodiscover ASGs: %s", err) } @@ -340,8 +365,9 @@ func (m *asgCache) regenerate() error { newInstanceToAsgCache := make(map[AwsInstanceRef]*asg) newAsgToInstancesCache := make(map[AwsRef][]AwsInstanceRef) + newInstanceStatusMap := make(map[AwsInstanceRef]*string) - // Build list of knowns ASG names + // Build list of known ASG names refreshNames, err := m.buildAsgNames() if err != nil { return err @@ -349,16 +375,11 @@ func (m *asgCache) regenerate() error { // Fetch details of all ASGs klog.V(4).Infof("Regenerating instance to ASG map for ASGs: %v", refreshNames) - groups, err := m.service.getAutoscalingGroupsByNames(refreshNames) + groups, err := m.awsService.getAutoscalingGroupsByNames(refreshNames) if err != nil { return err } - err = m.service.populateLaunchConfigurationInstanceTypeCache(groups) - if err != nil { - klog.Warningf("Failed to fully populate all launchConfigurations: %v", err) - } - // If currently any ASG has more Desired than running Instances, introduce placeholders // for the instances to come up. This is required to track Desired instances that // will never come up, like with Spot Request that can't be fulfilled @@ -381,6 +402,7 @@ func (m *asgCache) regenerate() error { ref := m.buildInstanceRefFromAWS(instance) newInstanceToAsgCache[ref] = asg newAsgToInstancesCache[asg.AwsRef][i] = ref + newInstanceStatusMap[ref] = instance.HealthStatus } } @@ -391,32 +413,80 @@ func (m *asgCache) regenerate() error { } } + err = m.asgInstanceTypeCache.populate(m.registeredAsgs) + if err != nil { + klog.Warningf("Failed to fully populate ASG->instanceType mapping: %v", err) + } + m.asgToInstances = newAsgToInstancesCache m.instanceToAsg = newInstanceToAsgCache + m.instanceStatus = newInstanceStatusMap return nil } func (m *asgCache) createPlaceholdersForDesiredNonStartedInstances(groups []*autoscaling.Group) []*autoscaling.Group { for _, g := range groups { desired := *g.DesiredCapacity - real := int64(len(g.Instances)) - if desired <= real { + realInstances := int64(len(g.Instances)) + if desired <= realInstances { continue } - for i := real; i < desired; i++ { + klog.V(4).Infof("Instance group %s has only %d instances created while requested count is %d. "+ + "Creating placeholder instances.", *g.AutoScalingGroupName, realInstances, desired) + + healthStatus := "" + isAvailable, err := m.isNodeGroupAvailable(g) + if err != nil { + klog.V(4).Infof("Could not check instance availability, creating placeholder node anyways: %v", err) + } else if !isAvailable { + klog.Warningf("Instance group %s cannot provision any more nodes!", *g.AutoScalingGroupName) + healthStatus = placeholderUnfulfillableStatus + } + + for i := realInstances; i < desired; i++ { id := fmt.Sprintf("%s-%s-%d", placeholderInstanceNamePrefix, *g.AutoScalingGroupName, i) klog.V(4).Infof("Instance group %s has only %d instances created while requested count is %d. "+ - "Creating placeholder instance with ID %s.", *g.AutoScalingGroupName, real, desired, id) + "Creating placeholder instance with ID %s.", *g.AutoScalingGroupName, realInstances, desired, id) g.Instances = append(g.Instances, &autoscaling.Instance{ InstanceId: &id, AvailabilityZone: g.AvailabilityZones[0], + HealthStatus: &healthStatus, }) } } return groups } +func (m *asgCache) isNodeGroupAvailable(group *autoscaling.Group) (bool, error) { + input := &autoscaling.DescribeScalingActivitiesInput{ + AutoScalingGroupName: group.AutoScalingGroupName, + } + + start := time.Now() + response, err := m.awsService.DescribeScalingActivities(input) + observeAWSRequest("DescribeScalingActivities", err, start) + if err != nil { + return true, err // If we can't describe the scaling activities we assume the node group is available + } + + for _, activity := range response.Activities { + asgRef := AwsRef{Name: *group.AutoScalingGroupName} + if a, ok := m.registeredAsgs[asgRef]; ok { + lut := a.lastUpdateTime + if activity.StartTime.Before(lut) { + break + } else if *activity.StatusCode == "Failed" { + klog.Warningf("ASG %s scaling failed with %s", asgRef.Name, *activity) + return false, nil + } + } else { + klog.V(4).Infof("asg %v is not registered yet, skipping DescribeScalingActivities check", asgRef.Name) + } + } + return true, nil +} + func (m *asgCache) buildAsgFromAWS(g *autoscaling.Group) (*asg, error) { spec := dynamic.NodeGroupSpec{ Name: aws.StringValue(g.AutoScalingGroupName), @@ -441,7 +511,7 @@ func (m *asgCache) buildAsgFromAWS(g *autoscaling.Group) (*asg, error) { } if g.LaunchTemplate != nil { - asg.LaunchTemplate = m.buildLaunchTemplateFromSpec(g.LaunchTemplate) + asg.LaunchTemplate = buildLaunchTemplateFromSpec(g.LaunchTemplate) } if g.MixedInstancesPolicy != nil { @@ -454,7 +524,7 @@ func (m *asgCache) buildAsgFromAWS(g *autoscaling.Group) (*asg, error) { } asg.MixedInstancesPolicy = &mixedInstancesPolicy{ - launchTemplate: m.buildLaunchTemplateFromSpec(g.MixedInstancesPolicy.LaunchTemplate.LaunchTemplateSpecification), + launchTemplate: buildLaunchTemplateFromSpec(g.MixedInstancesPolicy.LaunchTemplate.LaunchTemplateSpecification), instanceTypesOverrides: getInstanceTypes(g.MixedInstancesPolicy.LaunchTemplate.Overrides), } } @@ -462,36 +532,6 @@ func (m *asgCache) buildAsgFromAWS(g *autoscaling.Group) (*asg, error) { return asg, nil } -func (m *asgCache) buildLaunchTemplateFromSpec(ltSpec *autoscaling.LaunchTemplateSpecification) *launchTemplate { - // NOTE(jaypipes): The LaunchTemplateSpecification.Version is a pointer to - // string. When the pointer is nil, EC2 AutoScaling API considers the value - // to be "$Default", however aws.StringValue(ltSpec.Version) will return an - // empty string (which is not considered the same as "$Default" or a nil - // string pointer. So, in order to not pass an empty string as the version - // for the launch template when we communicate with the EC2 AutoScaling API - // using the information in the launchTemplate, we store the string - // "$Default" here when the ltSpec.Version is a nil pointer. - // - // See: - // - // https://github.com/kubernetes/autoscaler/issues/1728 - // https://github.com/aws/aws-sdk-go/blob/81fad3b797f4a9bd1b452a5733dd465eefef1060/service/autoscaling/api.go#L10666-L10671 - // - // A cleaner alternative might be to make launchTemplate.version a string - // pointer instead of a string, or even store the aws-sdk-go's - // LaunchTemplateSpecification structs directly. - var version string - if ltSpec.Version == nil { - version = "$Default" - } else { - version = aws.StringValue(ltSpec.Version) - } - return &launchTemplate{ - name: aws.StringValue(ltSpec.LaunchTemplateName), - version: version, - } -} - func (m *asgCache) buildInstanceRefFromAWS(instance *autoscaling.Instance) AwsInstanceRef { providerID := fmt.Sprintf("aws:///%s/%s", aws.StringValue(instance.AvailabilityZone), aws.StringValue(instance.InstanceId)) return AwsInstanceRef{ diff --git a/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups_test.go b/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups_test.go index 9efb50f7cc5d..c5078b03b5b7 100644 --- a/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups_test.go +++ b/cluster-autoscaler/cloudprovider/aws/auto_scaling_groups_test.go @@ -17,12 +17,13 @@ limitations under the License. package aws import ( + "errors" "testing" + "time" - "github.com/stretchr/testify/assert" - - sdkaws "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/stretchr/testify/assert" ) func TestBuildAsg(t *testing.T) { @@ -50,63 +51,118 @@ func validateAsg(t *testing.T, asg *asg, name string, minSize int, maxSize int) assert.Equal(t, maxSize, asg.maxSize) } -func TestBuildLaunchTemplateFromSpec(t *testing.T) { - assert := assert.New(t) +func TestCreatePlaceholders(t *testing.T) { + registeredAsgName := aws.String("test-asg") + registeredAsgRef := AwsRef{Name: *registeredAsgName} - units := []struct { - name string - in *autoscaling.LaunchTemplateSpecification - exp *launchTemplate + cases := []struct { + name string + desiredCapacity *int64 + activities []*autoscaling.Activity + groupLastUpdateTime time.Time + describeErr error + asgToCheck *string }{ { - name: "non-default, specified version", - in: &autoscaling.LaunchTemplateSpecification{ - LaunchTemplateName: sdkaws.String("foo"), - Version: sdkaws.String("1"), - }, - exp: &launchTemplate{ - name: "foo", - version: "1", - }, + name: "add placeholders successful", + desiredCapacity: aws.Int64(10), }, { - name: "non-default, specified $Latest", - in: &autoscaling.LaunchTemplateSpecification{ - LaunchTemplateName: sdkaws.String("foo"), - Version: sdkaws.String("$Latest"), - }, - exp: &launchTemplate{ - name: "foo", - version: "$Latest", - }, + name: "no placeholders needed", + desiredCapacity: aws.Int64(0), }, { - name: "specified $Default", - in: &autoscaling.LaunchTemplateSpecification{ - LaunchTemplateName: sdkaws.String("foo"), - Version: sdkaws.String("$Default"), - }, - exp: &launchTemplate{ - name: "foo", - version: "$Default", + name: "DescribeScalingActivities failed", + desiredCapacity: aws.Int64(1), + describeErr: errors.New("timeout"), + }, + { + name: "early abort if AWS scaling up fails", + desiredCapacity: aws.Int64(1), + activities: []*autoscaling.Activity{ + { + StatusCode: aws.String("Failed"), + StartTime: aws.Time(time.Unix(10, 0)), + }, }, + groupLastUpdateTime: time.Unix(9, 0), }, { - name: "no version specified", - in: &autoscaling.LaunchTemplateSpecification{ - LaunchTemplateName: sdkaws.String("foo"), - Version: nil, + name: "AWS scaling failed event before CA scale_up", + desiredCapacity: aws.Int64(1), + activities: []*autoscaling.Activity{ + { + StatusCode: aws.String("Failed"), + StartTime: aws.Time(time.Unix(9, 0)), + }, }, - exp: &launchTemplate{ - name: "foo", - version: "$Default", + groupLastUpdateTime: time.Unix(10, 0), + }, + { + name: "asg not registered", + desiredCapacity: aws.Int64(10), + activities: []*autoscaling.Activity{ + { + StatusCode: aws.String("Failed"), + StartTime: aws.Time(time.Unix(10, 0)), + }, }, + groupLastUpdateTime: time.Unix(9, 0), + asgToCheck: aws.String("unregisteredAsgName"), }, } - cache := &asgCache{} - for _, unit := range units { - got := cache.buildLaunchTemplateFromSpec(unit.in) - assert.Equal(unit.exp, got) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + shouldCallDescribeScalingActivities := true + if *tc.desiredCapacity == int64(0) { + shouldCallDescribeScalingActivities = false + } + + asgName := registeredAsgName + if tc.asgToCheck != nil { + asgName = tc.asgToCheck + } + + a := &autoScalingMock{} + if shouldCallDescribeScalingActivities { + a.On("DescribeScalingActivities", &autoscaling.DescribeScalingActivitiesInput{ + AutoScalingGroupName: asgName, + }).Return( + &autoscaling.DescribeScalingActivitiesOutput{Activities: tc.activities}, + tc.describeErr, + ).Once() + } + + asgCache := &asgCache{ + awsService: &awsWrapper{ + autoScalingI: a, + ec2I: nil, + }, + registeredAsgs: map[AwsRef]*asg{ + registeredAsgRef: { + AwsRef: registeredAsgRef, + lastUpdateTime: tc.groupLastUpdateTime, + }, + }, + } + + groups := []*autoscaling.Group{ + { + AutoScalingGroupName: asgName, + AvailabilityZones: []*string{aws.String("westeros-1a")}, + DesiredCapacity: tc.desiredCapacity, + Instances: []*autoscaling.Instance{}, + }, + } + asgCache.createPlaceholdersForDesiredNonStartedInstances(groups) + assert.Equal(t, int64(len(groups[0].Instances)), *tc.desiredCapacity) + if tc.activities != nil && *tc.activities[0].StatusCode == "Failed" && tc.activities[0].StartTime.After(tc.groupLastUpdateTime) && asgName == registeredAsgName { + assert.Equal(t, *groups[0].Instances[0].HealthStatus, placeholderUnfulfillableStatus) + } else if len(groups[0].Instances) > 0 { + assert.Equal(t, *groups[0].Instances[0].HealthStatus, "") + } + a.AssertExpectations(t) + }) } } diff --git a/cluster-autoscaler/cloudprovider/aws/auto_scaling_test.go b/cluster-autoscaler/cloudprovider/aws/auto_scaling_test.go deleted file mode 100644 index 8b85887574e8..000000000000 --- a/cluster-autoscaler/cloudprovider/aws/auto_scaling_test.go +++ /dev/null @@ -1,84 +0,0 @@ -/* -Copyright 2016 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package aws - -import ( - "fmt" - "testing" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/autoscaling" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" -) - -func TestMoreThen100Groups(t *testing.T) { - service := &AutoScalingMock{} - autoScalingWrapper := &autoScalingWrapper{ - autoScaling: service, - } - - // Generate 101 ASG names - names := make([]string, 101) - for i := 0; i < len(names); i++ { - names[i] = fmt.Sprintf("asg-%d", i) - } - - // First batch, first 100 elements - service.On("DescribeAutoScalingGroupsPages", - &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice(names[:100]), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), - }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testNamedDescribeAutoScalingGroupsOutput("asg-1", 1, "test-instance-id"), false) - }).Return(nil) - - // Second batch, element 101 - service.On("DescribeAutoScalingGroupsPages", - &autoscaling.DescribeAutoScalingGroupsInput{ - AutoScalingGroupNames: aws.StringSlice([]string{"asg-100"}), - MaxRecords: aws.Int64(maxRecordsReturnedByAPI), - }, - mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), - ).Run(func(args mock.Arguments) { - fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) - fn(testNamedDescribeAutoScalingGroupsOutput("asg-2", 1, "test-instance-id"), false) - }).Return(nil) - - asgs, err := autoScalingWrapper.getAutoscalingGroupsByNames(names) - assert.Nil(t, err) - assert.Equal(t, len(asgs), 2) - assert.Equal(t, *asgs[0].AutoScalingGroupName, "asg-1") - assert.Equal(t, *asgs[1].AutoScalingGroupName, "asg-2") -} - -func TestLaunchConfigurationCache(t *testing.T) { - c := newLaunchConfigurationInstanceTypeCache() - err := c.Add(instanceTypeCachedObject{ - name: "123", - instanceType: "t2.medium", - }) - require.NoError(t, err) - obj, ok, err := c.GetByKey("123") - require.NoError(t, err) - require.True(t, ok) - require.Equal(t, "t2.medium", obj.(instanceTypeCachedObject).instanceType) -} diff --git a/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go b/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go index d5ae80c29093..99fe9fd17bd4 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go @@ -84,12 +84,12 @@ func (aws *awsCloudProvider) GetAvailableGPUTypes() map[string]struct{} { // NodeGroups returns all node groups configured for this cloud provider. func (aws *awsCloudProvider) NodeGroups() []cloudprovider.NodeGroup { asgs := aws.awsManager.getAsgs() - ngs := make([]cloudprovider.NodeGroup, len(asgs)) - for i, asg := range asgs { - ngs[i] = &AwsNodeGroup{ + ngs := make([]cloudprovider.NodeGroup, 0, len(asgs)) + for _, asg := range asgs { + ngs = append(ngs, &AwsNodeGroup{ asg: asg, awsManager: aws.awsManager, - } + }) } return ngs @@ -309,7 +309,24 @@ func (ng *AwsNodeGroup) Nodes() ([]cloudprovider.Instance, error) { instances := make([]cloudprovider.Instance, len(asgNodes)) for i, asgNode := range asgNodes { - instances[i] = cloudprovider.Instance{Id: asgNode.ProviderID} + var status *cloudprovider.InstanceStatus + instanceStatusString, err := ng.awsManager.GetInstanceStatus(asgNode) + if err != nil { + klog.V(4).Infof("Could not get instance status, continuing anyways: %v", err) + } else if instanceStatusString != nil && *instanceStatusString == placeholderUnfulfillableStatus { + status = &cloudprovider.InstanceStatus{ + State: cloudprovider.InstanceCreating, + ErrorInfo: &cloudprovider.InstanceErrorInfo{ + ErrorClass: cloudprovider.OutOfResourcesErrorClass, + ErrorCode: placeholderUnfulfillableStatus, + ErrorMessage: "AWS cannot provision any more instances for this node group", + }, + } + } + instances[i] = cloudprovider.Instance{ + Id: asgNode.ProviderID, + Status: status, + } } return instances, nil } @@ -386,5 +403,6 @@ func BuildAWS(opts config.AutoscalingOptions, do cloudprovider.NodeGroupDiscover if err != nil { klog.Fatalf("Failed to create AWS cloud provider: %v", err) } + RegisterMetrics() return provider } diff --git a/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go b/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go index 587833779335..e6b6d4137608 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go @@ -21,88 +21,48 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/autoscaling" - "github.com/aws/aws-sdk-go/service/ec2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" apiv1 "k8s.io/api/core/v1" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" ) -type AutoScalingMock struct { - mock.Mock -} - -func (a *AutoScalingMock) DescribeAutoScalingGroupsPages(i *autoscaling.DescribeAutoScalingGroupsInput, fn func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) error { - args := a.Called(i, fn) - return args.Error(0) -} - -func (a *AutoScalingMock) DescribeLaunchConfigurations(i *autoscaling.DescribeLaunchConfigurationsInput) (*autoscaling.DescribeLaunchConfigurationsOutput, error) { - args := a.Called(i) - return args.Get(0).(*autoscaling.DescribeLaunchConfigurationsOutput), nil -} - -func (a *AutoScalingMock) DescribeTagsPages(i *autoscaling.DescribeTagsInput, fn func(*autoscaling.DescribeTagsOutput, bool) bool) error { - args := a.Called(i, fn) - return args.Error(0) -} - -func (a *AutoScalingMock) SetDesiredCapacity(input *autoscaling.SetDesiredCapacityInput) (*autoscaling.SetDesiredCapacityOutput, error) { - args := a.Called(input) - return args.Get(0).(*autoscaling.SetDesiredCapacityOutput), nil -} - -func (a *AutoScalingMock) TerminateInstanceInAutoScalingGroup(input *autoscaling.TerminateInstanceInAutoScalingGroupInput) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error) { - args := a.Called(input) - return args.Get(0).(*autoscaling.TerminateInstanceInAutoScalingGroupOutput), nil -} - -type EC2Mock struct { - mock.Mock -} - -func (e *EC2Mock) DescribeLaunchTemplateVersions(i *ec2.DescribeLaunchTemplateVersionsInput) (*ec2.DescribeLaunchTemplateVersionsOutput, error) { - args := e.Called(i) - return args.Get(0).(*ec2.DescribeLaunchTemplateVersionsOutput), nil -} - -var testService = autoScalingWrapper{&AutoScalingMock{}, newLaunchConfigurationInstanceTypeCache()} - var testAwsManager = &AwsManager{ asgCache: &asgCache{ - registeredAsgs: make([]*asg, 0), + registeredAsgs: make(map[AwsRef]*asg, 0), asgToInstances: make(map[AwsRef][]AwsInstanceRef), instanceToAsg: make(map[AwsInstanceRef]*asg), interrupt: make(chan struct{}), - service: testService, + awsService: &testAwsService, }, - autoScalingService: testService, + awsService: testAwsService, } -func newTestAwsManagerWithService(service autoScaling, autoDiscoverySpecs []asgAutoDiscoveryConfig) *AwsManager { - wrapper := autoScalingWrapper{service, newLaunchConfigurationInstanceTypeCache()} +func newTestAwsManagerWithMockServices(mockAutoScaling autoScalingI, mockEC2 ec2I, autoDiscoverySpecs []asgAutoDiscoveryConfig) *AwsManager { + awsService := awsWrapper{mockAutoScaling, mockEC2} return &AwsManager{ - autoScalingService: wrapper, + awsService: awsService, asgCache: &asgCache{ - registeredAsgs: make([]*asg, 0), + registeredAsgs: make(map[AwsRef]*asg, 0), asgToInstances: make(map[AwsRef][]AwsInstanceRef), instanceToAsg: make(map[AwsInstanceRef]*asg), + asgInstanceTypeCache: newAsgInstanceTypeCache(&awsService), explicitlyConfigured: make(map[AwsRef]bool), interrupt: make(chan struct{}), asgAutoDiscoverySpecs: autoDiscoverySpecs, - service: wrapper, + awsService: &awsService, }, } } -func newTestAwsManagerWithAsgs(t *testing.T, service autoScaling, specs []string) *AwsManager { - m := newTestAwsManagerWithService(service, nil) +func newTestAwsManagerWithAsgs(t *testing.T, mockAutoScaling autoScalingI, mockEC2 ec2I, specs []string) *AwsManager { + m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, nil) m.asgCache.parseExplicitAsgs(specs) return m } -func newTestAwsManagerWithAutoAsgs(t *testing.T, service autoScaling, specs []string, autoDiscoverySpecs []asgAutoDiscoveryConfig) *AwsManager { - m := newTestAwsManagerWithService(service, autoDiscoverySpecs) +func newTestAwsManagerWithAutoAsgs(t *testing.T, mockAutoScaling autoScalingI, mockEC2 ec2I, specs []string, autoDiscoverySpecs []asgAutoDiscoveryConfig) *AwsManager { + m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, autoDiscoverySpecs) m.asgCache.parseExplicitAsgs(specs) return m } @@ -154,7 +114,7 @@ func TestName(t *testing.T) { } func TestNodeGroups(t *testing.T) { - provider := testProvider(t, newTestAwsManagerWithAsgs(t, testService, []string{"1:5:test-asg"})) + provider := testProvider(t, newTestAwsManagerWithAsgs(t, testAwsService, nil, []string{"1:5:test-asg"})) nodeGroups := provider.NodeGroups() assert.Equal(t, len(nodeGroups), 1) @@ -164,14 +124,14 @@ func TestNodeGroups(t *testing.T) { } func TestAutoDiscoveredNodeGroups(t *testing.T) { - service := &AutoScalingMock{} - provider := testProvider(t, newTestAwsManagerWithAutoAsgs(t, service, []string{}, []asgAutoDiscoveryConfig{ + a := &autoScalingMock{} + provider := testProvider(t, newTestAwsManagerWithAutoAsgs(t, a, nil, []string{}, []asgAutoDiscoveryConfig{ { Tags: map[string]string{"test": ""}, }, })) - service.On("DescribeTagsPages", + a.On("DescribeTagsPages", &autoscaling.DescribeTagsInput{ Filters: []*autoscaling.Filter{ {Name: aws.String("key"), Values: aws.StringSlice([]string{"test"})}, @@ -187,7 +147,7 @@ func TestAutoDiscoveredNodeGroups(t *testing.T) { }}, false) }).Return(nil).Once() - service.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroupsPages", &autoscaling.DescribeAutoScalingGroupsInput{ AutoScalingGroupNames: aws.StringSlice([]string{"auto-asg"}), MaxRecords: aws.Int64(maxRecordsReturnedByAPI), @@ -213,10 +173,10 @@ func TestNodeGroupForNode(t *testing.T) { ProviderID: "aws:///us-east-1a/test-instance-id", }, } - service := &AutoScalingMock{} - provider := testProvider(t, newTestAwsManagerWithAsgs(t, service, []string{"1:5:test-asg"})) + a := &autoScalingMock{} + provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) - service.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroupsPages", &autoscaling.DescribeAutoScalingGroupsInput{ AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), MaxRecords: aws.Int64(maxRecordsReturnedByAPI), @@ -241,7 +201,7 @@ func TestNodeGroupForNode(t *testing.T) { assert.NoError(t, err) assert.Equal(t, []cloudprovider.Instance{{Id: "aws:///us-east-1a/test-instance-id"}}, nodes) - service.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) // test node in cluster that is not in a group managed by cluster autoscaler nodeNotInGroup := &apiv1.Node{ @@ -254,7 +214,7 @@ func TestNodeGroupForNode(t *testing.T) { assert.NoError(t, err) assert.Nil(t, group) - service.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) } func TestNodeGroupForNodeWithNoProviderId(t *testing.T) { @@ -263,8 +223,8 @@ func TestNodeGroupForNodeWithNoProviderId(t *testing.T) { ProviderID: "", }, } - service := &AutoScalingMock{} - provider := testProvider(t, newTestAwsManagerWithAsgs(t, service, []string{"1:5:test-asg"})) + a := &autoScalingMock{} + provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) group, err := provider.NodeGroupForNode(node) assert.NoError(t, err) @@ -325,11 +285,11 @@ func TestAwsRefFromProviderId(t *testing.T) { } func TestTargetSize(t *testing.T) { - service := &AutoScalingMock{} - provider := testProvider(t, newTestAwsManagerWithAsgs(t, service, []string{"1:5:test-asg"})) + a := &autoScalingMock{} + provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) asgs := provider.NodeGroups() - service.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroupsPages", &autoscaling.DescribeAutoScalingGroupsInput{ AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), MaxRecords: aws.Int64(maxRecordsReturnedByAPI), @@ -346,21 +306,21 @@ func TestTargetSize(t *testing.T) { assert.Equal(t, targetSize, 2) assert.NoError(t, err) - service.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) } func TestIncreaseSize(t *testing.T) { - service := &AutoScalingMock{} - provider := testProvider(t, newTestAwsManagerWithAsgs(t, service, []string{"1:5:test-asg"})) + a := &autoScalingMock{} + provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) asgs := provider.NodeGroups() - service.On("SetDesiredCapacity", &autoscaling.SetDesiredCapacityInput{ + a.On("SetDesiredCapacity", &autoscaling.SetDesiredCapacityInput{ AutoScalingGroupName: aws.String(asgs[0].Id()), DesiredCapacity: aws.Int64(3), HonorCooldown: aws.Bool(false), }).Return(&autoscaling.SetDesiredCapacityOutput{}) - service.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroupsPages", &autoscaling.DescribeAutoScalingGroupsInput{ AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), MaxRecords: aws.Int64(maxRecordsReturnedByAPI), @@ -379,8 +339,8 @@ func TestIncreaseSize(t *testing.T) { err = asgs[0].IncreaseSize(1) assert.NoError(t, err) - service.AssertNumberOfCalls(t, "SetDesiredCapacity", 1) - service.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "SetDesiredCapacity", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) newSize, err := asgs[0].TargetSize() assert.NoError(t, err) @@ -388,11 +348,11 @@ func TestIncreaseSize(t *testing.T) { } func TestBelongs(t *testing.T) { - service := &AutoScalingMock{} - provider := testProvider(t, newTestAwsManagerWithAsgs(t, service, []string{"1:5:test-asg"})) + a := &autoScalingMock{} + provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) asgs := provider.NodeGroups() - service.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroupsPages", &autoscaling.DescribeAutoScalingGroupsInput{ AutoScalingGroupNames: aws.StringSlice([]string{asgs[0].Id()}), MaxRecords: aws.Int64(maxRecordsReturnedByAPI), @@ -412,7 +372,7 @@ func TestBelongs(t *testing.T) { } _, err := asgs[0].(*AwsNodeGroup).Belongs(invalidNode) assert.Error(t, err) - service.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) validNode := &apiv1.Node{ Spec: apiv1.NodeSpec{ @@ -425,15 +385,15 @@ func TestBelongs(t *testing.T) { // As "test-instance-id" is already known to be managed by test-asg since // the first `Belongs` call, no additional DescribAutoScalingGroupsPages // call is made. - service.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) } func TestDeleteNodes(t *testing.T) { - service := &AutoScalingMock{} - provider := testProvider(t, newTestAwsManagerWithAsgs(t, service, []string{"1:5:test-asg"})) + a := &autoScalingMock{} + provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) asgs := provider.NodeGroups() - service.On("TerminateInstanceInAutoScalingGroup", &autoscaling.TerminateInstanceInAutoScalingGroupInput{ + a.On("TerminateInstanceInAutoScalingGroup", &autoscaling.TerminateInstanceInAutoScalingGroupInput{ InstanceId: aws.String("test-instance-id"), ShouldDecrementDesiredCapacity: aws.Bool(true), }).Return(&autoscaling.TerminateInstanceInAutoScalingGroupOutput{ @@ -442,7 +402,7 @@ func TestDeleteNodes(t *testing.T) { // Look up the current number of instances... var expectedInstancesCount int64 = 2 - service.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroupsPages", &autoscaling.DescribeAutoScalingGroupsInput{ AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), MaxRecords: aws.Int64(maxRecordsReturnedByAPI), @@ -468,8 +428,8 @@ func TestDeleteNodes(t *testing.T) { } err = asgs[0].DeleteNodes([]*apiv1.Node{node}) assert.NoError(t, err) - service.AssertNumberOfCalls(t, "TerminateInstanceInAutoScalingGroup", 1) - service.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 2) + a.AssertNumberOfCalls(t, "TerminateInstanceInAutoScalingGroup", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) newSize, err := asgs[0].TargetSize() assert.NoError(t, err) @@ -477,11 +437,11 @@ func TestDeleteNodes(t *testing.T) { } func TestDeleteNodesWithPlaceholder(t *testing.T) { - service := &AutoScalingMock{} - provider := testProvider(t, newTestAwsManagerWithAsgs(t, service, []string{"1:5:test-asg"})) + a := &autoScalingMock{} + provider := testProvider(t, newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"})) asgs := provider.NodeGroups() - service.On("SetDesiredCapacity", &autoscaling.SetDesiredCapacityInput{ + a.On("SetDesiredCapacity", &autoscaling.SetDesiredCapacityInput{ AutoScalingGroupName: aws.String(asgs[0].Id()), DesiredCapacity: aws.Int64(1), HonorCooldown: aws.Bool(false), @@ -489,7 +449,7 @@ func TestDeleteNodesWithPlaceholder(t *testing.T) { // Look up the current number of instances... var expectedInstancesCount int64 = 2 - service.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroupsPages", &autoscaling.DescribeAutoScalingGroupsInput{ AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), MaxRecords: aws.Int64(maxRecordsReturnedByAPI), @@ -502,6 +462,12 @@ func TestDeleteNodesWithPlaceholder(t *testing.T) { expectedInstancesCount = 1 }).Return(nil) + a.On("DescribeScalingActivities", + &autoscaling.DescribeScalingActivitiesInput{ + AutoScalingGroupName: aws.String("test-asg"), + }, + ).Return(&autoscaling.DescribeScalingActivitiesOutput{}, nil) + provider.Refresh() initialSize, err := asgs[0].TargetSize() @@ -515,8 +481,8 @@ func TestDeleteNodesWithPlaceholder(t *testing.T) { } err = asgs[0].DeleteNodes([]*apiv1.Node{node}) assert.NoError(t, err) - service.AssertNumberOfCalls(t, "SetDesiredCapacity", 1) - service.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 2) + a.AssertNumberOfCalls(t, "SetDesiredCapacity", 1) + a.AssertNumberOfCalls(t, "DescribeAutoScalingGroupsPages", 1) newSize, err := asgs[0].TargetSize() assert.NoError(t, err) @@ -524,12 +490,12 @@ func TestDeleteNodesWithPlaceholder(t *testing.T) { } func TestDeleteNodesAfterMultipleRefreshes(t *testing.T) { - service := &AutoScalingMock{} - manager := newTestAwsManagerWithAsgs(t, service, []string{"1:5:test-asg"}) + a := &autoScalingMock{} + manager := newTestAwsManagerWithAsgs(t, a, nil, []string{"1:5:test-asg"}) provider := testProvider(t, manager) asgs := provider.NodeGroups() - service.On("TerminateInstanceInAutoScalingGroup", &autoscaling.TerminateInstanceInAutoScalingGroupInput{ + a.On("TerminateInstanceInAutoScalingGroup", &autoscaling.TerminateInstanceInAutoScalingGroupInput{ InstanceId: aws.String("test-instance-id"), ShouldDecrementDesiredCapacity: aws.Bool(true), }).Return(&autoscaling.TerminateInstanceInAutoScalingGroupOutput{ @@ -537,7 +503,7 @@ func TestDeleteNodesAfterMultipleRefreshes(t *testing.T) { }) // Look up the current number of instances... - service.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroupsPages", &autoscaling.DescribeAutoScalingGroupsInput{ AutoScalingGroupNames: aws.StringSlice([]string{"test-asg"}), MaxRecords: aws.Int64(maxRecordsReturnedByAPI), @@ -562,8 +528,9 @@ func TestDeleteNodesAfterMultipleRefreshes(t *testing.T) { } func TestGetResourceLimiter(t *testing.T) { - service := &AutoScalingMock{} - m := newTestAwsManagerWithService(service, nil) + mockAutoScaling := &autoScalingMock{} + mockEC2 := &ec2Mock{} + m := newTestAwsManagerWithMockServices(mockAutoScaling, mockEC2, nil) provider := testProvider(t, m) _, err := provider.GetResourceLimiter() diff --git a/cluster-autoscaler/cloudprovider/aws/aws_manager.go b/cluster-autoscaler/cloudprovider/aws/aws_manager.go index 9d07e09e41ed..7bebd4a1ff65 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_manager.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_manager.go @@ -41,7 +41,6 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" klog "k8s.io/klog/v2" - kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" provider_aws "k8s.io/legacy-cloud-providers/aws" ) @@ -57,11 +56,10 @@ const ( // AwsManager is handles aws communication and data caching. type AwsManager struct { - autoScalingService autoScalingWrapper - ec2Service ec2Wrapper - asgCache *asgCache - lastRefresh time.Time - instanceTypes map[string]*InstanceType + awsService awsWrapper + asgCache *asgCache + lastRefresh time.Time + instanceTypes map[string]*InstanceType } type asgTemplate struct { @@ -162,7 +160,7 @@ func getRegion(cfg ...*aws.Config) string { return region } -// createAwsManagerInternal allows for a customer autoScalingWrapper to be passed in by tests +// createAwsManagerInternal allows for custom objects to be passed in by tests // // #1449 If running tests outside of AWS without AWS_REGION among environment // variables, avoid a 5+ second EC2 Metadata lookup timeout in getRegion by @@ -173,8 +171,7 @@ func getRegion(cfg ...*aws.Config) string { func createAWSManagerInternal( configReader io.Reader, discoveryOpts cloudprovider.NodeGroupDiscoveryOptions, - autoScalingService *autoScalingWrapper, - ec2Service *ec2Wrapper, + awsService *awsWrapper, instanceTypes map[string]*InstanceType, ) (*AwsManager, error) { @@ -189,7 +186,7 @@ func createAWSManagerInternal( return nil, err } - if autoScalingService == nil || ec2Service == nil { + if awsService == nil { awsSdkProvider := newAWSSDKProvider(cfg) sess, err := session.NewSession(aws.NewConfig().WithRegion(getRegion()). WithEndpointResolver(getResolver(awsSdkProvider.cfg))) @@ -197,14 +194,7 @@ func createAWSManagerInternal( return nil, err } - if autoScalingService == nil { - c := newLaunchConfigurationInstanceTypeCache() - autoScalingService = &autoScalingWrapper{autoscaling.New(sess), c} - } - - if ec2Service == nil { - ec2Service = &ec2Wrapper{ec2.New(sess)} - } + awsService = &awsWrapper{autoscaling.New(sess), ec2.New(sess)} } specs, err := parseASGAutoDiscoverySpecs(discoveryOpts) @@ -212,16 +202,15 @@ func createAWSManagerInternal( return nil, err } - cache, err := newASGCache(*autoScalingService, discoveryOpts.NodeGroupSpecs, specs) + cache, err := newASGCache(awsService, discoveryOpts.NodeGroupSpecs, specs) if err != nil { return nil, err } manager := &AwsManager{ - autoScalingService: *autoScalingService, - ec2Service: *ec2Service, - asgCache: cache, - instanceTypes: instanceTypes, + awsService: *awsService, + asgCache: cache, + instanceTypes: instanceTypes, } if err := manager.forceRefresh(); err != nil { @@ -248,7 +237,7 @@ func readAWSCloudConfig(config io.Reader) (*provider_aws.CloudConfig, error) { // CreateAwsManager constructs awsManager object. func CreateAwsManager(configReader io.Reader, discoveryOpts cloudprovider.NodeGroupDiscoveryOptions, instanceTypes map[string]*InstanceType) (*AwsManager, error) { - return createAWSManagerInternal(configReader, discoveryOpts, nil, nil, instanceTypes) + return createAWSManagerInternal(configReader, discoveryOpts, nil, instanceTypes) } // Refresh is called before every main loop and can be used to dynamically update cloud provider state. @@ -280,7 +269,7 @@ func (m *AwsManager) Cleanup() { m.asgCache.Cleanup() } -func (m *AwsManager) getAsgs() []*asg { +func (m *AwsManager) getAsgs() map[AwsRef]*asg { return m.asgCache.Get() } @@ -294,8 +283,9 @@ func (m *AwsManager) DeleteInstances(instances []*AwsInstanceRef) error { if err := m.asgCache.DeleteInstances(instances); err != nil { return err } - klog.V(2).Infof("Some ASG instances might have been deleted, forcing ASG list refresh") - return m.forceRefresh() + klog.V(2).Infof("DeleteInstances was called: scheduling an ASG list refresh for next main loop evaluation") + m.lastRefresh = time.Now().Add(-refreshInterval) + return nil } // GetAsgNodes returns Asg nodes. @@ -303,6 +293,11 @@ func (m *AwsManager) GetAsgNodes(ref AwsRef) ([]AwsInstanceRef, error) { return m.asgCache.InstancesByAsg(ref) } +// GetInstanceStatus returns the status of ASG nodes +func (m *AwsManager) GetInstanceStatus(ref AwsInstanceRef) (*string, error) { + return m.asgCache.InstanceStatus(ref) +} + func (m *AwsManager) getAsgTemplate(asg *asg) (*asgTemplate, error) { if len(asg.AvailabilityZones) < 1 { return nil, fmt.Errorf("unable to get first AvailabilityZone for ASG %q", asg.Name) @@ -315,7 +310,7 @@ func (m *AwsManager) getAsgTemplate(asg *asg) (*asgTemplate, error) { klog.V(4).Infof("Found multiple availability zones for ASG %q; using %s for %s label\n", asg.Name, az, apiv1.LabelFailureDomainBetaZone) } - instanceTypeName, err := m.buildInstanceType(asg) + instanceTypeName, err := getInstanceTypeForAsg(m.asgCache, asg) if err != nil { return nil, err } @@ -331,23 +326,6 @@ func (m *AwsManager) getAsgTemplate(asg *asg) (*asgTemplate, error) { return nil, fmt.Errorf("ASG %q uses the unknown EC2 instance type %q", asg.Name, instanceTypeName) } -func (m *AwsManager) buildInstanceType(asg *asg) (string, error) { - if asg.LaunchConfigurationName != "" { - return m.autoScalingService.getInstanceTypeByLCName(asg.LaunchConfigurationName) - } else if asg.LaunchTemplate != nil { - return m.ec2Service.getInstanceTypeByLT(asg.LaunchTemplate) - } else if asg.MixedInstancesPolicy != nil { - // always use first instance - if len(asg.MixedInstancesPolicy.instanceTypesOverrides) != 0 { - return asg.MixedInstancesPolicy.instanceTypesOverrides[0], nil - } - - return m.ec2Service.getInstanceTypeByLT(asg.MixedInstancesPolicy.launchTemplate) - } - - return "", errors.New("Unable to get instance type from launch config or launch template") -} - func (m *AwsManager) buildNodeFromTemplate(asg *asg, template *asgTemplate) (*apiv1.Node, error) { node := apiv1.Node{} nodeName := fmt.Sprintf("%s-asg-%d", asg.Name, rand.Int63()) @@ -389,14 +367,14 @@ func (m *AwsManager) buildNodeFromTemplate(asg *asg, template *asgTemplate) (*ap func buildGenericLabels(template *asgTemplate, nodeName string) map[string]string { result := make(map[string]string) - // TODO: extract it somehow - result[kubeletapis.LabelArch] = cloudprovider.DefaultArch - result[kubeletapis.LabelOS] = cloudprovider.DefaultOS - result[apiv1.LabelInstanceType] = template.InstanceType.InstanceType + result[apiv1.LabelArchStable] = template.InstanceType.Architecture + result[apiv1.LabelOSStable] = cloudprovider.DefaultOS + + result[apiv1.LabelInstanceTypeStable] = template.InstanceType.InstanceType - result[apiv1.LabelZoneRegion] = template.Region - result[apiv1.LabelZoneFailureDomain] = template.Zone + result[apiv1.LabelTopologyRegion] = template.Region + result[apiv1.LabelTopologyZone] = template.Zone result[apiv1.LabelHostname] = nodeName return result } diff --git a/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go b/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go index 86e5d0ff2c37..7426a629088a 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go @@ -38,7 +38,6 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" - kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" provider_aws "k8s.io/legacy-cloud-providers/aws" ) @@ -77,14 +76,15 @@ func TestBuildGenericLabels(t *testing.T) { InstanceType: "c4.large", VCPU: 2, MemoryMb: 3840, + Architecture: cloudprovider.DefaultArch, }, Region: "us-east-1", }, "sillyname") - assert.Equal(t, "us-east-1", labels[apiv1.LabelZoneRegion]) + assert.Equal(t, "us-east-1", labels[apiv1.LabelZoneRegionStable]) assert.Equal(t, "sillyname", labels[apiv1.LabelHostname]) - assert.Equal(t, "c4.large", labels[apiv1.LabelInstanceType]) - assert.Equal(t, cloudprovider.DefaultArch, labels[kubeletapis.LabelArch]) - assert.Equal(t, cloudprovider.DefaultOS, labels[kubeletapis.LabelOS]) + assert.Equal(t, "c4.large", labels[apiv1.LabelInstanceTypeStable]) + assert.Equal(t, cloudprovider.DefaultArch, labels[apiv1.LabelArchStable]) + assert.Equal(t, cloudprovider.DefaultOS, labels[apiv1.LabelOSStable]) } func TestExtractAllocatableResourcesFromAsg(t *testing.T) { @@ -259,9 +259,10 @@ func makeTaintSet(taints []apiv1.Taint) map[apiv1.Taint]bool { func TestFetchExplicitAsgs(t *testing.T) { min, max, groupname := 1, 10, "coolasg" + asgRef := AwsRef{Name: groupname} - s := &AutoScalingMock{} - s.On("DescribeAutoScalingGroups", &autoscaling.DescribeAutoScalingGroupsInput{ + a := &autoScalingMock{} + a.On("DescribeAutoScalingGroups", &autoscaling.DescribeAutoScalingGroupsInput{ AutoScalingGroupNames: []*string{aws.String(groupname)}, MaxRecords: aws.Int64(1), }).Return(&autoscaling.DescribeAutoScalingGroupsOutput{ @@ -270,7 +271,7 @@ func TestFetchExplicitAsgs(t *testing.T) { }, }) - s.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroupsPages", &autoscaling.DescribeAutoScalingGroupsInput{ AutoScalingGroupNames: aws.StringSlice([]string{groupname}), MaxRecords: aws.Int64(maxRecordsReturnedByAPI), @@ -291,6 +292,12 @@ func TestFetchExplicitAsgs(t *testing.T) { }}, false) }).Return(nil) + a.On("DescribeScalingActivities", + &autoscaling.DescribeScalingActivitiesInput{ + AutoScalingGroupName: aws.String("coolasg"), + }, + ).Return(&autoscaling.DescribeScalingActivitiesOutput{}, nil) + do := cloudprovider.NodeGroupDiscoveryOptions{ // Register the same node group twice with different max nodes. // The intention is to test that the asgs.Register method will update @@ -304,112 +311,17 @@ func TestFetchExplicitAsgs(t *testing.T) { defer resetAWSRegion(os.LookupEnv("AWS_REGION")) os.Setenv("AWS_REGION", "fanghorn") instanceTypes, _ := GetStaticEC2InstanceTypes() - m, err := createAWSManagerInternal(nil, do, &autoScalingWrapper{s, newLaunchConfigurationInstanceTypeCache()}, nil, instanceTypes) + m, err := createAWSManagerInternal(nil, do, &awsWrapper{a, nil}, instanceTypes) assert.NoError(t, err) asgs := m.asgCache.Get() assert.Equal(t, 1, len(asgs)) - validateAsg(t, asgs[0], groupname, min, max) -} - -func TestBuildInstanceType(t *testing.T) { - ltName, ltVersion, instanceType := "launcher", "1", "t2.large" - - s := &EC2Mock{} - s.On("DescribeLaunchTemplateVersions", &ec2.DescribeLaunchTemplateVersionsInput{ - LaunchTemplateName: aws.String(ltName), - Versions: []*string{aws.String(ltVersion)}, - }).Return(&ec2.DescribeLaunchTemplateVersionsOutput{ - LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{ - { - LaunchTemplateData: &ec2.ResponseLaunchTemplateData{ - InstanceType: aws.String(instanceType), - }, - }, - }, - }) - - // #1449 Without AWS_REGION getRegion() lookup runs till timeout during tests. - defer resetAWSRegion(os.LookupEnv("AWS_REGION")) - os.Setenv("AWS_REGION", "fanghorn") - instanceTypes, _ := GetStaticEC2InstanceTypes() - m, err := createAWSManagerInternal(nil, cloudprovider.NodeGroupDiscoveryOptions{}, nil, &ec2Wrapper{s}, instanceTypes) - assert.NoError(t, err) - - asg := asg{ - LaunchTemplate: &launchTemplate{name: ltName, version: ltVersion}, - } - - builtInstanceType, err := m.buildInstanceType(&asg) - - assert.NoError(t, err) - assert.Equal(t, instanceType, builtInstanceType) -} - -func TestBuildInstanceTypeMixedInstancePolicyOverride(t *testing.T) { - ltName, ltVersion, instanceType := "launcher", "1", "t2.large" - instanceTypeOverrides := []string{} - - s := &EC2Mock{} - s.On("DescribeLaunchTemplateVersions", &ec2.DescribeLaunchTemplateVersionsInput{ - LaunchTemplateName: aws.String(ltName), - Versions: []*string{aws.String(ltVersion)}, - }).Return(&ec2.DescribeLaunchTemplateVersionsOutput{ - LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{ - { - LaunchTemplateData: &ec2.ResponseLaunchTemplateData{ - InstanceType: aws.String(instanceType), - }, - }, - }, - }) - - defer resetAWSRegion(os.LookupEnv("AWS_REGION")) - os.Setenv("AWS_REGION", "fanghorn") - instanceTypes, _ := GetStaticEC2InstanceTypes() - m, err := createAWSManagerInternal(nil, cloudprovider.NodeGroupDiscoveryOptions{}, nil, &ec2Wrapper{s}, instanceTypes) - assert.NoError(t, err) - - lt := &launchTemplate{name: ltName, version: ltVersion} - asg := asg{ - MixedInstancesPolicy: &mixedInstancesPolicy{ - launchTemplate: lt, - instanceTypesOverrides: instanceTypeOverrides, - }, - } - - builtInstanceType, err := m.buildInstanceType(&asg) - - assert.NoError(t, err) - assert.Equal(t, instanceType, builtInstanceType) -} - -func TestBuildInstanceTypeMixedInstancePolicyNoOverride(t *testing.T) { - ltName, ltVersion := "launcher", "1" - instanceTypeOverrides := []string{"m4.xlarge", "m5.xlarge"} - - defer resetAWSRegion(os.LookupEnv("AWS_REGION")) - os.Setenv("AWS_REGION", "fanghorn") - instanceTypes, _ := GetStaticEC2InstanceTypes() - m, err := createAWSManagerInternal(nil, cloudprovider.NodeGroupDiscoveryOptions{}, nil, &ec2Wrapper{}, instanceTypes) - assert.NoError(t, err) - - lt := &launchTemplate{name: ltName, version: ltVersion} - asg := asg{ - MixedInstancesPolicy: &mixedInstancesPolicy{ - launchTemplate: lt, - instanceTypesOverrides: instanceTypeOverrides, - }, - } - - builtInstanceType, err := m.buildInstanceType(&asg) - - assert.NoError(t, err) - assert.Equal(t, instanceTypeOverrides[0], builtInstanceType) + validateAsg(t, asgs[asgRef], groupname, min, max) } func TestGetASGTemplate(t *testing.T) { const ( + asgName = "sample" knownInstanceType = "t3.micro" region = "us-east-1" az = region + "a" @@ -417,6 +329,8 @@ func TestGetASGTemplate(t *testing.T) { ltVersion = "1" ) + asgRef := AwsRef{Name: asgName} + tags := []*autoscaling.TagDescription{ { Key: aws.String("k8s.io/cluster-autoscaler/node-template/taint/dedicated"), @@ -442,8 +356,8 @@ func TestGetASGTemplate(t *testing.T) { for _, test := range tests { t.Run(test.description, func(t *testing.T) { - s := &EC2Mock{} - s.On("DescribeLaunchTemplateVersions", &ec2.DescribeLaunchTemplateVersionsInput{ + e := &ec2Mock{} + e.On("DescribeLaunchTemplateVersions", &ec2.DescribeLaunchTemplateVersionsInput{ LaunchTemplateName: aws.String(ltName), Versions: []*string{aws.String(ltVersion)}, }).Return(&ec2.DescribeLaunchTemplateVersionsOutput{ @@ -460,11 +374,18 @@ func TestGetASGTemplate(t *testing.T) { defer resetAWSRegion(os.LookupEnv("AWS_REGION")) os.Setenv("AWS_REGION", "fanghorn") instanceTypes, _ := GetStaticEC2InstanceTypes() - m, err := createAWSManagerInternal(nil, cloudprovider.NodeGroupDiscoveryOptions{}, nil, &ec2Wrapper{s}, instanceTypes) + do := cloudprovider.NodeGroupDiscoveryOptions{} + + m, err := createAWSManagerInternal(nil, do, &awsWrapper{nil, e}, instanceTypes) + origGetInstanceTypeFunc := getInstanceTypeForAsg + defer func() { getInstanceTypeForAsg = origGetInstanceTypeFunc }() + getInstanceTypeForAsg = func(m *asgCache, asg *asg) (string, error) { + return test.instanceType, nil + } assert.NoError(t, err) asg := &asg{ - AwsRef: AwsRef{Name: "sample"}, + AwsRef: asgRef, AvailabilityZones: test.availabilityZones, LaunchTemplate: &launchTemplate{ name: ltName, @@ -491,8 +412,9 @@ func TestGetASGTemplate(t *testing.T) { func TestFetchAutoAsgs(t *testing.T) { min, max := 1, 10 groupname, tags := "coolasg", []string{"tag", "anothertag"} + asgRef := AwsRef{Name: groupname} - s := &AutoScalingMock{} + a := &autoScalingMock{} // Lookup groups associated with tags expectedTagsInput := &autoscaling.DescribeTagsInput{ Filters: []*autoscaling.Filter{ @@ -502,7 +424,7 @@ func TestFetchAutoAsgs(t *testing.T) { MaxRecords: aws.Int64(maxRecordsReturnedByAPI), } // Use MatchedBy pattern to avoid list order issue https://github.com/kubernetes/autoscaler/issues/1346 - s.On("DescribeTagsPages", mock.MatchedBy(tagsMatcher(expectedTagsInput)), + a.On("DescribeTagsPages", mock.MatchedBy(tagsMatcher(expectedTagsInput)), mock.AnythingOfType("func(*autoscaling.DescribeTagsOutput, bool) bool"), ).Run(func(args mock.Arguments) { fn := args.Get(1).(func(*autoscaling.DescribeTagsOutput, bool) bool) @@ -515,7 +437,7 @@ func TestFetchAutoAsgs(t *testing.T) { // Describe the group to register it, then again to generate the instance // cache. - s.On("DescribeAutoScalingGroupsPages", + a.On("DescribeAutoScalingGroupsPages", &autoscaling.DescribeAutoScalingGroupsInput{ AutoScalingGroupNames: aws.StringSlice([]string{groupname}), MaxRecords: aws.Int64(maxRecordsReturnedByAPI), @@ -534,6 +456,12 @@ func TestFetchAutoAsgs(t *testing.T) { }}}, false) }).Return(nil).Twice() + a.On("DescribeScalingActivities", + &autoscaling.DescribeScalingActivitiesInput{ + AutoScalingGroupName: aws.String("coolasg"), + }, + ).Return(&autoscaling.DescribeScalingActivitiesOutput{}, nil) + do := cloudprovider.NodeGroupDiscoveryOptions{ NodeGroupAutoDiscoverySpecs: []string{fmt.Sprintf("asg:tag=%s", strings.Join(tags, ","))}, } @@ -543,15 +471,15 @@ func TestFetchAutoAsgs(t *testing.T) { os.Setenv("AWS_REGION", "fanghorn") // fetchAutoASGs is called at manager creation time, via forceRefresh instanceTypes, _ := GetStaticEC2InstanceTypes() - m, err := createAWSManagerInternal(nil, do, &autoScalingWrapper{s, newLaunchConfigurationInstanceTypeCache()}, nil, instanceTypes) + m, err := createAWSManagerInternal(nil, do, &awsWrapper{a, nil}, instanceTypes) assert.NoError(t, err) asgs := m.asgCache.Get() assert.Equal(t, 1, len(asgs)) - validateAsg(t, asgs[0], groupname, min, max) + validateAsg(t, asgs[asgRef], groupname, min, max) // Simulate the previously discovered ASG disappearing - s.On("DescribeTagsPages", mock.MatchedBy(tagsMatcher(expectedTagsInput)), + a.On("DescribeTagsPages", mock.MatchedBy(tagsMatcher(expectedTagsInput)), mock.AnythingOfType("func(*autoscaling.DescribeTagsOutput, bool) bool"), ).Run(func(args mock.Arguments) { fn := args.Get(1).(func(*autoscaling.DescribeTagsOutput, bool) bool) diff --git a/cluster-autoscaler/cloudprovider/aws/aws_metrics.go b/cluster-autoscaler/cloudprovider/aws/aws_metrics.go new file mode 100644 index 000000000000..5f7cb1fe6e7a --- /dev/null +++ b/cluster-autoscaler/cloudprovider/aws/aws_metrics.go @@ -0,0 +1,59 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "time" + + "github.com/aws/aws-sdk-go/aws/awserr" + k8smetrics "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" +) + +const ( + caNamespace = "cluster_autoscaler" +) + +var ( + /**** Metrics related to AWS API usage ****/ + requestSummary = k8smetrics.NewHistogramVec( + &k8smetrics.HistogramOpts{ + Namespace: caNamespace, + Name: "aws_request_duration_seconds", + Help: "Time taken by AWS requests, by method and status code, in seconds", + Buckets: []float64{0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 60.0}, + }, []string{"endpoint", "status"}, + ) +) + +// RegisterMetrics registers all AWS metrics. +func RegisterMetrics() { + legacyregistry.MustRegister(requestSummary) +} + +// observeAWSRequest records AWS API calls counts and durations +func observeAWSRequest(endpoint string, err error, start time.Time) { + duration := time.Since(start).Seconds() + status := "success" + if err != nil { + status = "error" + if awsErr, ok := err.(awserr.Error); ok { + status = awsErr.Code() + } + } + requestSummary.WithLabelValues(endpoint, status).Observe(duration) +} diff --git a/cluster-autoscaler/cloudprovider/aws/aws_util.go b/cluster-autoscaler/cloudprovider/aws/aws_util.go index 8176f96ba07a..23f5ee0cdeda 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_util.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_util.go @@ -40,6 +40,7 @@ var ( ec2PricingServiceUrlTemplate = "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/%s/index.json" ec2PricingServiceUrlTemplateCN = "https://pricing.cn-north-1.amazonaws.com.cn/offers/v1.0/cn/AmazonEC2/current/%s/index.json" staticListLastUpdateTime = "2022-06-02" + ec2Arm64Processors = []string{"AWS Graviton Processor", "AWS Graviton2 Processor"} ) type response struct { @@ -55,6 +56,7 @@ type productAttributes struct { VCPU string `json:"vcpu"` Memory string `json:"memory"` GPU string `json:"gpu"` + Architecture string `json:"physicalProcessor"` } // GenerateEC2InstanceTypes returns a map of ec2 resources @@ -108,6 +110,9 @@ func GenerateEC2InstanceTypes(region string) (map[string]*InstanceType, error) { if attr.GPU != "" { instanceTypes[attr.InstanceType].GPU = parseCPU(attr.GPU) } + if attr.Architecture != "" { + instanceTypes[attr.InstanceType].Architecture = parseArchitecture(attr.Architecture) + } } } } @@ -200,6 +205,15 @@ func parseCPU(cpu string) int64 { return i } +func parseArchitecture(archName string) string { + for _, processor := range ec2Arm64Processors { + if archName == processor { + return "arm64" + } + } + return "amd64" +} + // GetCurrentAwsRegion return region of current cluster without building awsManager func GetCurrentAwsRegion() (string, error) { region, present := os.LookupEnv("AWS_REGION") diff --git a/cluster-autoscaler/cloudprovider/aws/aws_util_test.go b/cluster-autoscaler/cloudprovider/aws/aws_util_test.go index e29860b41f6c..243f96043a14 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_util_test.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_util_test.go @@ -79,6 +79,31 @@ func TestParseCPU(t *testing.T) { } } +func TestParseArchitecture(t *testing.T) { + tests := []struct { + input string + expect string + }{ + { + input: "Intel Xeon Platinum 8259 (Cascade Lake)", + expect: "amd64", + }, + { + input: "AWS Graviton2 Processor", + expect: "arm64", + }, + { + input: "anything default", + expect: "amd64", + }, + } + + for _, test := range tests { + got := parseArchitecture(test.input) + assert.Equal(t, test.expect, got) + } +} + func TestGetCurrentAwsRegion(t *testing.T) { region := "us-west-2" if oldRegion, found := os.LookupEnv("AWS_REGION"); found { diff --git a/cluster-autoscaler/cloudprovider/aws/aws_wrapper.go b/cluster-autoscaler/cloudprovider/aws/aws_wrapper.go new file mode 100644 index 000000000000..5c23da622bbb --- /dev/null +++ b/cluster-autoscaler/cloudprovider/aws/aws_wrapper.go @@ -0,0 +1,274 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/aws/aws-sdk-go/service/ec2" + klog "k8s.io/klog/v2" +) + +// autoScalingI is the interface abstracting specific API calls of the auto-scaling service provided by AWS SDK for use in CA +type autoScalingI interface { + DescribeAutoScalingGroupsPages(input *autoscaling.DescribeAutoScalingGroupsInput, fn func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) error + DescribeLaunchConfigurations(*autoscaling.DescribeLaunchConfigurationsInput) (*autoscaling.DescribeLaunchConfigurationsOutput, error) + DescribeScalingActivities(*autoscaling.DescribeScalingActivitiesInput) (*autoscaling.DescribeScalingActivitiesOutput, error) + DescribeTagsPages(input *autoscaling.DescribeTagsInput, fn func(*autoscaling.DescribeTagsOutput, bool) bool) error + SetDesiredCapacity(input *autoscaling.SetDesiredCapacityInput) (*autoscaling.SetDesiredCapacityOutput, error) + TerminateInstanceInAutoScalingGroup(input *autoscaling.TerminateInstanceInAutoScalingGroupInput) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error) +} + +// ec2I is the interface abstracting specific API calls of the EC2 service provided by AWS SDK for use in CA +type ec2I interface { + DescribeLaunchTemplateVersions(input *ec2.DescribeLaunchTemplateVersionsInput) (*ec2.DescribeLaunchTemplateVersionsOutput, error) +} + +// awsWrapper provides several utility methods over the services provided by the AWS SDK +type awsWrapper struct { + autoScalingI + ec2I +} + +func (m *awsWrapper) getInstanceTypeByLaunchConfigNames(launchConfigToQuery []*string) (map[string]string, error) { + launchConfigurationsToInstanceType := map[string]string{} + + for i := 0; i < len(launchConfigToQuery); i += 50 { + end := i + 50 + + if end > len(launchConfigToQuery) { + end = len(launchConfigToQuery) + } + params := &autoscaling.DescribeLaunchConfigurationsInput{ + LaunchConfigurationNames: launchConfigToQuery[i:end], + MaxRecords: aws.Int64(50), + } + start := time.Now() + r, err := m.DescribeLaunchConfigurations(params) + observeAWSRequest("DescribeLaunchConfigurations", err, start) + if err != nil { + return nil, err + } + for _, lc := range r.LaunchConfigurations { + launchConfigurationsToInstanceType[*lc.LaunchConfigurationName] = *lc.InstanceType + } + } + return launchConfigurationsToInstanceType, nil +} + +func (m *awsWrapper) getAutoscalingGroupsByNames(names []string) ([]*autoscaling.Group, error) { + if len(names) == 0 { + return nil, nil + } + + asgs := make([]*autoscaling.Group, 0) + + // AWS only accepts up to 50 ASG names as input, describe them in batches + for i := 0; i < len(names); i += maxAsgNamesPerDescribe { + end := i + maxAsgNamesPerDescribe + + if end > len(names) { + end = len(names) + } + + input := &autoscaling.DescribeAutoScalingGroupsInput{ + AutoScalingGroupNames: aws.StringSlice(names[i:end]), + MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + } + start := time.Now() + err := m.DescribeAutoScalingGroupsPages(input, func(output *autoscaling.DescribeAutoScalingGroupsOutput, _ bool) bool { + asgs = append(asgs, output.AutoScalingGroups...) + // We return true while we want to be called with the next page of + // results, if any. + return true + }) + observeAWSRequest("DescribeAutoScalingGroupsPages", err, start) + if err != nil { + return nil, err + } + } + + return asgs, nil +} + +func (m *awsWrapper) getAutoscalingGroupNamesByTags(kvs map[string]string) ([]string, error) { + // DescribeTags does an OR query when multiple filters on different tags are + // specified. In other words, DescribeTags returns [asg1, asg1] for keys + // [t1, t2] when there's only one asg tagged both t1 and t2. + filters := []*autoscaling.Filter{} + for key, value := range kvs { + filter := &autoscaling.Filter{ + Name: aws.String("key"), + Values: []*string{aws.String(key)}, + } + filters = append(filters, filter) + if value != "" { + filters = append(filters, &autoscaling.Filter{ + Name: aws.String("value"), + Values: []*string{aws.String(value)}, + }) + } + } + + tags := []*autoscaling.TagDescription{} + input := &autoscaling.DescribeTagsInput{ + Filters: filters, + MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + } + start := time.Now() + err := m.DescribeTagsPages(input, func(out *autoscaling.DescribeTagsOutput, _ bool) bool { + tags = append(tags, out.Tags...) + // We return true while we want to be called with the next page of + // results, if any. + return true + }) + observeAWSRequest("DescribeTagsPages", err, start) + + if err != nil { + return nil, err + } + + // According to how DescribeTags API works, the result contains ASGs which + // not all but only subset of tags are associated. Explicitly select ASGs to + // which all the tags are associated so that we won't end up calling + // DescribeAutoScalingGroups API multiple times on an ASG. + asgNames := []string{} + asgNameOccurrences := make(map[string]int) + for _, t := range tags { + asgName := aws.StringValue(t.ResourceId) + occurrences := asgNameOccurrences[asgName] + 1 + if occurrences >= len(kvs) { + asgNames = append(asgNames, asgName) + } + asgNameOccurrences[asgName] = occurrences + } + + return asgNames, nil +} + +func (m *awsWrapper) getInstanceTypeByLaunchTemplate(launchTemplate *launchTemplate) (string, error) { + params := &ec2.DescribeLaunchTemplateVersionsInput{ + LaunchTemplateName: aws.String(launchTemplate.name), + Versions: []*string{aws.String(launchTemplate.version)}, + } + + start := time.Now() + describeData, err := m.DescribeLaunchTemplateVersions(params) + observeAWSRequest("DescribeLaunchTemplateVersions", err, start) + if err != nil { + return "", err + } + + if len(describeData.LaunchTemplateVersions) == 0 { + return "", fmt.Errorf("unable to find template versions") + } + + lt := describeData.LaunchTemplateVersions[0] + instanceType := lt.LaunchTemplateData.InstanceType + + if instanceType == nil { + return "", fmt.Errorf("unable to find instance type within launch template") + } + + return aws.StringValue(instanceType), nil +} + +func (m *awsWrapper) getInstanceTypesForAsgs(asgs []*asg) (map[string]string, error) { + results := map[string]string{} + launchConfigsToQuery := map[string]string{} + launchTemplatesToQuery := map[string]*launchTemplate{} + + for _, asg := range asgs { + name := asg.AwsRef.Name + if asg.LaunchConfigurationName != "" { + launchConfigsToQuery[name] = asg.LaunchConfigurationName + } else if asg.LaunchTemplate != nil { + launchTemplatesToQuery[name] = asg.LaunchTemplate + } else if asg.MixedInstancesPolicy != nil { + if len(asg.MixedInstancesPolicy.instanceTypesOverrides) > 0 { + results[name] = asg.MixedInstancesPolicy.instanceTypesOverrides[0] + } else { + launchTemplatesToQuery[name] = asg.MixedInstancesPolicy.launchTemplate + } + } + } + + klog.V(4).Infof("%d launch configurations to query", len(launchConfigsToQuery)) + klog.V(4).Infof("%d launch templates to query", len(launchTemplatesToQuery)) + + // Query these all at once to minimize AWS API calls + launchConfigNames := make([]*string, 0, len(launchConfigsToQuery)) + for _, cfgName := range launchConfigsToQuery { + launchConfigNames = append(launchConfigNames, aws.String(cfgName)) + } + launchConfigs, err := m.getInstanceTypeByLaunchConfigNames(launchConfigNames) + if err != nil { + klog.Errorf("Failed to query %d launch configurations", len(launchConfigsToQuery)) + return nil, err + } + + for asgName, cfgName := range launchConfigsToQuery { + results[asgName] = launchConfigs[cfgName] + } + klog.V(4).Infof("Successfully queried %d launch configurations", len(launchConfigsToQuery)) + + // Have to query LaunchTemplates one-at-a-time, since there's no way to query pairs in bulk + for asgName, lt := range launchTemplatesToQuery { + instanceType, err := m.getInstanceTypeByLaunchTemplate(lt) + if err != nil { + klog.Errorf("Failed to query launch tempate %s: %v", lt.name, err) + continue + } + results[asgName] = instanceType + } + klog.V(4).Infof("Successfully queried %d launch templates", len(launchTemplatesToQuery)) + + return results, nil +} + +func buildLaunchTemplateFromSpec(ltSpec *autoscaling.LaunchTemplateSpecification) *launchTemplate { + // NOTE(jaypipes): The LaunchTemplateSpecification.Version is a pointer to + // string. When the pointer is nil, EC2 AutoScaling API considers the value + // to be "$Default", however aws.StringValue(ltSpec.Version) will return an + // empty string (which is not considered the same as "$Default" or a nil + // string pointer. So, in order to not pass an empty string as the version + // for the launch template when we communicate with the EC2 AutoScaling API + // using the information in the launchTemplate, we store the string + // "$Default" here when the ltSpec.Version is a nil pointer. + // + // See: + // + // https://github.com/kubernetes/autoscaler/issues/1728 + // https://github.com/aws/aws-sdk-go/blob/81fad3b797f4a9bd1b452a5733dd465eefef1060/service/autoscaling/api.go#L10666-L10671 + // + // A cleaner alternative might be to make launchTemplate.version a string + // pointer instead of a string, or even store the aws-sdk-go's + // LaunchTemplateSpecification structs directly. + var version string + if ltSpec.Version == nil { + version = "$Default" + } else { + version = aws.StringValue(ltSpec.Version) + } + return &launchTemplate{ + name: aws.StringValue(ltSpec.LaunchTemplateName), + version: version, + } +} diff --git a/cluster-autoscaler/cloudprovider/aws/aws_wrapper_test.go b/cluster-autoscaler/cloudprovider/aws/aws_wrapper_test.go new file mode 100644 index 000000000000..c16c290b104a --- /dev/null +++ b/cluster-autoscaler/cloudprovider/aws/aws_wrapper_test.go @@ -0,0 +1,274 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "fmt" + "os" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +type autoScalingMock struct { + mock.Mock +} + +func (a *autoScalingMock) DescribeAutoScalingGroupsPages(i *autoscaling.DescribeAutoScalingGroupsInput, fn func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) error { + args := a.Called(i, fn) + return args.Error(0) +} + +func (a *autoScalingMock) DescribeLaunchConfigurations(i *autoscaling.DescribeLaunchConfigurationsInput) (*autoscaling.DescribeLaunchConfigurationsOutput, error) { + args := a.Called(i) + return args.Get(0).(*autoscaling.DescribeLaunchConfigurationsOutput), nil +} + +func (a *autoScalingMock) DescribeScalingActivities(i *autoscaling.DescribeScalingActivitiesInput) (*autoscaling.DescribeScalingActivitiesOutput, error) { + args := a.Called(i) + return args.Get(0).(*autoscaling.DescribeScalingActivitiesOutput), args.Error(1) +} + +func (a *autoScalingMock) DescribeTagsPages(i *autoscaling.DescribeTagsInput, fn func(*autoscaling.DescribeTagsOutput, bool) bool) error { + args := a.Called(i, fn) + return args.Error(0) +} + +func (a *autoScalingMock) SetDesiredCapacity(input *autoscaling.SetDesiredCapacityInput) (*autoscaling.SetDesiredCapacityOutput, error) { + args := a.Called(input) + return args.Get(0).(*autoscaling.SetDesiredCapacityOutput), nil +} + +func (a *autoScalingMock) TerminateInstanceInAutoScalingGroup(input *autoscaling.TerminateInstanceInAutoScalingGroupInput) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error) { + args := a.Called(input) + return args.Get(0).(*autoscaling.TerminateInstanceInAutoScalingGroupOutput), nil +} + +type ec2Mock struct { + mock.Mock +} + +func (e *ec2Mock) DescribeLaunchTemplateVersions(i *ec2.DescribeLaunchTemplateVersionsInput) (*ec2.DescribeLaunchTemplateVersionsOutput, error) { + args := e.Called(i) + return args.Get(0).(*ec2.DescribeLaunchTemplateVersionsOutput), nil +} + +var testAwsService = awsWrapper{&autoScalingMock{}, &ec2Mock{}} + +func TestMoreThen100Groups(t *testing.T) { + a := &autoScalingMock{} + awsWrapper := &awsWrapper{ + autoScalingI: a, + ec2I: nil, + } + + // Generate 101 ASG names + names := make([]string, 101) + for i := 0; i < len(names); i++ { + names[i] = fmt.Sprintf("asg-%d", i) + } + + // First batch, first 100 elements + a.On("DescribeAutoScalingGroupsPages", + &autoscaling.DescribeAutoScalingGroupsInput{ + AutoScalingGroupNames: aws.StringSlice(names[:100]), + MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + }, + mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), + ).Run(func(args mock.Arguments) { + fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) + fn(testNamedDescribeAutoScalingGroupsOutput("asg-1", 1, "test-instance-id"), false) + }).Return(nil) + + // Second batch, element 101 + a.On("DescribeAutoScalingGroupsPages", + &autoscaling.DescribeAutoScalingGroupsInput{ + AutoScalingGroupNames: aws.StringSlice([]string{"asg-100"}), + MaxRecords: aws.Int64(maxRecordsReturnedByAPI), + }, + mock.AnythingOfType("func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool"), + ).Run(func(args mock.Arguments) { + fn := args.Get(1).(func(*autoscaling.DescribeAutoScalingGroupsOutput, bool) bool) + fn(testNamedDescribeAutoScalingGroupsOutput("asg-2", 1, "test-instance-id"), false) + }).Return(nil) + + asgs, err := awsWrapper.getAutoscalingGroupsByNames(names) + assert.Nil(t, err) + assert.Equal(t, len(asgs), 2) + assert.Equal(t, *asgs[0].AutoScalingGroupName, "asg-1") + assert.Equal(t, *asgs[1].AutoScalingGroupName, "asg-2") +} + +func TestGetInstanceTypesForAsgs(t *testing.T) { + asgName, ltName, ltVersion, instanceType := "testasg", "launcher", "1", "t2.large" + ltSpec := launchTemplate{ + name: ltName, + version: ltVersion, + } + + a := &autoScalingMock{} + e := &ec2Mock{} + a.On("DescribeLaunchConfigurations", &autoscaling.DescribeLaunchConfigurationsInput{ + LaunchConfigurationNames: []*string{aws.String(ltName)}, + MaxRecords: aws.Int64(50), + }).Return(&autoscaling.DescribeLaunchConfigurationsOutput{ + LaunchConfigurations: []*autoscaling.LaunchConfiguration{ + { + LaunchConfigurationName: aws.String(ltName), + InstanceType: aws.String(instanceType), + }, + }, + }) + e.On("DescribeLaunchTemplateVersions", &ec2.DescribeLaunchTemplateVersionsInput{ + LaunchTemplateName: aws.String(ltName), + Versions: []*string{aws.String(ltVersion)}, + }).Return(&ec2.DescribeLaunchTemplateVersionsOutput{ + LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{ + { + LaunchTemplateData: &ec2.ResponseLaunchTemplateData{ + InstanceType: aws.String(instanceType), + }, + }, + }, + }) + + // #1449 Without AWS_REGION getRegion() lookup runs till timeout during tests. + defer resetAWSRegion(os.LookupEnv("AWS_REGION")) + os.Setenv("AWS_REGION", "fanghorn") + + awsWrapper := &awsWrapper{ + autoScalingI: a, + ec2I: e, + } + + cases := []struct { + name string + launchConfigurationName string + launchTemplate *launchTemplate + mixedInstancesPolicy *mixedInstancesPolicy + }{ + { + "AsgWithLaunchConfiguration", + ltName, + nil, + nil, + }, + { + "AsgWithLaunchTemplate", + "", + <Spec, + nil, + }, + { + "AsgWithLaunchTemplateMixedInstancePolicyOverride", + "", + nil, + &mixedInstancesPolicy{ + instanceTypesOverrides: []string{instanceType}, + }, + }, + { + "AsgWithLaunchTemplateMixedInstancePolicyNoOverride", + "", + nil, + &mixedInstancesPolicy{ + launchTemplate: <Spec, + }, + }, + } + + for _, tc := range cases { + results, err := awsWrapper.getInstanceTypesForAsgs([]*asg{ + { + AwsRef: AwsRef{Name: asgName}, + LaunchConfigurationName: tc.launchConfigurationName, + LaunchTemplate: tc.launchTemplate, + MixedInstancesPolicy: tc.mixedInstancesPolicy, + }, + }) + assert.NoError(t, err) + + foundInstanceType, exists := results[asgName] + assert.NoErrorf(t, err, "%s had error %v", tc.name, err) + assert.Truef(t, exists, "%s did not find asg", tc.name) + assert.Equalf(t, foundInstanceType, instanceType, "%s had %s, expected %s", tc.name, foundInstanceType, instanceType) + } +} + +func TestBuildLaunchTemplateFromSpec(t *testing.T) { + assert := assert.New(t) + + units := []struct { + name string + in *autoscaling.LaunchTemplateSpecification + exp *launchTemplate + }{ + { + name: "non-default, specified version", + in: &autoscaling.LaunchTemplateSpecification{ + LaunchTemplateName: aws.String("foo"), + Version: aws.String("1"), + }, + exp: &launchTemplate{ + name: "foo", + version: "1", + }, + }, + { + name: "non-default, specified $Latest", + in: &autoscaling.LaunchTemplateSpecification{ + LaunchTemplateName: aws.String("foo"), + Version: aws.String("$Latest"), + }, + exp: &launchTemplate{ + name: "foo", + version: "$Latest", + }, + }, + { + name: "specified $Default", + in: &autoscaling.LaunchTemplateSpecification{ + LaunchTemplateName: aws.String("foo"), + Version: aws.String("$Default"), + }, + exp: &launchTemplate{ + name: "foo", + version: "$Default", + }, + }, + { + name: "no version specified", + in: &autoscaling.LaunchTemplateSpecification{ + LaunchTemplateName: aws.String("foo"), + Version: nil, + }, + exp: &launchTemplate{ + name: "foo", + version: "$Default", + }, + }, + } + + for _, unit := range units { + got := buildLaunchTemplateFromSpec(unit.in) + assert.Equal(unit.exp, got) + } +} diff --git a/cluster-autoscaler/cloudprovider/aws/ec2.go b/cluster-autoscaler/cloudprovider/aws/ec2.go deleted file mode 100644 index dee33760ee1a..000000000000 --- a/cluster-autoscaler/cloudprovider/aws/ec2.go +++ /dev/null @@ -1,57 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package aws - -import ( - "fmt" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" -) - -type ec2I interface { - DescribeLaunchTemplateVersions(input *ec2.DescribeLaunchTemplateVersionsInput) (*ec2.DescribeLaunchTemplateVersionsOutput, error) -} - -type ec2Wrapper struct { - ec2I -} - -func (m ec2Wrapper) getInstanceTypeByLT(launchTemplate *launchTemplate) (string, error) { - params := &ec2.DescribeLaunchTemplateVersionsInput{ - LaunchTemplateName: aws.String(launchTemplate.name), - Versions: []*string{aws.String(launchTemplate.version)}, - } - - describeData, err := m.DescribeLaunchTemplateVersions(params) - if err != nil { - return "", err - } - - if len(describeData.LaunchTemplateVersions) == 0 { - return "", fmt.Errorf("unable to find template versions") - } - - lt := describeData.LaunchTemplateVersions[0] - instanceType := lt.LaunchTemplateData.InstanceType - - if instanceType == nil { - return "", fmt.Errorf("unable to find instance type within launch template") - } - - return aws.StringValue(instanceType), nil -} diff --git a/cluster-autoscaler/cloudprovider/aws/ec2_instance_types.go b/cluster-autoscaler/cloudprovider/aws/ec2_instance_types.go index dc6a6032ae0c..aae69c9c9e09 100644 --- a/cluster-autoscaler/cloudprovider/aws/ec2_instance_types.go +++ b/cluster-autoscaler/cloudprovider/aws/ec2_instance_types.go @@ -24,3728 +24,3407 @@ type InstanceType struct { VCPU int64 MemoryMb int64 GPU int64 + Architecture string } +// StaticListLastUpdateTime is a string declaring the last time the static list was updated. +var StaticListLastUpdateTime = "2022-02-16" + // InstanceTypes is a map of ec2 resources var InstanceTypes = map[string]*InstanceType{ - "a1": { - InstanceType: "a1", - VCPU: 16, - MemoryMb: 0, - GPU: 0, - }, "a1.2xlarge": { InstanceType: "a1.2xlarge", VCPU: 8, MemoryMb: 16384, GPU: 0, + Architecture: "arm64", }, "a1.4xlarge": { InstanceType: "a1.4xlarge", VCPU: 16, MemoryMb: 32768, GPU: 0, + Architecture: "arm64", }, "a1.large": { InstanceType: "a1.large", VCPU: 2, MemoryMb: 4096, GPU: 0, + Architecture: "arm64", }, "a1.medium": { InstanceType: "a1.medium", VCPU: 1, MemoryMb: 2048, GPU: 0, + Architecture: "arm64", }, "a1.metal": { InstanceType: "a1.metal", VCPU: 16, MemoryMb: 32768, GPU: 0, + Architecture: "arm64", }, "a1.xlarge": { InstanceType: "a1.xlarge", VCPU: 4, MemoryMb: 8192, GPU: 0, + Architecture: "arm64", }, "c1.medium": { InstanceType: "c1.medium", VCPU: 2, MemoryMb: 1740, GPU: 0, + Architecture: "amd64", }, "c1.xlarge": { InstanceType: "c1.xlarge", VCPU: 8, MemoryMb: 7168, GPU: 0, - }, - "c3": { - InstanceType: "c3", - VCPU: 32, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "c3.2xlarge": { InstanceType: "c3.2xlarge", VCPU: 8, MemoryMb: 15360, GPU: 0, + Architecture: "amd64", }, "c3.4xlarge": { InstanceType: "c3.4xlarge", VCPU: 16, MemoryMb: 30720, GPU: 0, + Architecture: "amd64", }, "c3.8xlarge": { InstanceType: "c3.8xlarge", VCPU: 32, MemoryMb: 61440, GPU: 0, + Architecture: "amd64", }, "c3.large": { InstanceType: "c3.large", VCPU: 2, MemoryMb: 3840, GPU: 0, + Architecture: "amd64", }, "c3.xlarge": { InstanceType: "c3.xlarge", VCPU: 4, MemoryMb: 7680, GPU: 0, - }, - "c4": { - InstanceType: "c4", - VCPU: 36, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "c4.2xlarge": { InstanceType: "c4.2xlarge", VCPU: 8, MemoryMb: 15360, GPU: 0, + Architecture: "amd64", }, "c4.4xlarge": { InstanceType: "c4.4xlarge", VCPU: 16, MemoryMb: 30720, GPU: 0, + Architecture: "amd64", }, "c4.8xlarge": { InstanceType: "c4.8xlarge", VCPU: 36, MemoryMb: 61440, GPU: 0, + Architecture: "amd64", }, "c4.large": { InstanceType: "c4.large", VCPU: 2, MemoryMb: 3840, GPU: 0, + Architecture: "amd64", }, "c4.xlarge": { InstanceType: "c4.xlarge", VCPU: 4, MemoryMb: 7680, GPU: 0, - }, - "c5": { - InstanceType: "c5", - VCPU: 72, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "c5.12xlarge": { InstanceType: "c5.12xlarge", VCPU: 48, MemoryMb: 98304, GPU: 0, + Architecture: "amd64", }, "c5.18xlarge": { InstanceType: "c5.18xlarge", VCPU: 72, MemoryMb: 147456, GPU: 0, + Architecture: "amd64", }, "c5.24xlarge": { InstanceType: "c5.24xlarge", VCPU: 96, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "c5.2xlarge": { InstanceType: "c5.2xlarge", VCPU: 8, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "c5.4xlarge": { InstanceType: "c5.4xlarge", VCPU: 16, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "c5.9xlarge": { InstanceType: "c5.9xlarge", VCPU: 36, MemoryMb: 73728, GPU: 0, + Architecture: "amd64", }, "c5.large": { InstanceType: "c5.large", VCPU: 2, MemoryMb: 4096, GPU: 0, + Architecture: "amd64", }, "c5.metal": { InstanceType: "c5.metal", VCPU: 96, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "c5.xlarge": { InstanceType: "c5.xlarge", VCPU: 4, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "c5a.12xlarge": { InstanceType: "c5a.12xlarge", VCPU: 48, MemoryMb: 98304, GPU: 0, + Architecture: "amd64", }, "c5a.16xlarge": { InstanceType: "c5a.16xlarge", VCPU: 64, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "c5a.24xlarge": { InstanceType: "c5a.24xlarge", VCPU: 96, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "c5a.2xlarge": { InstanceType: "c5a.2xlarge", VCPU: 8, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "c5a.4xlarge": { InstanceType: "c5a.4xlarge", VCPU: 16, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "c5a.8xlarge": { InstanceType: "c5a.8xlarge", VCPU: 32, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "c5a.large": { InstanceType: "c5a.large", VCPU: 2, MemoryMb: 4096, GPU: 0, + Architecture: "amd64", }, "c5a.xlarge": { InstanceType: "c5a.xlarge", VCPU: 4, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "c5ad.12xlarge": { InstanceType: "c5ad.12xlarge", VCPU: 48, MemoryMb: 98304, GPU: 0, + Architecture: "amd64", }, "c5ad.16xlarge": { InstanceType: "c5ad.16xlarge", VCPU: 64, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "c5ad.24xlarge": { InstanceType: "c5ad.24xlarge", VCPU: 96, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "c5ad.2xlarge": { InstanceType: "c5ad.2xlarge", VCPU: 8, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "c5ad.4xlarge": { InstanceType: "c5ad.4xlarge", VCPU: 16, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "c5ad.8xlarge": { InstanceType: "c5ad.8xlarge", VCPU: 32, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "c5ad.large": { InstanceType: "c5ad.large", VCPU: 2, MemoryMb: 4096, GPU: 0, + Architecture: "amd64", }, "c5ad.xlarge": { InstanceType: "c5ad.xlarge", VCPU: 4, MemoryMb: 8192, GPU: 0, - }, - "c5d": { - InstanceType: "c5d", - VCPU: 72, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "c5d.12xlarge": { InstanceType: "c5d.12xlarge", VCPU: 48, MemoryMb: 98304, GPU: 0, + Architecture: "amd64", }, "c5d.18xlarge": { InstanceType: "c5d.18xlarge", VCPU: 72, MemoryMb: 147456, GPU: 0, + Architecture: "amd64", }, "c5d.24xlarge": { InstanceType: "c5d.24xlarge", VCPU: 96, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "c5d.2xlarge": { InstanceType: "c5d.2xlarge", VCPU: 8, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "c5d.4xlarge": { InstanceType: "c5d.4xlarge", VCPU: 16, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "c5d.9xlarge": { InstanceType: "c5d.9xlarge", VCPU: 36, MemoryMb: 73728, GPU: 0, + Architecture: "amd64", }, "c5d.large": { InstanceType: "c5d.large", VCPU: 2, MemoryMb: 4096, GPU: 0, + Architecture: "amd64", }, "c5d.metal": { InstanceType: "c5d.metal", VCPU: 96, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "c5d.xlarge": { InstanceType: "c5d.xlarge", VCPU: 4, MemoryMb: 8192, GPU: 0, - }, - "c5n": { - InstanceType: "c5n", - VCPU: 72, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "c5n.18xlarge": { InstanceType: "c5n.18xlarge", VCPU: 72, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "c5n.2xlarge": { InstanceType: "c5n.2xlarge", VCPU: 8, MemoryMb: 21504, GPU: 0, + Architecture: "amd64", }, "c5n.4xlarge": { InstanceType: "c5n.4xlarge", VCPU: 16, MemoryMb: 43008, GPU: 0, + Architecture: "amd64", }, "c5n.9xlarge": { InstanceType: "c5n.9xlarge", VCPU: 36, MemoryMb: 98304, GPU: 0, + Architecture: "amd64", }, "c5n.large": { InstanceType: "c5n.large", VCPU: 2, MemoryMb: 5376, GPU: 0, + Architecture: "amd64", }, "c5n.metal": { InstanceType: "c5n.metal", VCPU: 72, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "c5n.xlarge": { InstanceType: "c5n.xlarge", VCPU: 4, MemoryMb: 10752, GPU: 0, - }, - "c6a": { - InstanceType: "c6a", - VCPU: 192, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "c6a.12xlarge": { InstanceType: "c6a.12xlarge", VCPU: 48, MemoryMb: 98304, GPU: 0, + Architecture: "amd64", }, "c6a.16xlarge": { InstanceType: "c6a.16xlarge", VCPU: 64, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "c6a.24xlarge": { InstanceType: "c6a.24xlarge", VCPU: 96, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "c6a.2xlarge": { InstanceType: "c6a.2xlarge", VCPU: 8, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "c6a.32xlarge": { InstanceType: "c6a.32xlarge", VCPU: 128, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "c6a.48xlarge": { InstanceType: "c6a.48xlarge", VCPU: 192, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "c6a.4xlarge": { InstanceType: "c6a.4xlarge", VCPU: 16, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "c6a.8xlarge": { InstanceType: "c6a.8xlarge", VCPU: 32, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "c6a.large": { InstanceType: "c6a.large", VCPU: 2, MemoryMb: 4096, GPU: 0, - }, - "c6a.metal": { - InstanceType: "c6a.metal", - VCPU: 192, - MemoryMb: 393216, - GPU: 0, + Architecture: "amd64", }, "c6a.xlarge": { InstanceType: "c6a.xlarge", VCPU: 4, MemoryMb: 8192, GPU: 0, - }, - "c6g": { - InstanceType: "c6g", - VCPU: 64, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "c6g.12xlarge": { InstanceType: "c6g.12xlarge", VCPU: 48, MemoryMb: 98304, GPU: 0, + Architecture: "arm64", }, "c6g.16xlarge": { InstanceType: "c6g.16xlarge", VCPU: 64, MemoryMb: 131072, GPU: 0, + Architecture: "arm64", }, "c6g.2xlarge": { InstanceType: "c6g.2xlarge", VCPU: 8, MemoryMb: 16384, GPU: 0, + Architecture: "arm64", }, "c6g.4xlarge": { InstanceType: "c6g.4xlarge", VCPU: 16, MemoryMb: 32768, GPU: 0, + Architecture: "arm64", }, "c6g.8xlarge": { InstanceType: "c6g.8xlarge", VCPU: 32, MemoryMb: 65536, GPU: 0, + Architecture: "arm64", }, "c6g.large": { InstanceType: "c6g.large", VCPU: 2, MemoryMb: 4096, GPU: 0, + Architecture: "arm64", }, "c6g.medium": { InstanceType: "c6g.medium", VCPU: 1, MemoryMb: 2048, GPU: 0, + Architecture: "arm64", }, "c6g.metal": { InstanceType: "c6g.metal", VCPU: 64, MemoryMb: 131072, GPU: 0, + Architecture: "arm64", }, "c6g.xlarge": { InstanceType: "c6g.xlarge", VCPU: 4, MemoryMb: 8192, GPU: 0, - }, - "c6gd": { - InstanceType: "c6gd", - VCPU: 64, - MemoryMb: 0, - GPU: 0, + Architecture: "arm64", }, "c6gd.12xlarge": { InstanceType: "c6gd.12xlarge", VCPU: 48, MemoryMb: 98304, GPU: 0, + Architecture: "arm64", }, "c6gd.16xlarge": { InstanceType: "c6gd.16xlarge", VCPU: 64, MemoryMb: 131072, GPU: 0, + Architecture: "arm64", }, "c6gd.2xlarge": { InstanceType: "c6gd.2xlarge", VCPU: 8, MemoryMb: 16384, GPU: 0, + Architecture: "arm64", }, "c6gd.4xlarge": { InstanceType: "c6gd.4xlarge", VCPU: 16, MemoryMb: 32768, GPU: 0, + Architecture: "arm64", }, "c6gd.8xlarge": { InstanceType: "c6gd.8xlarge", VCPU: 32, MemoryMb: 65536, GPU: 0, + Architecture: "arm64", }, "c6gd.large": { InstanceType: "c6gd.large", VCPU: 2, MemoryMb: 4096, GPU: 0, + Architecture: "arm64", }, "c6gd.medium": { InstanceType: "c6gd.medium", VCPU: 1, MemoryMb: 2048, GPU: 0, + Architecture: "arm64", }, "c6gd.metal": { InstanceType: "c6gd.metal", VCPU: 64, MemoryMb: 131072, GPU: 0, + Architecture: "arm64", }, "c6gd.xlarge": { InstanceType: "c6gd.xlarge", VCPU: 4, MemoryMb: 8192, GPU: 0, - }, - "c6gn": { - InstanceType: "c6gn", - VCPU: 64, - MemoryMb: 0, - GPU: 0, + Architecture: "arm64", }, "c6gn.12xlarge": { InstanceType: "c6gn.12xlarge", VCPU: 48, MemoryMb: 98304, GPU: 0, + Architecture: "arm64", }, "c6gn.16xlarge": { InstanceType: "c6gn.16xlarge", VCPU: 64, MemoryMb: 131072, GPU: 0, + Architecture: "arm64", }, "c6gn.2xlarge": { InstanceType: "c6gn.2xlarge", VCPU: 8, MemoryMb: 16384, GPU: 0, + Architecture: "arm64", }, "c6gn.4xlarge": { InstanceType: "c6gn.4xlarge", VCPU: 16, MemoryMb: 32768, GPU: 0, + Architecture: "arm64", }, "c6gn.8xlarge": { InstanceType: "c6gn.8xlarge", VCPU: 32, MemoryMb: 65536, GPU: 0, + Architecture: "arm64", }, "c6gn.large": { InstanceType: "c6gn.large", VCPU: 2, MemoryMb: 4096, GPU: 0, + Architecture: "arm64", }, "c6gn.medium": { InstanceType: "c6gn.medium", VCPU: 1, MemoryMb: 2048, GPU: 0, - }, - "c6gn.metal": { - InstanceType: "c6gn.metal", - VCPU: 64, - MemoryMb: 131072, - GPU: 0, + Architecture: "arm64", }, "c6gn.xlarge": { InstanceType: "c6gn.xlarge", VCPU: 4, MemoryMb: 8192, GPU: 0, - }, - "c6i": { - InstanceType: "c6i", - VCPU: 128, - MemoryMb: 0, - GPU: 0, + Architecture: "arm64", }, "c6i.12xlarge": { InstanceType: "c6i.12xlarge", VCPU: 48, MemoryMb: 98304, GPU: 0, + Architecture: "amd64", }, "c6i.16xlarge": { InstanceType: "c6i.16xlarge", VCPU: 64, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "c6i.24xlarge": { InstanceType: "c6i.24xlarge", VCPU: 96, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "c6i.2xlarge": { InstanceType: "c6i.2xlarge", VCPU: 8, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "c6i.32xlarge": { InstanceType: "c6i.32xlarge", VCPU: 128, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "c6i.4xlarge": { InstanceType: "c6i.4xlarge", VCPU: 16, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "c6i.8xlarge": { InstanceType: "c6i.8xlarge", VCPU: 32, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "c6i.large": { InstanceType: "c6i.large", VCPU: 2, MemoryMb: 4096, GPU: 0, + Architecture: "amd64", }, "c6i.metal": { InstanceType: "c6i.metal", VCPU: 128, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "c6i.xlarge": { InstanceType: "c6i.xlarge", VCPU: 4, MemoryMb: 8192, GPU: 0, - }, - "c6id": { - InstanceType: "c6id", - VCPU: 128, - MemoryMb: 0, - GPU: 0, - }, - "c6id.12xlarge": { - InstanceType: "c6id.12xlarge", - VCPU: 48, - MemoryMb: 98304, - GPU: 0, - }, - "c6id.16xlarge": { - InstanceType: "c6id.16xlarge", - VCPU: 64, - MemoryMb: 131072, - GPU: 0, - }, - "c6id.24xlarge": { - InstanceType: "c6id.24xlarge", - VCPU: 96, - MemoryMb: 196608, - GPU: 0, - }, - "c6id.2xlarge": { - InstanceType: "c6id.2xlarge", - VCPU: 8, - MemoryMb: 16384, - GPU: 0, - }, - "c6id.32xlarge": { - InstanceType: "c6id.32xlarge", - VCPU: 128, - MemoryMb: 262144, - GPU: 0, - }, - "c6id.4xlarge": { - InstanceType: "c6id.4xlarge", - VCPU: 16, - MemoryMb: 32768, - GPU: 0, - }, - "c6id.8xlarge": { - InstanceType: "c6id.8xlarge", - VCPU: 32, - MemoryMb: 65536, - GPU: 0, - }, - "c6id.large": { - InstanceType: "c6id.large", - VCPU: 2, - MemoryMb: 4096, - GPU: 0, - }, - "c6id.metal": { - InstanceType: "c6id.metal", - VCPU: 128, - MemoryMb: 262144, - GPU: 0, - }, - "c6id.xlarge": { - InstanceType: "c6id.xlarge", - VCPU: 4, - MemoryMb: 8192, - GPU: 0, - }, - "c7g": { - InstanceType: "c7g", - VCPU: 64, - MemoryMb: 0, - GPU: 0, - }, - "c7g.12xlarge": { - InstanceType: "c7g.12xlarge", - VCPU: 48, - MemoryMb: 98304, - GPU: 0, - }, - "c7g.16xlarge": { - InstanceType: "c7g.16xlarge", - VCPU: 64, - MemoryMb: 131072, - GPU: 0, - }, - "c7g.2xlarge": { - InstanceType: "c7g.2xlarge", - VCPU: 8, - MemoryMb: 16384, - GPU: 0, - }, - "c7g.4xlarge": { - InstanceType: "c7g.4xlarge", - VCPU: 16, - MemoryMb: 32768, - GPU: 0, - }, - "c7g.8xlarge": { - InstanceType: "c7g.8xlarge", - VCPU: 32, - MemoryMb: 65536, - GPU: 0, - }, - "c7g.large": { - InstanceType: "c7g.large", - VCPU: 2, - MemoryMb: 4096, - GPU: 0, - }, - "c7g.medium": { - InstanceType: "c7g.medium", - VCPU: 1, - MemoryMb: 2048, - GPU: 0, - }, - "c7g.xlarge": { - InstanceType: "c7g.xlarge", - VCPU: 4, - MemoryMb: 8192, - GPU: 0, + Architecture: "amd64", }, "cc2.8xlarge": { InstanceType: "cc2.8xlarge", VCPU: 32, MemoryMb: 61952, GPU: 0, - }, - "cr1.8xlarge": { - InstanceType: "cr1.8xlarge", - VCPU: 32, - MemoryMb: 249856, - GPU: 0, - }, - "d2": { - InstanceType: "d2", - VCPU: 36, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "d2.2xlarge": { InstanceType: "d2.2xlarge", VCPU: 8, MemoryMb: 62464, GPU: 0, + Architecture: "amd64", }, "d2.4xlarge": { InstanceType: "d2.4xlarge", VCPU: 16, MemoryMb: 124928, GPU: 0, + Architecture: "amd64", }, "d2.8xlarge": { InstanceType: "d2.8xlarge", VCPU: 36, MemoryMb: 249856, GPU: 0, + Architecture: "amd64", }, "d2.xlarge": { InstanceType: "d2.xlarge", VCPU: 4, MemoryMb: 31232, GPU: 0, + Architecture: "amd64", }, "d3.2xlarge": { InstanceType: "d3.2xlarge", VCPU: 8, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "d3.4xlarge": { InstanceType: "d3.4xlarge", VCPU: 16, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "d3.8xlarge": { InstanceType: "d3.8xlarge", VCPU: 32, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "d3.xlarge": { InstanceType: "d3.xlarge", VCPU: 4, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "d3en.12xlarge": { InstanceType: "d3en.12xlarge", VCPU: 48, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "d3en.2xlarge": { InstanceType: "d3en.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "d3en.4xlarge": { InstanceType: "d3en.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "d3en.6xlarge": { InstanceType: "d3en.6xlarge", VCPU: 24, MemoryMb: 98304, GPU: 0, + Architecture: "amd64", }, "d3en.8xlarge": { InstanceType: "d3en.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "d3en.xlarge": { InstanceType: "d3en.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, - }, - "dl1": { - InstanceType: "dl1", - VCPU: 96, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "dl1.24xlarge": { InstanceType: "dl1.24xlarge", VCPU: 96, MemoryMb: 786432, - GPU: 0, - }, - "f1": { - InstanceType: "f1", - VCPU: 64, - MemoryMb: 0, - GPU: 0, + GPU: 8, + Architecture: "amd64", }, "f1.16xlarge": { InstanceType: "f1.16xlarge", VCPU: 64, MemoryMb: 999424, GPU: 0, + Architecture: "amd64", }, "f1.2xlarge": { InstanceType: "f1.2xlarge", VCPU: 8, MemoryMb: 124928, GPU: 0, + Architecture: "amd64", }, "f1.4xlarge": { InstanceType: "f1.4xlarge", VCPU: 16, MemoryMb: 249856, GPU: 0, - }, - "g2": { - InstanceType: "g2", - VCPU: 32, - MemoryMb: 0, - GPU: 4, + Architecture: "amd64", }, "g2.2xlarge": { InstanceType: "g2.2xlarge", VCPU: 8, MemoryMb: 15360, GPU: 1, + Architecture: "amd64", }, "g2.8xlarge": { InstanceType: "g2.8xlarge", VCPU: 32, MemoryMb: 61440, GPU: 4, - }, - "g3": { - InstanceType: "g3", - VCPU: 64, - MemoryMb: 0, - GPU: 4, + Architecture: "amd64", }, "g3.16xlarge": { InstanceType: "g3.16xlarge", VCPU: 64, MemoryMb: 499712, GPU: 4, + Architecture: "amd64", }, "g3.4xlarge": { InstanceType: "g3.4xlarge", VCPU: 16, MemoryMb: 124928, GPU: 1, + Architecture: "amd64", }, "g3.8xlarge": { InstanceType: "g3.8xlarge", VCPU: 32, MemoryMb: 249856, GPU: 2, + Architecture: "amd64", }, "g3s.xlarge": { InstanceType: "g3s.xlarge", VCPU: 4, MemoryMb: 31232, GPU: 1, - }, - "g4ad": { - InstanceType: "g4ad", - VCPU: 192, - MemoryMb: 0, - GPU: 2, + Architecture: "amd64", }, "g4ad.16xlarge": { InstanceType: "g4ad.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 4, + Architecture: "amd64", }, "g4ad.2xlarge": { InstanceType: "g4ad.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 1, + Architecture: "amd64", }, "g4ad.4xlarge": { InstanceType: "g4ad.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 1, + Architecture: "amd64", }, "g4ad.8xlarge": { InstanceType: "g4ad.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 2, + Architecture: "amd64", }, "g4ad.xlarge": { InstanceType: "g4ad.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 1, - }, - "g4dn": { - InstanceType: "g4dn", - VCPU: 96, - MemoryMb: 0, - GPU: 8, + Architecture: "amd64", }, "g4dn.12xlarge": { InstanceType: "g4dn.12xlarge", VCPU: 48, MemoryMb: 196608, GPU: 4, + Architecture: "amd64", }, "g4dn.16xlarge": { InstanceType: "g4dn.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 1, + Architecture: "amd64", }, "g4dn.2xlarge": { InstanceType: "g4dn.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 1, + Architecture: "amd64", }, "g4dn.4xlarge": { InstanceType: "g4dn.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 1, + Architecture: "amd64", }, "g4dn.8xlarge": { InstanceType: "g4dn.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 1, + Architecture: "amd64", }, "g4dn.metal": { InstanceType: "g4dn.metal", VCPU: 96, MemoryMb: 393216, GPU: 8, + Architecture: "amd64", }, "g4dn.xlarge": { InstanceType: "g4dn.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 1, - }, - "g5": { - InstanceType: "g5", - VCPU: 192, - MemoryMb: 0, - GPU: 8, + Architecture: "amd64", }, "g5.12xlarge": { InstanceType: "g5.12xlarge", VCPU: 48, MemoryMb: 196608, GPU: 4, + Architecture: "amd64", }, "g5.16xlarge": { InstanceType: "g5.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 1, + Architecture: "amd64", }, "g5.24xlarge": { InstanceType: "g5.24xlarge", VCPU: 96, MemoryMb: 393216, GPU: 4, + Architecture: "amd64", }, "g5.2xlarge": { InstanceType: "g5.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 1, + Architecture: "amd64", }, "g5.48xlarge": { InstanceType: "g5.48xlarge", VCPU: 192, MemoryMb: 786432, GPU: 8, + Architecture: "amd64", }, "g5.4xlarge": { InstanceType: "g5.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 1, + Architecture: "amd64", }, "g5.8xlarge": { InstanceType: "g5.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 1, + Architecture: "amd64", }, "g5.xlarge": { InstanceType: "g5.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 1, - }, - "g5g": { - InstanceType: "g5g", - VCPU: 64, - MemoryMb: 0, - GPU: 2, + Architecture: "amd64", }, "g5g.16xlarge": { InstanceType: "g5g.16xlarge", VCPU: 64, MemoryMb: 131072, GPU: 2, + Architecture: "arm64", }, "g5g.2xlarge": { InstanceType: "g5g.2xlarge", VCPU: 8, MemoryMb: 16384, GPU: 1, + Architecture: "arm64", }, "g5g.4xlarge": { InstanceType: "g5g.4xlarge", VCPU: 16, MemoryMb: 32768, GPU: 1, + Architecture: "arm64", }, "g5g.8xlarge": { InstanceType: "g5g.8xlarge", VCPU: 32, MemoryMb: 65536, GPU: 1, + Architecture: "arm64", }, "g5g.metal": { InstanceType: "g5g.metal", VCPU: 64, MemoryMb: 131072, GPU: 2, + Architecture: "arm64", }, "g5g.xlarge": { InstanceType: "g5g.xlarge", VCPU: 4, MemoryMb: 8192, GPU: 1, - }, - "h1": { - InstanceType: "h1", - VCPU: 64, - MemoryMb: 0, - GPU: 0, + Architecture: "arm64", }, "h1.16xlarge": { InstanceType: "h1.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "h1.2xlarge": { InstanceType: "h1.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "h1.4xlarge": { InstanceType: "h1.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "h1.8xlarge": { InstanceType: "h1.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 0, - }, - "hpc6a.48xlarge": { - InstanceType: "hpc6a.48xlarge", - VCPU: 96, - MemoryMb: 393216, - GPU: 0, - }, - "hs1.8xlarge": { - InstanceType: "hs1.8xlarge", - VCPU: 16, - MemoryMb: 119808, - GPU: 0, - }, - "i2": { - InstanceType: "i2", - VCPU: 32, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "i2.2xlarge": { InstanceType: "i2.2xlarge", VCPU: 8, MemoryMb: 62464, GPU: 0, + Architecture: "amd64", }, "i2.4xlarge": { InstanceType: "i2.4xlarge", VCPU: 16, MemoryMb: 124928, GPU: 0, + Architecture: "amd64", }, "i2.8xlarge": { InstanceType: "i2.8xlarge", VCPU: 32, MemoryMb: 249856, GPU: 0, - }, - "i2.large": { - InstanceType: "i2.large", - VCPU: 2, - MemoryMb: 15360, - GPU: 0, + Architecture: "amd64", }, "i2.xlarge": { InstanceType: "i2.xlarge", VCPU: 4, MemoryMb: 31232, GPU: 0, - }, - "i3": { - InstanceType: "i3", - VCPU: 64, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "i3.16xlarge": { InstanceType: "i3.16xlarge", VCPU: 64, MemoryMb: 499712, GPU: 0, + Architecture: "amd64", }, "i3.2xlarge": { InstanceType: "i3.2xlarge", VCPU: 8, MemoryMb: 62464, GPU: 0, + Architecture: "amd64", }, "i3.4xlarge": { InstanceType: "i3.4xlarge", VCPU: 16, MemoryMb: 124928, GPU: 0, + Architecture: "amd64", }, "i3.8xlarge": { InstanceType: "i3.8xlarge", VCPU: 32, MemoryMb: 249856, GPU: 0, + Architecture: "amd64", }, "i3.large": { InstanceType: "i3.large", VCPU: 2, MemoryMb: 15616, GPU: 0, + Architecture: "amd64", }, "i3.metal": { InstanceType: "i3.metal", - VCPU: 64, + VCPU: 72, MemoryMb: 524288, GPU: 0, + Architecture: "amd64", }, "i3.xlarge": { InstanceType: "i3.xlarge", VCPU: 4, MemoryMb: 31232, GPU: 0, - }, - "i3en": { - InstanceType: "i3en", - VCPU: 96, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "i3en.12xlarge": { InstanceType: "i3en.12xlarge", VCPU: 48, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "i3en.24xlarge": { InstanceType: "i3en.24xlarge", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "i3en.2xlarge": { InstanceType: "i3en.2xlarge", VCPU: 8, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "i3en.3xlarge": { InstanceType: "i3en.3xlarge", VCPU: 12, MemoryMb: 98304, GPU: 0, + Architecture: "amd64", }, "i3en.6xlarge": { InstanceType: "i3en.6xlarge", VCPU: 24, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "i3en.large": { InstanceType: "i3en.large", VCPU: 2, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "i3en.metal": { InstanceType: "i3en.metal", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "i3en.xlarge": { InstanceType: "i3en.xlarge", VCPU: 4, MemoryMb: 32768, GPU: 0, - }, - "i3p.16xlarge": { - InstanceType: "i3p.16xlarge", - VCPU: 64, - MemoryMb: 499712, - GPU: 0, - }, - "i4i": { - InstanceType: "i4i", - VCPU: 128, - MemoryMb: 0, - GPU: 0, - }, - "i4i.16xlarge": { - InstanceType: "i4i.16xlarge", - VCPU: 64, - MemoryMb: 524288, - GPU: 0, - }, - "i4i.2xlarge": { - InstanceType: "i4i.2xlarge", - VCPU: 8, - MemoryMb: 65536, - GPU: 0, - }, - "i4i.32xlarge": { - InstanceType: "i4i.32xlarge", - VCPU: 128, - MemoryMb: 1048576, - GPU: 0, - }, - "i4i.4xlarge": { - InstanceType: "i4i.4xlarge", - VCPU: 16, - MemoryMb: 131072, - GPU: 0, - }, - "i4i.8xlarge": { - InstanceType: "i4i.8xlarge", - VCPU: 32, - MemoryMb: 262144, - GPU: 0, - }, - "i4i.large": { - InstanceType: "i4i.large", - VCPU: 2, - MemoryMb: 16384, - GPU: 0, - }, - "i4i.metal": { - InstanceType: "i4i.metal", - VCPU: 128, - MemoryMb: 1048576, - GPU: 0, - }, - "i4i.xlarge": { - InstanceType: "i4i.xlarge", - VCPU: 4, - MemoryMb: 32768, - GPU: 0, - }, - "im4gn": { - InstanceType: "im4gn", - VCPU: 64, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "im4gn.16xlarge": { InstanceType: "im4gn.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "arm64", }, "im4gn.2xlarge": { InstanceType: "im4gn.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "arm64", }, "im4gn.4xlarge": { InstanceType: "im4gn.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 0, + Architecture: "arm64", }, "im4gn.8xlarge": { InstanceType: "im4gn.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 0, + Architecture: "arm64", }, "im4gn.large": { InstanceType: "im4gn.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "arm64", }, "im4gn.xlarge": { InstanceType: "im4gn.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, - }, - "inf1": { - InstanceType: "inf1", - VCPU: 96, - MemoryMb: 0, - GPU: 0, + Architecture: "arm64", }, "inf1.24xlarge": { InstanceType: "inf1.24xlarge", VCPU: 96, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "inf1.2xlarge": { InstanceType: "inf1.2xlarge", VCPU: 8, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "inf1.6xlarge": { InstanceType: "inf1.6xlarge", VCPU: 24, MemoryMb: 49152, GPU: 0, + Architecture: "amd64", }, "inf1.xlarge": { InstanceType: "inf1.xlarge", VCPU: 4, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "is4gen.2xlarge": { InstanceType: "is4gen.2xlarge", VCPU: 8, MemoryMb: 49152, GPU: 0, + Architecture: "arm64", }, "is4gen.4xlarge": { InstanceType: "is4gen.4xlarge", VCPU: 16, MemoryMb: 98304, GPU: 0, + Architecture: "arm64", }, "is4gen.8xlarge": { InstanceType: "is4gen.8xlarge", VCPU: 32, MemoryMb: 196608, GPU: 0, + Architecture: "arm64", }, "is4gen.large": { InstanceType: "is4gen.large", VCPU: 2, MemoryMb: 12288, GPU: 0, + Architecture: "arm64", }, "is4gen.medium": { InstanceType: "is4gen.medium", VCPU: 1, MemoryMb: 6144, GPU: 0, + Architecture: "arm64", }, "is4gen.xlarge": { InstanceType: "is4gen.xlarge", VCPU: 4, MemoryMb: 24576, GPU: 0, + Architecture: "arm64", }, "m1.large": { InstanceType: "m1.large", VCPU: 2, MemoryMb: 7680, GPU: 0, + Architecture: "amd64", }, "m1.medium": { InstanceType: "m1.medium", VCPU: 1, - MemoryMb: 3840, + MemoryMb: 3788, GPU: 0, + Architecture: "amd64", }, "m1.small": { InstanceType: "m1.small", VCPU: 1, MemoryMb: 1740, GPU: 0, + Architecture: "amd64", }, "m1.xlarge": { InstanceType: "m1.xlarge", VCPU: 4, MemoryMb: 15360, GPU: 0, + Architecture: "amd64", }, "m2.2xlarge": { InstanceType: "m2.2xlarge", VCPU: 4, MemoryMb: 35020, GPU: 0, + Architecture: "amd64", }, "m2.4xlarge": { InstanceType: "m2.4xlarge", VCPU: 8, MemoryMb: 70041, GPU: 0, + Architecture: "amd64", }, "m2.xlarge": { InstanceType: "m2.xlarge", VCPU: 2, MemoryMb: 17510, GPU: 0, - }, - "m3": { - InstanceType: "m3", - VCPU: 8, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "m3.2xlarge": { InstanceType: "m3.2xlarge", VCPU: 8, MemoryMb: 30720, GPU: 0, + Architecture: "amd64", }, "m3.large": { InstanceType: "m3.large", VCPU: 2, MemoryMb: 7680, GPU: 0, + Architecture: "amd64", }, "m3.medium": { InstanceType: "m3.medium", VCPU: 1, MemoryMb: 3840, GPU: 0, + Architecture: "amd64", }, "m3.xlarge": { InstanceType: "m3.xlarge", VCPU: 4, MemoryMb: 15360, GPU: 0, - }, - "m4": { - InstanceType: "m4", - VCPU: 40, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "m4.10xlarge": { InstanceType: "m4.10xlarge", VCPU: 40, MemoryMb: 163840, GPU: 0, + Architecture: "amd64", }, "m4.16xlarge": { InstanceType: "m4.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "m4.2xlarge": { InstanceType: "m4.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "m4.4xlarge": { InstanceType: "m4.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "m4.large": { InstanceType: "m4.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "m4.xlarge": { InstanceType: "m4.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, - }, - "m5": { - InstanceType: "m5", - VCPU: 96, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "m5.12xlarge": { InstanceType: "m5.12xlarge", VCPU: 48, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "m5.16xlarge": { InstanceType: "m5.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "m5.24xlarge": { InstanceType: "m5.24xlarge", VCPU: 96, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "m5.2xlarge": { InstanceType: "m5.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "m5.4xlarge": { InstanceType: "m5.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "m5.8xlarge": { InstanceType: "m5.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "m5.large": { InstanceType: "m5.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "m5.metal": { InstanceType: "m5.metal", VCPU: 96, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "m5.xlarge": { InstanceType: "m5.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "m5a.12xlarge": { InstanceType: "m5a.12xlarge", VCPU: 48, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "m5a.16xlarge": { InstanceType: "m5a.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "m5a.24xlarge": { InstanceType: "m5a.24xlarge", VCPU: 96, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "m5a.2xlarge": { InstanceType: "m5a.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "m5a.4xlarge": { InstanceType: "m5a.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "m5a.8xlarge": { InstanceType: "m5a.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "m5a.large": { InstanceType: "m5a.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "m5a.xlarge": { InstanceType: "m5a.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "m5ad.12xlarge": { InstanceType: "m5ad.12xlarge", VCPU: 48, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "m5ad.16xlarge": { InstanceType: "m5ad.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "m5ad.24xlarge": { InstanceType: "m5ad.24xlarge", VCPU: 96, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "m5ad.2xlarge": { InstanceType: "m5ad.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "m5ad.4xlarge": { InstanceType: "m5ad.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "m5ad.8xlarge": { InstanceType: "m5ad.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "m5ad.large": { InstanceType: "m5ad.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "m5ad.xlarge": { InstanceType: "m5ad.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, - }, - "m5d": { - InstanceType: "m5d", - VCPU: 96, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "m5d.12xlarge": { InstanceType: "m5d.12xlarge", VCPU: 48, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "m5d.16xlarge": { InstanceType: "m5d.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "m5d.24xlarge": { InstanceType: "m5d.24xlarge", VCPU: 96, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "m5d.2xlarge": { InstanceType: "m5d.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "m5d.4xlarge": { InstanceType: "m5d.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "m5d.8xlarge": { InstanceType: "m5d.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "m5d.large": { InstanceType: "m5d.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "m5d.metal": { InstanceType: "m5d.metal", VCPU: 96, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "m5d.xlarge": { InstanceType: "m5d.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, - }, - "m5dn": { - InstanceType: "m5dn", - VCPU: 96, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "m5dn.12xlarge": { InstanceType: "m5dn.12xlarge", VCPU: 48, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "m5dn.16xlarge": { InstanceType: "m5dn.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "m5dn.24xlarge": { InstanceType: "m5dn.24xlarge", VCPU: 96, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "m5dn.2xlarge": { InstanceType: "m5dn.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "m5dn.4xlarge": { InstanceType: "m5dn.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "m5dn.8xlarge": { InstanceType: "m5dn.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "m5dn.large": { InstanceType: "m5dn.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "m5dn.metal": { InstanceType: "m5dn.metal", VCPU: 96, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "m5dn.xlarge": { InstanceType: "m5dn.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, - }, - "m5n": { - InstanceType: "m5n", - VCPU: 96, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "m5n.12xlarge": { InstanceType: "m5n.12xlarge", VCPU: 48, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "m5n.16xlarge": { InstanceType: "m5n.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "m5n.24xlarge": { InstanceType: "m5n.24xlarge", VCPU: 96, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "m5n.2xlarge": { InstanceType: "m5n.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "m5n.4xlarge": { InstanceType: "m5n.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "m5n.8xlarge": { InstanceType: "m5n.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "m5n.large": { InstanceType: "m5n.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "m5n.metal": { InstanceType: "m5n.metal", VCPU: 96, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "m5n.xlarge": { InstanceType: "m5n.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, - }, - "m5zn": { - InstanceType: "m5zn", - VCPU: 48, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "m5zn.12xlarge": { InstanceType: "m5zn.12xlarge", VCPU: 48, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "m5zn.2xlarge": { InstanceType: "m5zn.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "m5zn.3xlarge": { InstanceType: "m5zn.3xlarge", VCPU: 12, MemoryMb: 49152, GPU: 0, + Architecture: "amd64", }, "m5zn.6xlarge": { InstanceType: "m5zn.6xlarge", VCPU: 24, MemoryMb: 98304, GPU: 0, + Architecture: "amd64", }, "m5zn.large": { InstanceType: "m5zn.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "m5zn.metal": { InstanceType: "m5zn.metal", VCPU: 48, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "m5zn.xlarge": { InstanceType: "m5zn.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, - }, - "m6a": { - InstanceType: "m6a", - VCPU: 192, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "m6a.12xlarge": { InstanceType: "m6a.12xlarge", VCPU: 48, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "m6a.16xlarge": { InstanceType: "m6a.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "m6a.24xlarge": { InstanceType: "m6a.24xlarge", VCPU: 96, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "m6a.2xlarge": { InstanceType: "m6a.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "m6a.32xlarge": { InstanceType: "m6a.32xlarge", VCPU: 128, MemoryMb: 524288, GPU: 0, + Architecture: "amd64", }, "m6a.48xlarge": { InstanceType: "m6a.48xlarge", VCPU: 192, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "m6a.4xlarge": { InstanceType: "m6a.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "m6a.8xlarge": { InstanceType: "m6a.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "m6a.large": { InstanceType: "m6a.large", VCPU: 2, MemoryMb: 8192, GPU: 0, - }, - "m6a.metal": { - InstanceType: "m6a.metal", - VCPU: 192, - MemoryMb: 786432, - GPU: 0, + Architecture: "amd64", }, "m6a.xlarge": { InstanceType: "m6a.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, - }, - "m6g": { - InstanceType: "m6g", - VCPU: 64, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "m6g.12xlarge": { InstanceType: "m6g.12xlarge", VCPU: 48, MemoryMb: 196608, GPU: 0, + Architecture: "arm64", }, "m6g.16xlarge": { InstanceType: "m6g.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "arm64", }, "m6g.2xlarge": { InstanceType: "m6g.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "arm64", }, "m6g.4xlarge": { InstanceType: "m6g.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 0, + Architecture: "arm64", }, "m6g.8xlarge": { InstanceType: "m6g.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 0, + Architecture: "arm64", }, "m6g.large": { InstanceType: "m6g.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "arm64", }, "m6g.medium": { InstanceType: "m6g.medium", VCPU: 1, MemoryMb: 4096, GPU: 0, + Architecture: "arm64", }, "m6g.metal": { InstanceType: "m6g.metal", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "arm64", }, "m6g.xlarge": { InstanceType: "m6g.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, - }, - "m6gd": { - InstanceType: "m6gd", - VCPU: 64, - MemoryMb: 0, - GPU: 0, + Architecture: "arm64", }, "m6gd.12xlarge": { InstanceType: "m6gd.12xlarge", VCPU: 48, MemoryMb: 196608, GPU: 0, + Architecture: "arm64", }, "m6gd.16xlarge": { InstanceType: "m6gd.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "arm64", }, "m6gd.2xlarge": { InstanceType: "m6gd.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "arm64", }, "m6gd.4xlarge": { InstanceType: "m6gd.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 0, + Architecture: "arm64", }, "m6gd.8xlarge": { InstanceType: "m6gd.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 0, + Architecture: "arm64", }, "m6gd.large": { InstanceType: "m6gd.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "arm64", }, "m6gd.medium": { InstanceType: "m6gd.medium", VCPU: 1, MemoryMb: 4096, GPU: 0, + Architecture: "arm64", }, "m6gd.metal": { InstanceType: "m6gd.metal", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "arm64", }, "m6gd.xlarge": { InstanceType: "m6gd.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, - }, - "m6i": { - InstanceType: "m6i", - VCPU: 128, - MemoryMb: 0, - GPU: 0, + Architecture: "arm64", }, "m6i.12xlarge": { InstanceType: "m6i.12xlarge", VCPU: 48, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "m6i.16xlarge": { InstanceType: "m6i.16xlarge", VCPU: 64, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "m6i.24xlarge": { InstanceType: "m6i.24xlarge", VCPU: 96, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "m6i.2xlarge": { InstanceType: "m6i.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "m6i.32xlarge": { InstanceType: "m6i.32xlarge", VCPU: 128, MemoryMb: 524288, GPU: 0, + Architecture: "amd64", }, "m6i.4xlarge": { InstanceType: "m6i.4xlarge", VCPU: 16, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "m6i.8xlarge": { InstanceType: "m6i.8xlarge", VCPU: 32, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "m6i.large": { InstanceType: "m6i.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "m6i.metal": { InstanceType: "m6i.metal", VCPU: 128, MemoryMb: 524288, GPU: 0, + Architecture: "amd64", }, "m6i.xlarge": { InstanceType: "m6i.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, - }, - "m6id": { - InstanceType: "m6id", - VCPU: 128, - MemoryMb: 0, - GPU: 0, - }, - "m6id.12xlarge": { - InstanceType: "m6id.12xlarge", - VCPU: 48, - MemoryMb: 196608, - GPU: 0, - }, - "m6id.16xlarge": { - InstanceType: "m6id.16xlarge", - VCPU: 64, - MemoryMb: 262144, - GPU: 0, - }, - "m6id.24xlarge": { - InstanceType: "m6id.24xlarge", - VCPU: 96, - MemoryMb: 393216, - GPU: 0, - }, - "m6id.2xlarge": { - InstanceType: "m6id.2xlarge", - VCPU: 8, - MemoryMb: 32768, - GPU: 0, - }, - "m6id.32xlarge": { - InstanceType: "m6id.32xlarge", - VCPU: 128, - MemoryMb: 524288, - GPU: 0, - }, - "m6id.4xlarge": { - InstanceType: "m6id.4xlarge", - VCPU: 16, - MemoryMb: 65536, - GPU: 0, - }, - "m6id.8xlarge": { - InstanceType: "m6id.8xlarge", - VCPU: 32, - MemoryMb: 131072, - GPU: 0, - }, - "m6id.large": { - InstanceType: "m6id.large", - VCPU: 2, - MemoryMb: 8192, - GPU: 0, - }, - "m6id.metal": { - InstanceType: "m6id.metal", - VCPU: 128, - MemoryMb: 524288, - GPU: 0, - }, - "m6id.xlarge": { - InstanceType: "m6id.xlarge", - VCPU: 4, - MemoryMb: 16384, - GPU: 0, - }, - "mac1": { - InstanceType: "mac1", - VCPU: 12, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "mac1.metal": { InstanceType: "mac1.metal", VCPU: 12, MemoryMb: 32768, GPU: 0, - }, - "mac2": { - InstanceType: "mac2", - VCPU: 12, - MemoryMb: 0, - GPU: 0, - }, - "mac2.metal": { - InstanceType: "mac2.metal", - VCPU: 12, - MemoryMb: 16384, - GPU: 0, - }, - "p2": { - InstanceType: "p2", - VCPU: 64, - MemoryMb: 0, - GPU: 16, + Architecture: "amd64", }, "p2.16xlarge": { InstanceType: "p2.16xlarge", VCPU: 64, MemoryMb: 749568, GPU: 16, + Architecture: "amd64", }, "p2.8xlarge": { InstanceType: "p2.8xlarge", VCPU: 32, MemoryMb: 499712, GPU: 8, + Architecture: "amd64", }, "p2.xlarge": { InstanceType: "p2.xlarge", VCPU: 4, MemoryMb: 62464, GPU: 1, - }, - "p3": { - InstanceType: "p3", - VCPU: 64, - MemoryMb: 0, - GPU: 8, + Architecture: "amd64", }, "p3.16xlarge": { InstanceType: "p3.16xlarge", VCPU: 64, MemoryMb: 499712, GPU: 8, + Architecture: "amd64", }, "p3.2xlarge": { InstanceType: "p3.2xlarge", VCPU: 8, MemoryMb: 62464, GPU: 1, + Architecture: "amd64", }, "p3.8xlarge": { InstanceType: "p3.8xlarge", VCPU: 32, MemoryMb: 249856, GPU: 4, - }, - "p3dn": { - InstanceType: "p3dn", - VCPU: 96, - MemoryMb: 0, - GPU: 8, + Architecture: "amd64", }, "p3dn.24xlarge": { InstanceType: "p3dn.24xlarge", VCPU: 96, MemoryMb: 786432, GPU: 8, - }, - "p4d": { - InstanceType: "p4d", - VCPU: 96, - MemoryMb: 0, - GPU: 8, + Architecture: "amd64", }, "p4d.24xlarge": { InstanceType: "p4d.24xlarge", VCPU: 96, MemoryMb: 1179648, GPU: 8, - }, - "p4de": { - InstanceType: "p4de", - VCPU: 96, - MemoryMb: 0, - GPU: 8, - }, - "p4de.24xlarge": { - InstanceType: "p4de.24xlarge", - VCPU: 96, - MemoryMb: 1179648, - GPU: 8, - }, - "r3": { - InstanceType: "r3", - VCPU: 32, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "r3.2xlarge": { InstanceType: "r3.2xlarge", VCPU: 8, MemoryMb: 62464, GPU: 0, + Architecture: "amd64", }, "r3.4xlarge": { InstanceType: "r3.4xlarge", VCPU: 16, MemoryMb: 124928, GPU: 0, + Architecture: "amd64", }, "r3.8xlarge": { InstanceType: "r3.8xlarge", VCPU: 32, MemoryMb: 249856, GPU: 0, + Architecture: "amd64", }, "r3.large": { InstanceType: "r3.large", VCPU: 2, - MemoryMb: 15616, + MemoryMb: 15360, GPU: 0, + Architecture: "amd64", }, "r3.xlarge": { InstanceType: "r3.xlarge", VCPU: 4, MemoryMb: 31232, GPU: 0, - }, - "r4": { - InstanceType: "r4", - VCPU: 64, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "r4.16xlarge": { InstanceType: "r4.16xlarge", VCPU: 64, MemoryMb: 499712, GPU: 0, + Architecture: "amd64", }, "r4.2xlarge": { InstanceType: "r4.2xlarge", VCPU: 8, MemoryMb: 62464, GPU: 0, + Architecture: "amd64", }, "r4.4xlarge": { InstanceType: "r4.4xlarge", VCPU: 16, MemoryMb: 124928, GPU: 0, + Architecture: "amd64", }, "r4.8xlarge": { InstanceType: "r4.8xlarge", VCPU: 32, MemoryMb: 249856, GPU: 0, + Architecture: "amd64", }, "r4.large": { InstanceType: "r4.large", VCPU: 2, MemoryMb: 15616, GPU: 0, + Architecture: "amd64", }, "r4.xlarge": { InstanceType: "r4.xlarge", VCPU: 4, MemoryMb: 31232, GPU: 0, - }, - "r5": { - InstanceType: "r5", - VCPU: 96, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "r5.12xlarge": { InstanceType: "r5.12xlarge", VCPU: 48, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "r5.16xlarge": { InstanceType: "r5.16xlarge", VCPU: 64, MemoryMb: 524288, GPU: 0, + Architecture: "amd64", }, "r5.24xlarge": { InstanceType: "r5.24xlarge", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "r5.2xlarge": { InstanceType: "r5.2xlarge", VCPU: 8, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "r5.4xlarge": { InstanceType: "r5.4xlarge", VCPU: 16, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "r5.8xlarge": { InstanceType: "r5.8xlarge", VCPU: 32, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "r5.large": { InstanceType: "r5.large", VCPU: 2, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "r5.metal": { InstanceType: "r5.metal", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "r5.xlarge": { InstanceType: "r5.xlarge", VCPU: 4, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "r5a.12xlarge": { InstanceType: "r5a.12xlarge", VCPU: 48, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "r5a.16xlarge": { InstanceType: "r5a.16xlarge", VCPU: 64, MemoryMb: 524288, GPU: 0, + Architecture: "amd64", }, "r5a.24xlarge": { InstanceType: "r5a.24xlarge", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "r5a.2xlarge": { InstanceType: "r5a.2xlarge", VCPU: 8, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "r5a.4xlarge": { InstanceType: "r5a.4xlarge", VCPU: 16, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "r5a.8xlarge": { InstanceType: "r5a.8xlarge", VCPU: 32, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "r5a.large": { InstanceType: "r5a.large", VCPU: 2, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "r5a.xlarge": { InstanceType: "r5a.xlarge", VCPU: 4, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "r5ad.12xlarge": { InstanceType: "r5ad.12xlarge", VCPU: 48, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "r5ad.16xlarge": { InstanceType: "r5ad.16xlarge", VCPU: 64, MemoryMb: 524288, GPU: 0, + Architecture: "amd64", }, "r5ad.24xlarge": { InstanceType: "r5ad.24xlarge", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "r5ad.2xlarge": { InstanceType: "r5ad.2xlarge", VCPU: 8, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "r5ad.4xlarge": { InstanceType: "r5ad.4xlarge", VCPU: 16, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "r5ad.8xlarge": { InstanceType: "r5ad.8xlarge", VCPU: 32, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "r5ad.large": { InstanceType: "r5ad.large", VCPU: 2, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "r5ad.xlarge": { InstanceType: "r5ad.xlarge", VCPU: 4, MemoryMb: 32768, GPU: 0, - }, - "r5b": { - InstanceType: "r5b", - VCPU: 96, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "r5b.12xlarge": { InstanceType: "r5b.12xlarge", VCPU: 48, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "r5b.16xlarge": { InstanceType: "r5b.16xlarge", VCPU: 64, MemoryMb: 524288, GPU: 0, + Architecture: "amd64", }, "r5b.24xlarge": { InstanceType: "r5b.24xlarge", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "r5b.2xlarge": { InstanceType: "r5b.2xlarge", VCPU: 8, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "r5b.4xlarge": { InstanceType: "r5b.4xlarge", VCPU: 16, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "r5b.8xlarge": { InstanceType: "r5b.8xlarge", VCPU: 32, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "r5b.large": { InstanceType: "r5b.large", VCPU: 2, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "r5b.metal": { InstanceType: "r5b.metal", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "r5b.xlarge": { InstanceType: "r5b.xlarge", VCPU: 4, MemoryMb: 32768, GPU: 0, - }, - "r5d": { - InstanceType: "r5d", - VCPU: 96, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "r5d.12xlarge": { InstanceType: "r5d.12xlarge", VCPU: 48, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "r5d.16xlarge": { InstanceType: "r5d.16xlarge", VCPU: 64, MemoryMb: 524288, GPU: 0, + Architecture: "amd64", }, "r5d.24xlarge": { InstanceType: "r5d.24xlarge", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "r5d.2xlarge": { InstanceType: "r5d.2xlarge", VCPU: 8, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "r5d.4xlarge": { InstanceType: "r5d.4xlarge", VCPU: 16, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "r5d.8xlarge": { InstanceType: "r5d.8xlarge", VCPU: 32, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "r5d.large": { InstanceType: "r5d.large", VCPU: 2, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "r5d.metal": { InstanceType: "r5d.metal", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "r5d.xlarge": { InstanceType: "r5d.xlarge", VCPU: 4, MemoryMb: 32768, GPU: 0, - }, - "r5dn": { - InstanceType: "r5dn", - VCPU: 96, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "r5dn.12xlarge": { InstanceType: "r5dn.12xlarge", VCPU: 48, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "r5dn.16xlarge": { InstanceType: "r5dn.16xlarge", VCPU: 64, MemoryMb: 524288, GPU: 0, + Architecture: "amd64", }, "r5dn.24xlarge": { InstanceType: "r5dn.24xlarge", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "r5dn.2xlarge": { InstanceType: "r5dn.2xlarge", VCPU: 8, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "r5dn.4xlarge": { InstanceType: "r5dn.4xlarge", VCPU: 16, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "r5dn.8xlarge": { InstanceType: "r5dn.8xlarge", VCPU: 32, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "r5dn.large": { InstanceType: "r5dn.large", VCPU: 2, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "r5dn.metal": { InstanceType: "r5dn.metal", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "r5dn.xlarge": { InstanceType: "r5dn.xlarge", VCPU: 4, MemoryMb: 32768, GPU: 0, - }, - "r5n": { - InstanceType: "r5n", - VCPU: 96, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "r5n.12xlarge": { InstanceType: "r5n.12xlarge", VCPU: 48, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "r5n.16xlarge": { InstanceType: "r5n.16xlarge", VCPU: 64, MemoryMb: 524288, GPU: 0, + Architecture: "amd64", }, "r5n.24xlarge": { InstanceType: "r5n.24xlarge", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "r5n.2xlarge": { InstanceType: "r5n.2xlarge", VCPU: 8, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "r5n.4xlarge": { InstanceType: "r5n.4xlarge", VCPU: 16, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "r5n.8xlarge": { InstanceType: "r5n.8xlarge", VCPU: 32, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "r5n.large": { InstanceType: "r5n.large", VCPU: 2, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "r5n.metal": { InstanceType: "r5n.metal", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "r5n.xlarge": { InstanceType: "r5n.xlarge", VCPU: 4, MemoryMb: 32768, GPU: 0, - }, - "r6g": { - InstanceType: "r6g", - VCPU: 64, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "r6g.12xlarge": { InstanceType: "r6g.12xlarge", VCPU: 48, MemoryMb: 393216, GPU: 0, + Architecture: "arm64", }, "r6g.16xlarge": { InstanceType: "r6g.16xlarge", VCPU: 64, MemoryMb: 524288, GPU: 0, + Architecture: "arm64", }, "r6g.2xlarge": { InstanceType: "r6g.2xlarge", VCPU: 8, MemoryMb: 65536, GPU: 0, + Architecture: "arm64", }, "r6g.4xlarge": { InstanceType: "r6g.4xlarge", VCPU: 16, MemoryMb: 131072, GPU: 0, + Architecture: "arm64", }, "r6g.8xlarge": { InstanceType: "r6g.8xlarge", VCPU: 32, MemoryMb: 262144, GPU: 0, + Architecture: "arm64", }, "r6g.large": { InstanceType: "r6g.large", VCPU: 2, MemoryMb: 16384, GPU: 0, + Architecture: "arm64", }, "r6g.medium": { InstanceType: "r6g.medium", VCPU: 1, MemoryMb: 8192, GPU: 0, + Architecture: "arm64", }, "r6g.metal": { InstanceType: "r6g.metal", VCPU: 64, MemoryMb: 524288, GPU: 0, + Architecture: "arm64", }, "r6g.xlarge": { InstanceType: "r6g.xlarge", VCPU: 4, MemoryMb: 32768, GPU: 0, - }, - "r6gd": { - InstanceType: "r6gd", - VCPU: 64, - MemoryMb: 0, - GPU: 0, + Architecture: "arm64", }, "r6gd.12xlarge": { InstanceType: "r6gd.12xlarge", VCPU: 48, MemoryMb: 393216, GPU: 0, + Architecture: "arm64", }, "r6gd.16xlarge": { InstanceType: "r6gd.16xlarge", VCPU: 64, MemoryMb: 524288, GPU: 0, + Architecture: "arm64", }, "r6gd.2xlarge": { InstanceType: "r6gd.2xlarge", VCPU: 8, MemoryMb: 65536, GPU: 0, + Architecture: "arm64", }, "r6gd.4xlarge": { InstanceType: "r6gd.4xlarge", VCPU: 16, MemoryMb: 131072, GPU: 0, + Architecture: "arm64", }, "r6gd.8xlarge": { InstanceType: "r6gd.8xlarge", VCPU: 32, MemoryMb: 262144, GPU: 0, + Architecture: "arm64", }, "r6gd.large": { InstanceType: "r6gd.large", VCPU: 2, MemoryMb: 16384, GPU: 0, + Architecture: "arm64", }, "r6gd.medium": { InstanceType: "r6gd.medium", VCPU: 1, MemoryMb: 8192, GPU: 0, + Architecture: "arm64", }, "r6gd.metal": { InstanceType: "r6gd.metal", VCPU: 64, MemoryMb: 524288, GPU: 0, + Architecture: "arm64", }, "r6gd.xlarge": { InstanceType: "r6gd.xlarge", VCPU: 4, MemoryMb: 32768, GPU: 0, - }, - "r6i": { - InstanceType: "r6i", - VCPU: 128, - MemoryMb: 0, - GPU: 0, + Architecture: "arm64", }, "r6i.12xlarge": { InstanceType: "r6i.12xlarge", VCPU: 48, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "r6i.16xlarge": { InstanceType: "r6i.16xlarge", VCPU: 64, MemoryMb: 524288, GPU: 0, + Architecture: "amd64", }, "r6i.24xlarge": { InstanceType: "r6i.24xlarge", VCPU: 96, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "r6i.2xlarge": { InstanceType: "r6i.2xlarge", VCPU: 8, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "r6i.32xlarge": { InstanceType: "r6i.32xlarge", VCPU: 128, MemoryMb: 1048576, GPU: 0, + Architecture: "amd64", }, "r6i.4xlarge": { InstanceType: "r6i.4xlarge", VCPU: 16, MemoryMb: 131072, GPU: 0, + Architecture: "amd64", }, "r6i.8xlarge": { InstanceType: "r6i.8xlarge", VCPU: 32, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "r6i.large": { InstanceType: "r6i.large", VCPU: 2, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "r6i.metal": { InstanceType: "r6i.metal", VCPU: 128, MemoryMb: 1048576, GPU: 0, + Architecture: "amd64", }, "r6i.xlarge": { InstanceType: "r6i.xlarge", VCPU: 4, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "t1.micro": { InstanceType: "t1.micro", VCPU: 1, MemoryMb: 627, GPU: 0, + Architecture: "amd64", }, "t2.2xlarge": { InstanceType: "t2.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "t2.large": { InstanceType: "t2.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "t2.medium": { InstanceType: "t2.medium", VCPU: 2, MemoryMb: 4096, GPU: 0, + Architecture: "amd64", }, "t2.micro": { InstanceType: "t2.micro", VCPU: 1, MemoryMb: 1024, GPU: 0, + Architecture: "amd64", }, "t2.nano": { InstanceType: "t2.nano", VCPU: 1, MemoryMb: 512, GPU: 0, + Architecture: "amd64", }, "t2.small": { InstanceType: "t2.small", VCPU: 1, MemoryMb: 2048, GPU: 0, + Architecture: "amd64", }, "t2.xlarge": { InstanceType: "t2.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, - }, - "t3": { - InstanceType: "t3", - VCPU: 8, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "t3.2xlarge": { InstanceType: "t3.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "t3.large": { InstanceType: "t3.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "t3.medium": { InstanceType: "t3.medium", VCPU: 2, MemoryMb: 4096, GPU: 0, + Architecture: "amd64", }, "t3.micro": { InstanceType: "t3.micro", VCPU: 2, MemoryMb: 1024, GPU: 0, + Architecture: "amd64", }, "t3.nano": { InstanceType: "t3.nano", VCPU: 2, MemoryMb: 512, GPU: 0, + Architecture: "amd64", }, "t3.small": { InstanceType: "t3.small", VCPU: 2, MemoryMb: 2048, GPU: 0, + Architecture: "amd64", }, "t3.xlarge": { InstanceType: "t3.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "t3a.2xlarge": { InstanceType: "t3a.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, "t3a.large": { InstanceType: "t3a.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "amd64", }, "t3a.medium": { InstanceType: "t3a.medium", VCPU: 2, MemoryMb: 4096, GPU: 0, + Architecture: "amd64", }, "t3a.micro": { InstanceType: "t3a.micro", VCPU: 2, MemoryMb: 1024, GPU: 0, + Architecture: "amd64", }, "t3a.nano": { InstanceType: "t3a.nano", VCPU: 2, MemoryMb: 512, GPU: 0, + Architecture: "amd64", }, "t3a.small": { InstanceType: "t3a.small", VCPU: 2, MemoryMb: 2048, GPU: 0, + Architecture: "amd64", }, "t3a.xlarge": { InstanceType: "t3a.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "t4g.2xlarge": { InstanceType: "t4g.2xlarge", VCPU: 8, MemoryMb: 32768, GPU: 0, + Architecture: "arm64", }, "t4g.large": { InstanceType: "t4g.large", VCPU: 2, MemoryMb: 8192, GPU: 0, + Architecture: "arm64", }, "t4g.medium": { InstanceType: "t4g.medium", VCPU: 2, MemoryMb: 4096, GPU: 0, + Architecture: "arm64", }, "t4g.micro": { InstanceType: "t4g.micro", VCPU: 2, MemoryMb: 1024, GPU: 0, + Architecture: "arm64", }, "t4g.nano": { InstanceType: "t4g.nano", VCPU: 2, MemoryMb: 512, GPU: 0, + Architecture: "arm64", }, "t4g.small": { InstanceType: "t4g.small", VCPU: 2, MemoryMb: 2048, GPU: 0, + Architecture: "arm64", }, "t4g.xlarge": { InstanceType: "t4g.xlarge", VCPU: 4, MemoryMb: 16384, GPU: 0, - }, - "u-12tb1": { - InstanceType: "u-12tb1", - VCPU: 448, - MemoryMb: 0, - GPU: 0, + Architecture: "arm64", }, "u-12tb1.112xlarge": { InstanceType: "u-12tb1.112xlarge", VCPU: 448, MemoryMb: 12582912, GPU: 0, - }, - "u-12tb1.metal": { - InstanceType: "u-12tb1.metal", - VCPU: 448, - MemoryMb: 12582912, - GPU: 0, - }, - "u-18tb1": { - InstanceType: "u-18tb1", - VCPU: 448, - MemoryMb: 0, - GPU: 0, - }, - "u-18tb1.metal": { - InstanceType: "u-18tb1.metal", - VCPU: 448, - MemoryMb: 18874368, - GPU: 0, - }, - "u-24tb1": { - InstanceType: "u-24tb1", - VCPU: 448, - MemoryMb: 0, - GPU: 0, - }, - "u-24tb1.metal": { - InstanceType: "u-24tb1.metal", - VCPU: 448, - MemoryMb: 25165824, - GPU: 0, + Architecture: "amd64", }, "u-3tb1.56xlarge": { InstanceType: "u-3tb1.56xlarge", VCPU: 224, MemoryMb: 3145728, GPU: 0, - }, - "u-6tb1": { - InstanceType: "u-6tb1", - VCPU: 448, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "u-6tb1.112xlarge": { InstanceType: "u-6tb1.112xlarge", VCPU: 448, MemoryMb: 6291456, GPU: 0, + Architecture: "amd64", }, "u-6tb1.56xlarge": { InstanceType: "u-6tb1.56xlarge", VCPU: 224, MemoryMb: 6291456, GPU: 0, - }, - "u-6tb1.metal": { - InstanceType: "u-6tb1.metal", - VCPU: 448, - MemoryMb: 6291456, - GPU: 0, - }, - "u-9tb1": { - InstanceType: "u-9tb1", - VCPU: 448, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "u-9tb1.112xlarge": { InstanceType: "u-9tb1.112xlarge", VCPU: 448, MemoryMb: 9437184, GPU: 0, - }, - "u-9tb1.metal": { - InstanceType: "u-9tb1.metal", - VCPU: 448, - MemoryMb: 9437184, - GPU: 0, - }, - "vt1": { - InstanceType: "vt1", - VCPU: 96, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "vt1.24xlarge": { InstanceType: "vt1.24xlarge", VCPU: 96, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "vt1.3xlarge": { InstanceType: "vt1.3xlarge", VCPU: 12, MemoryMb: 24576, GPU: 0, + Architecture: "amd64", }, "vt1.6xlarge": { InstanceType: "vt1.6xlarge", VCPU: 24, MemoryMb: 49152, GPU: 0, - }, - "x1": { - InstanceType: "x1", - VCPU: 128, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "x1.16xlarge": { InstanceType: "x1.16xlarge", VCPU: 64, MemoryMb: 999424, GPU: 0, + Architecture: "amd64", }, "x1.32xlarge": { InstanceType: "x1.32xlarge", VCPU: 128, MemoryMb: 1998848, GPU: 0, - }, - "x1e": { - InstanceType: "x1e", - VCPU: 128, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "x1e.16xlarge": { InstanceType: "x1e.16xlarge", VCPU: 64, MemoryMb: 1998848, GPU: 0, + Architecture: "amd64", }, "x1e.2xlarge": { InstanceType: "x1e.2xlarge", VCPU: 8, MemoryMb: 249856, GPU: 0, + Architecture: "amd64", }, "x1e.32xlarge": { InstanceType: "x1e.32xlarge", VCPU: 128, MemoryMb: 3997696, GPU: 0, + Architecture: "amd64", }, "x1e.4xlarge": { InstanceType: "x1e.4xlarge", VCPU: 16, MemoryMb: 499712, GPU: 0, + Architecture: "amd64", }, "x1e.8xlarge": { InstanceType: "x1e.8xlarge", VCPU: 32, MemoryMb: 999424, GPU: 0, + Architecture: "amd64", }, "x1e.xlarge": { InstanceType: "x1e.xlarge", VCPU: 4, MemoryMb: 124928, GPU: 0, - }, - "x2gd": { - InstanceType: "x2gd", - VCPU: 64, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "x2gd.12xlarge": { InstanceType: "x2gd.12xlarge", VCPU: 48, MemoryMb: 786432, GPU: 0, + Architecture: "arm64", }, "x2gd.16xlarge": { InstanceType: "x2gd.16xlarge", VCPU: 64, MemoryMb: 1048576, GPU: 0, + Architecture: "arm64", }, "x2gd.2xlarge": { InstanceType: "x2gd.2xlarge", VCPU: 8, MemoryMb: 131072, GPU: 0, + Architecture: "arm64", }, "x2gd.4xlarge": { InstanceType: "x2gd.4xlarge", VCPU: 16, MemoryMb: 262144, GPU: 0, + Architecture: "arm64", }, "x2gd.8xlarge": { InstanceType: "x2gd.8xlarge", VCPU: 32, MemoryMb: 524288, GPU: 0, + Architecture: "arm64", }, "x2gd.large": { InstanceType: "x2gd.large", VCPU: 2, MemoryMb: 32768, GPU: 0, + Architecture: "arm64", }, "x2gd.medium": { InstanceType: "x2gd.medium", VCPU: 1, MemoryMb: 16384, GPU: 0, + Architecture: "arm64", }, "x2gd.metal": { InstanceType: "x2gd.metal", VCPU: 64, MemoryMb: 1048576, GPU: 0, + Architecture: "arm64", }, "x2gd.xlarge": { InstanceType: "x2gd.xlarge", VCPU: 4, MemoryMb: 65536, GPU: 0, - }, - "x2idn": { - InstanceType: "x2idn", - VCPU: 128, - MemoryMb: 0, - GPU: 0, - }, - "x2idn.16xlarge": { - InstanceType: "x2idn.16xlarge", - VCPU: 64, - MemoryMb: 1048576, - GPU: 0, - }, - "x2idn.24xlarge": { - InstanceType: "x2idn.24xlarge", - VCPU: 96, - MemoryMb: 1572864, - GPU: 0, - }, - "x2idn.32xlarge": { - InstanceType: "x2idn.32xlarge", - VCPU: 128, - MemoryMb: 2097152, - GPU: 0, - }, - "x2idn.metal": { - InstanceType: "x2idn.metal", - VCPU: 128, - MemoryMb: 2097152, - GPU: 0, - }, - "x2iedn": { - InstanceType: "x2iedn", - VCPU: 128, - MemoryMb: 0, - GPU: 0, - }, - "x2iedn.16xlarge": { - InstanceType: "x2iedn.16xlarge", - VCPU: 64, - MemoryMb: 2097152, - GPU: 0, - }, - "x2iedn.24xlarge": { - InstanceType: "x2iedn.24xlarge", - VCPU: 96, - MemoryMb: 3145728, - GPU: 0, - }, - "x2iedn.2xlarge": { - InstanceType: "x2iedn.2xlarge", - VCPU: 8, - MemoryMb: 262144, - GPU: 0, - }, - "x2iedn.32xlarge": { - InstanceType: "x2iedn.32xlarge", - VCPU: 128, - MemoryMb: 4194304, - GPU: 0, - }, - "x2iedn.4xlarge": { - InstanceType: "x2iedn.4xlarge", - VCPU: 16, - MemoryMb: 524288, - GPU: 0, - }, - "x2iedn.8xlarge": { - InstanceType: "x2iedn.8xlarge", - VCPU: 32, - MemoryMb: 1048576, - GPU: 0, - }, - "x2iedn.metal": { - InstanceType: "x2iedn.metal", - VCPU: 128, - MemoryMb: 4194304, - GPU: 0, - }, - "x2iedn.xlarge": { - InstanceType: "x2iedn.xlarge", - VCPU: 4, - MemoryMb: 131072, - GPU: 0, - }, - "x2iezn": { - InstanceType: "x2iezn", - VCPU: 48, - MemoryMb: 0, - GPU: 0, + Architecture: "arm64", }, "x2iezn.12xlarge": { InstanceType: "x2iezn.12xlarge", VCPU: 48, MemoryMb: 1572864, GPU: 0, + Architecture: "amd64", }, "x2iezn.2xlarge": { InstanceType: "x2iezn.2xlarge", VCPU: 8, MemoryMb: 262144, GPU: 0, + Architecture: "amd64", }, "x2iezn.4xlarge": { InstanceType: "x2iezn.4xlarge", VCPU: 16, MemoryMb: 524288, GPU: 0, + Architecture: "amd64", }, "x2iezn.6xlarge": { InstanceType: "x2iezn.6xlarge", VCPU: 24, MemoryMb: 786432, GPU: 0, + Architecture: "amd64", }, "x2iezn.8xlarge": { InstanceType: "x2iezn.8xlarge", VCPU: 32, MemoryMb: 1048576, GPU: 0, + Architecture: "amd64", }, "x2iezn.metal": { InstanceType: "x2iezn.metal", VCPU: 48, MemoryMb: 1572864, GPU: 0, - }, - "z1d": { - InstanceType: "z1d", - VCPU: 48, - MemoryMb: 0, - GPU: 0, + Architecture: "amd64", }, "z1d.12xlarge": { InstanceType: "z1d.12xlarge", VCPU: 48, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "z1d.2xlarge": { InstanceType: "z1d.2xlarge", VCPU: 8, MemoryMb: 65536, GPU: 0, + Architecture: "amd64", }, "z1d.3xlarge": { InstanceType: "z1d.3xlarge", VCPU: 12, MemoryMb: 98304, GPU: 0, + Architecture: "amd64", }, "z1d.6xlarge": { InstanceType: "z1d.6xlarge", VCPU: 24, MemoryMb: 196608, GPU: 0, + Architecture: "amd64", }, "z1d.large": { InstanceType: "z1d.large", VCPU: 2, MemoryMb: 16384, GPU: 0, + Architecture: "amd64", }, "z1d.metal": { InstanceType: "z1d.metal", VCPU: 48, MemoryMb: 393216, GPU: 0, + Architecture: "amd64", }, "z1d.xlarge": { InstanceType: "z1d.xlarge", VCPU: 4, MemoryMb: 32768, GPU: 0, + Architecture: "amd64", }, } diff --git a/cluster-autoscaler/cloudprovider/aws/ec2_instance_types/gen.go b/cluster-autoscaler/cloudprovider/aws/ec2_instance_types/gen.go index 74fe1a887a14..2f6b094b2bbf 100644 --- a/cluster-autoscaler/cloudprovider/aws/ec2_instance_types/gen.go +++ b/cluster-autoscaler/cloudprovider/aws/ec2_instance_types/gen.go @@ -52,6 +52,7 @@ type InstanceType struct { VCPU int64 MemoryMb int64 GPU int64 + Architecture string } // InstanceTypes is a map of ec2 resources @@ -62,6 +63,7 @@ var InstanceTypes = map[string]*InstanceType{ VCPU: {{ .VCPU }}, MemoryMb: {{ .MemoryMb }}, GPU: {{ .GPU }}, + Architecture: "{{ .Architecture }}", }, {{- end }} } diff --git a/cluster-autoscaler/cloudprovider/aws/instance_type_cache.go b/cluster-autoscaler/cloudprovider/aws/instance_type_cache.go new file mode 100644 index 000000000000..f67b0cc3791f --- /dev/null +++ b/cluster-autoscaler/cloudprovider/aws/instance_type_cache.go @@ -0,0 +1,128 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "sync" + "time" + + "k8s.io/apimachinery/pkg/util/clock" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/tools/cache" +) + +const ( + asgInstanceTypeCacheTTL = time.Minute * 20 + cacheMinTTL = 120 + cacheMaxTTL = 600 +) + +// instanceTypeExpirationStore caches the canonical instance type for an ASG. +// The store expires its keys based on a TTL. This TTL can have a jitter applied to it. +// This allows to get a better repartition of the AWS queries. +type instanceTypeExpirationStore struct { + cache.Store + jitterClock clock.Clock + awsService *awsWrapper +} + +type instanceTypeCachedObject struct { + name string + instanceType string +} + +type jitterClock struct { + clock.Clock + + jitter bool + sync.RWMutex +} + +func newAsgInstanceTypeCache(awsService *awsWrapper) *instanceTypeExpirationStore { + jc := &jitterClock{} + return newAsgInstanceTypeCacheWithClock( + awsService, + jc, + cache.NewExpirationStore(func(obj interface{}) (s string, e error) { + return obj.(instanceTypeCachedObject).name, nil + }, &cache.TTLPolicy{ + TTL: asgInstanceTypeCacheTTL, + Clock: jc, + }), + ) +} + +func newAsgInstanceTypeCacheWithClock(awsService *awsWrapper, jc clock.Clock, store cache.Store) *instanceTypeExpirationStore { + return &instanceTypeExpirationStore{ + store, + jc, + awsService, + } +} + +func (c *jitterClock) Since(ts time.Time) time.Duration { + since := time.Since(ts) + c.RLock() + defer c.RUnlock() + if c.jitter { + return since + (time.Second * time.Duration(rand.IntnRange(cacheMinTTL, cacheMaxTTL))) + } + return since +} + +func (es instanceTypeExpirationStore) populate(autoscalingGroups map[AwsRef]*asg) error { + asgsToQuery := []*asg{} + + if c, ok := es.jitterClock.(*jitterClock); ok { + c.Lock() + c.jitter = true + c.Unlock() + } + + for _, asg := range autoscalingGroups { + if asg == nil { + continue + } + _, found, _ := es.GetByKey(asg.AwsRef.Name) + if found { + continue + } + asgsToQuery = append(asgsToQuery, asg) + } + + if c, ok := es.jitterClock.(*jitterClock); ok { + c.Lock() + c.jitter = false + c.Unlock() + } + + // List expires old entries + _ = es.List() + + instanceTypesByAsg, err := es.awsService.getInstanceTypesForAsgs(asgsToQuery) + if err != nil { + return err + } + + for asgName, instanceType := range instanceTypesByAsg { + es.Add(instanceTypeCachedObject{ + name: asgName, + instanceType: instanceType, + }) + } + return nil +} diff --git a/cluster-autoscaler/cloudprovider/aws/instance_type_cache_test.go b/cluster-autoscaler/cloudprovider/aws/instance_type_cache_test.go new file mode 100644 index 000000000000..7b9f56b77c9b --- /dev/null +++ b/cluster-autoscaler/cloudprovider/aws/instance_type_cache_test.go @@ -0,0 +1,104 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/clock" + "k8s.io/client-go/tools/cache" +) + +func TestInstanceTypeCache(t *testing.T) { + c := newAsgInstanceTypeCache(nil) + err := c.Add(instanceTypeCachedObject{ + name: "123", + instanceType: "t2.medium", + }) + require.NoError(t, err) + obj, ok, err := c.GetByKey("123") + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, "t2.medium", obj.(instanceTypeCachedObject).instanceType) +} + +func TestLTVersionChange(t *testing.T) { + asgName, ltName := "testasg", "launcher" + ltVersions := []*string{aws.String("1"), aws.String("2")} + instanceTypes := []*string{aws.String("t2.large"), aws.String("m4.xlarge")} + + a := &autoScalingMock{} + e := &ec2Mock{} + + for i := 0; i < 2; i++ { + e.On("DescribeLaunchTemplateVersions", &ec2.DescribeLaunchTemplateVersionsInput{ + LaunchTemplateName: aws.String(ltName), + Versions: []*string{ltVersions[i]}, + }).Return(&ec2.DescribeLaunchTemplateVersionsOutput{ + LaunchTemplateVersions: []*ec2.LaunchTemplateVersion{ + { + LaunchTemplateData: &ec2.ResponseLaunchTemplateData{ + InstanceType: instanceTypes[i], + }, + }, + }, + }) + } + + fakeClock := clock.NewFakeClock(time.Unix(0, 0)) + fakeStore := cache.NewFakeExpirationStore( + func(obj interface{}) (s string, e error) { + return obj.(instanceTypeCachedObject).name, nil + }, + nil, + &cache.TTLPolicy{ + TTL: asgInstanceTypeCacheTTL, + Clock: fakeClock, + }, + fakeClock, + ) + m := newAsgInstanceTypeCacheWithClock(&awsWrapper{a, e}, fakeClock, fakeStore) + + for i := 0; i < 2; i++ { + asgRef := AwsRef{Name: asgName} + err := m.populate(map[AwsRef]*asg{ + asgRef: { + AwsRef: asgRef, + LaunchTemplate: &launchTemplate{ + name: ltName, + version: aws.StringValue(ltVersions[i]), + }, + }, + }) + assert.NoError(t, err) + + result, found, err := m.GetByKey(asgName) + assert.NoError(t, err) + assert.Truef(t, found, "%s did not find asg (iteration %d)", asgName, i) + + foundInstanceType := result.(instanceTypeCachedObject).instanceType + assert.Equalf(t, foundInstanceType, *instanceTypes[i], "%s had %s, expected %s (iteration %d)", asgName, foundInstanceType, *instanceTypes[i], i) + + // Expire the first instance + fakeClock.SetTime(time.Now().Add(asgInstanceTypeCacheTTL + 10*time.Minute)) + } +} diff --git a/cluster-autoscaler/cloudprovider/azure/azure_template.go b/cluster-autoscaler/cloudprovider/azure/azure_template.go index 81c7e9ae0348..2995f53622ba 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_template.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_template.go @@ -26,7 +26,6 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" cloudvolume "k8s.io/cloud-provider/volume" "k8s.io/klog/v2" - kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "math/rand" "regexp" "strings" @@ -44,14 +43,11 @@ func buildInstanceOS(template compute.VirtualMachineScaleSet) string { func buildGenericLabels(template compute.VirtualMachineScaleSet, nodeName string) map[string]string { result := make(map[string]string) - result[kubeletapis.LabelArch] = cloudprovider.DefaultArch result[apiv1.LabelArchStable] = cloudprovider.DefaultArch - - result[kubeletapis.LabelOS] = buildInstanceOS(template) result[apiv1.LabelOSStable] = buildInstanceOS(template) - result[apiv1.LabelInstanceType] = *template.Sku.Name - result[apiv1.LabelZoneRegion] = strings.ToLower(*template.Location) + result[apiv1.LabelInstanceTypeStable] = *template.Sku.Name + result[apiv1.LabelTopologyRegion] = strings.ToLower(*template.Location) if template.Zones != nil && len(*template.Zones) > 0 { failureDomains := make([]string, len(*template.Zones)) @@ -59,9 +55,9 @@ func buildGenericLabels(template compute.VirtualMachineScaleSet, nodeName string failureDomains[k] = strings.ToLower(*template.Location) + "-" + v } - result[apiv1.LabelZoneFailureDomain] = strings.Join(failureDomains[:], cloudvolume.LabelMultiZoneDelimiter) + result[apiv1.LabelTopologyZone] = strings.Join(failureDomains[:], cloudvolume.LabelMultiZoneDelimiter) } else { - result[apiv1.LabelZoneFailureDomain] = "0" + result[apiv1.LabelTopologyZone] = "0" } result[apiv1.LabelHostname] = nodeName diff --git a/cluster-autoscaler/cloudprovider/gce/gce_price_model.go b/cluster-autoscaler/cloudprovider/gce/gce_price_model.go index 4262061a9749..06f379e20dd4 100644 --- a/cluster-autoscaler/cloudprovider/gce/gce_price_model.go +++ b/cluster-autoscaler/cloudprovider/gce/gce_price_model.go @@ -331,7 +331,7 @@ func (model *GcePriceModel) NodePrice(node *apiv1.Node, startTime time.Time, end // Base instance price if node.Labels != nil { isPreemptible = node.Labels[preemptibleLabel] == "true" - if machineType, found := node.Labels[apiv1.LabelInstanceType]; found { + if machineType, found := getInstanceTypeFromLabels(node.Labels); found { priceMapToUse := instancePrices if isPreemptible { priceMapToUse = preemptiblePrices @@ -345,8 +345,10 @@ func (model *GcePriceModel) NodePrice(node *apiv1.Node, startTime time.Time, end } } if !basePriceFound { - price = getBasePrice(node.Status.Capacity, node.Labels[apiv1.LabelInstanceType], startTime, endTime) - price = price * getPreemptibleDiscount(node) + if machineType, found := getInstanceTypeFromLabels(node.Labels); found { + price = getBasePrice(node.Status.Capacity, machineType, startTime, endTime) + price = price * getPreemptibleDiscount(node) + } } // GPUs @@ -390,7 +392,10 @@ func getPreemptibleDiscount(node *apiv1.Node) float64 { if node.Labels[preemptibleLabel] != "true" { return 1.0 } - instanceType := node.Labels[apiv1.LabelInstanceType] + instanceType, found := getInstanceTypeFromLabels(node.Labels) + if !found { + return 1.0 + } instanceFamily := getInstanceFamily(instanceType) discountMap := predefinedPreemptibleDiscount @@ -459,3 +464,11 @@ func getAdditionalPrice(resources apiv1.ResourceList, startTime time.Time, endTi price += float64(gpu.MilliValue()) / 1000.0 * gpuPricePerHour * hours return price } + +func getInstanceTypeFromLabels(labels map[string]string) (string, bool) { + machineType, found := labels[apiv1.LabelInstanceTypeStable] + if !found { + machineType, found = labels[apiv1.LabelInstanceType] + } + return machineType, found +} diff --git a/cluster-autoscaler/cloudprovider/gce/templates.go b/cluster-autoscaler/cloudprovider/gce/templates.go index 38c131e71476..d3a37edce482 100644 --- a/cluster-autoscaler/cloudprovider/gce/templates.go +++ b/cluster-autoscaler/cloudprovider/gce/templates.go @@ -22,15 +22,13 @@ import ( "regexp" "strings" + "github.com/ghodss/yaml" gce "google.golang.org/api/compute/v1" apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" - kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" - - "github.com/ghodss/yaml" klog "k8s.io/klog/v2" ) @@ -203,21 +201,16 @@ func BuildGenericLabels(ref GceRef, machineType string, nodeName string, os Oper } // TODO: extract it somehow - result[kubeletapis.LabelArch] = cloudprovider.DefaultArch result[apiv1.LabelArchStable] = cloudprovider.DefaultArch - result[kubeletapis.LabelOS] = string(os) result[apiv1.LabelOSStable] = string(os) - result[apiv1.LabelInstanceType] = machineType result[apiv1.LabelInstanceTypeStable] = machineType ix := strings.LastIndex(ref.Zone, "-") if ix == -1 { return nil, fmt.Errorf("unexpected zone: %s", ref.Zone) } - result[apiv1.LabelZoneRegion] = ref.Zone[:ix] - result[apiv1.LabelZoneRegionStable] = ref.Zone[:ix] - result[apiv1.LabelZoneFailureDomain] = ref.Zone - result[apiv1.LabelZoneFailureDomainStable] = ref.Zone + result[apiv1.LabelTopologyRegion] = ref.Zone[:ix] + result[apiv1.LabelTopologyZone] = ref.Zone result[gceCSITopologyKeyZone] = ref.Zone result[apiv1.LabelHostname] = nodeName return result, nil diff --git a/cluster-autoscaler/cloudprovider/gce/templates_test.go b/cluster-autoscaler/cloudprovider/gce/templates_test.go index a9314ebae9a6..57ccd16a0bb7 100644 --- a/cluster-autoscaler/cloudprovider/gce/templates_test.go +++ b/cluster-autoscaler/cloudprovider/gce/templates_test.go @@ -25,13 +25,11 @@ import ( gpuUtils "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" "k8s.io/autoscaler/cluster-autoscaler/utils/units" + "github.com/stretchr/testify/assert" gce "google.golang.org/api/compute/v1" apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" quota "k8s.io/apiserver/pkg/quota/v1" - kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" - - "github.com/stretchr/testify/assert" ) func TestBuildNodeFromTemplateSetsResources(t *testing.T) { @@ -166,18 +164,13 @@ func TestBuildGenericLabels(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { expectedLabels := map[string]string{ - apiv1.LabelZoneRegion: "us-central1", - apiv1.LabelZoneRegionStable: "us-central1", - apiv1.LabelZoneFailureDomain: "us-central1-b", - apiv1.LabelZoneFailureDomainStable: "us-central1-b", - gceCSITopologyKeyZone: "us-central1-b", - apiv1.LabelHostname: "sillyname", - apiv1.LabelInstanceType: "n1-standard-8", - apiv1.LabelInstanceTypeStable: "n1-standard-8", - kubeletapis.LabelArch: cloudprovider.DefaultArch, - kubeletapis.LabelOS: tc.expectedOsLabel, - apiv1.LabelArchStable: cloudprovider.DefaultArch, - apiv1.LabelOSStable: tc.expectedOsLabel, + apiv1.LabelTopologyRegion: "us-central1", + apiv1.LabelTopologyZone: "us-central1-b", + gceCSITopologyKeyZone: "us-central1-b", + apiv1.LabelHostname: "sillyname", + apiv1.LabelInstanceTypeStable: "n1-standard-8", + apiv1.LabelArchStable: cloudprovider.DefaultArch, + apiv1.LabelOSStable: tc.expectedOsLabel, } labels, err := BuildGenericLabels(GceRef{ Name: "kubernetes-minion-group", diff --git a/cluster-autoscaler/cloudprovider/huaweicloud/huaweicloud_service_manager.go b/cluster-autoscaler/cloudprovider/huaweicloud/huaweicloud_service_manager.go index 533e373db8db..23d1741e96a3 100644 --- a/cluster-autoscaler/cloudprovider/huaweicloud/huaweicloud_service_manager.go +++ b/cluster-autoscaler/cloudprovider/huaweicloud/huaweicloud_service_manager.go @@ -35,7 +35,6 @@ import ( huaweicloudsdkecsmodel "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/huaweicloud/huaweicloud-sdk-go-v3/services/ecs/v2/model" "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" "k8s.io/klog/v2" - kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" ) // ElasticCloudServerService represents the elastic cloud server interfaces. @@ -569,13 +568,13 @@ func (csm *cloudServiceManager) buildNodeFromTemplate(asgName string, template * func buildGenericLabels(template *asgTemplate, nodeName string) map[string]string { result := make(map[string]string) - result[kubeletapis.LabelArch] = cloudprovider.DefaultArch - result[kubeletapis.LabelOS] = cloudprovider.DefaultOS + result[apiv1.LabelArchStable] = cloudprovider.DefaultArch + result[apiv1.LabelOSStable] = cloudprovider.DefaultOS - result[apiv1.LabelInstanceType] = template.name + result[apiv1.LabelInstanceTypeStable] = template.name - result[apiv1.LabelZoneRegion] = template.region - result[apiv1.LabelZoneFailureDomain] = template.zone + result[apiv1.LabelTopologyRegion] = template.region + result[apiv1.LabelTopologyZone] = template.zone result[apiv1.LabelHostname] = nodeName // append custom node labels diff --git a/cluster-autoscaler/config/autoscaling_options.go b/cluster-autoscaler/config/autoscaling_options.go index f3cce4044929..82d28e557a49 100644 --- a/cluster-autoscaler/config/autoscaling_options.go +++ b/cluster-autoscaler/config/autoscaling_options.go @@ -61,8 +61,12 @@ type AutoscalingOptions struct { NodeGroupAutoDiscovery []string // EstimatorName is the estimator used to estimate the number of needed nodes in scale up. EstimatorName string - // ExpanderName sets the type of node group expander to be used in scale up - ExpanderName string + // ExpanderNames sets the chain of node group expanders to be used in scale up + ExpanderNames string + // GRPCExpanderCert is the location of the cert passed to the gRPC server for TLS when using the gRPC expander + GRPCExpanderCert string + // GRPCExpanderURL is the url of the gRPC server when using the gRPC expander + GRPCExpanderURL string // IgnoreDaemonSetsUtilization is whether CA will ignore DaemonSet pods when calculating resource utilization for scaling down IgnoreDaemonSetsUtilization bool // IgnoreMirrorPodsUtilization is whether CA will ignore Mirror pods when calculating resource utilization for scaling down diff --git a/cluster-autoscaler/core/autoscaler.go b/cluster-autoscaler/core/autoscaler.go index 3735baaac083..62216018769d 100644 --- a/cluster-autoscaler/core/autoscaler.go +++ b/cluster-autoscaler/core/autoscaler.go @@ -17,6 +17,7 @@ limitations under the License. package core import ( + "strings" "time" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" @@ -101,8 +102,8 @@ func initializeDefaultOptions(opts *AutoscalerOptions) error { opts.CloudProvider = cloudBuilder.NewCloudProvider(opts.AutoscalingOptions) } if opts.ExpanderStrategy == nil { - expanderStrategy, err := factory.ExpanderStrategyFromString(opts.ExpanderName, - opts.CloudProvider, opts.AutoscalingKubeClients, opts.KubeClient, opts.ConfigNamespace) + expanderStrategy, err := factory.ExpanderStrategyFromStrings(strings.Split(opts.ExpanderNames, ","), opts.CloudProvider, + opts.AutoscalingKubeClients, opts.KubeClient, opts.ConfigNamespace, opts.GRPCExpanderCert, opts.GRPCExpanderURL) if err != nil { return err } diff --git a/cluster-autoscaler/core/static_autoscaler.go b/cluster-autoscaler/core/static_autoscaler.go index acdfd34b2be0..731ceb97d980 100644 --- a/cluster-autoscaler/core/static_autoscaler.go +++ b/cluster-autoscaler/core/static_autoscaler.go @@ -23,7 +23,6 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/uuid" schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" @@ -42,6 +41,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/utils/deletetaint" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" + scheduler_utils "k8s.io/autoscaler/cluster-autoscaler/utils/scheduler" "k8s.io/autoscaler/cluster-autoscaler/utils/taints" "k8s.io/autoscaler/cluster-autoscaler/utils/tpu" @@ -775,21 +775,6 @@ func allPodsAreNew(pods []*apiv1.Pod, currentTime time.Time) bool { return found && oldest.Add(unschedulablePodWithGpuTimeBuffer).After(currentTime) } -func deepCopyNodeInfo(nodeTemplate *schedulerframework.NodeInfo, index int) *schedulerframework.NodeInfo { - node := nodeTemplate.Node().DeepCopy() - node.Name = fmt.Sprintf("%s-%d", node.Name, index) - node.UID = uuid.NewUUID() - nodeInfo := schedulerframework.NewNodeInfo() - nodeInfo.SetNode(node) - for _, podInfo := range nodeTemplate.Pods { - pod := podInfo.Pod.DeepCopy() - pod.Name = fmt.Sprintf("%s-%d", podInfo.Pod.Name, index) - pod.UID = uuid.NewUUID() - nodeInfo.AddPod(pod) - } - return nodeInfo -} - func getUpcomingNodeInfos(registry *clusterstate.ClusterStateRegistry, nodeInfos map[string]*schedulerframework.NodeInfo) []*schedulerframework.NodeInfo { upcomingNodes := make([]*schedulerframework.NodeInfo, 0) for nodeGroup, numberOfNodes := range registry.GetUpcomingNodes() { @@ -808,7 +793,7 @@ func getUpcomingNodeInfos(registry *clusterstate.ClusterStateRegistry, nodeInfos // Ensure new nodes have different names because nodeName // will be used as a map key. Also deep copy pods (daemonsets & // any pods added by cloud provider on template). - upcomingNodes = append(upcomingNodes, deepCopyNodeInfo(nodeTemplate, i)) + upcomingNodes = append(upcomingNodes, scheduler_utils.DeepCopyTemplateNode(nodeTemplate, i)) } } return upcomingNodes diff --git a/cluster-autoscaler/core/utils/utils.go b/cluster-autoscaler/core/utils/utils.go index c60b050c2c4d..849c0bfe6fbb 100644 --- a/cluster-autoscaler/core/utils/utils.go +++ b/cluster-autoscaler/core/utils/utils.go @@ -18,6 +18,7 @@ package utils import ( "fmt" + kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" "math/rand" "reflect" "time" @@ -172,6 +173,8 @@ func GetNodeInfoFromTemplate(nodeGroup cloudprovider.NodeGroup, daemonsets []*ap return nil, errors.ToAutoscalerError(errors.CloudProviderError, err) } + updateDeprecatedTemplateLabels(baseNodeInfo) + pods, err := daemonset.GetDaemonSetPodsForNode(baseNodeInfo, daemonsets, predicateChecker) if err != nil { return nil, errors.ToAutoscalerError(errors.InternalError, err) @@ -188,6 +191,26 @@ func GetNodeInfoFromTemplate(nodeGroup cloudprovider.NodeGroup, daemonsets []*ap return sanitizedNodeInfo, nil } +// UpdateDeprecatedTemplateLabels updates beta and deprecated labels from stable labels +func updateDeprecatedTemplateLabels(nodeInfo *schedulerframework.NodeInfo) { + node := nodeInfo.Node() + if v, ok := node.ObjectMeta.Labels[apiv1.LabelArchStable]; ok { + node.ObjectMeta.Labels[kubeletapis.LabelArch] = v + } + if v, ok := node.ObjectMeta.Labels[apiv1.LabelOSStable]; ok { + node.ObjectMeta.Labels[kubeletapis.LabelOS] = v + } + if v, ok := node.ObjectMeta.Labels[apiv1.LabelInstanceTypeStable]; ok { + node.ObjectMeta.Labels[apiv1.LabelInstanceType] = v + } + if v, ok := node.ObjectMeta.Labels[apiv1.LabelTopologyRegion]; ok { + node.ObjectMeta.Labels[apiv1.LabelZoneRegion] = v + } + if v, ok := node.ObjectMeta.Labels[apiv1.LabelTopologyZone]; ok { + node.ObjectMeta.Labels[apiv1.LabelZoneFailureDomain] = v + } +} + // isVirtualNode determines if the node is created by virtual kubelet func isVirtualNode(node *apiv1.Node) bool { return node.ObjectMeta.Labels["type"] == "virtual-kubelet" diff --git a/cluster-autoscaler/estimator/binpacking_estimator.go b/cluster-autoscaler/estimator/binpacking_estimator.go index 6cbbf389aaf4..b998143cce70 100644 --- a/cluster-autoscaler/estimator/binpacking_estimator.go +++ b/cluster-autoscaler/estimator/binpacking_estimator.go @@ -17,13 +17,12 @@ limitations under the License. package estimator import ( - "fmt" "sort" - "time" apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/autoscaler/cluster-autoscaler/simulator" + "k8s.io/autoscaler/cluster-autoscaler/utils/scheduler" klog "k8s.io/klog/v2" schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" ) @@ -75,7 +74,6 @@ func (estimator *BinpackingNodeEstimator) Estimate( } }() - newNodeNameTimestamp := time.Now() newNodeNameIndex := 0 for _, podInfo := range podInfos { @@ -94,7 +92,7 @@ func (estimator *BinpackingNodeEstimator) Estimate( if !found { // Add new node - newNodeName, err := estimator.addNewNodeToSnapshot(nodeTemplate, newNodeNameTimestamp, newNodeNameIndex) + newNodeName, err := estimator.addNewNodeToSnapshot(nodeTemplate, newNodeNameIndex) if err != nil { klog.Errorf("Error while adding new node for template to ClusterSnapshot; %v", err) return 0 @@ -113,23 +111,17 @@ func (estimator *BinpackingNodeEstimator) Estimate( func (estimator *BinpackingNodeEstimator) addNewNodeToSnapshot( template *schedulerframework.NodeInfo, - nameTimestamp time.Time, nameIndex int) (string, error) { - newNode := template.Node().DeepCopy() - newNode.Name = fmt.Sprintf("%s-%d-%d", newNode.Name, nameTimestamp.Unix(), nameIndex) - if newNode.Labels == nil { - newNode.Labels = make(map[string]string) - } - newNode.Labels["kubernetes.io/hostname"] = newNode.Name + newNodeInfo := scheduler.DeepCopyTemplateNode(template, nameIndex) var pods []*apiv1.Pod - for _, podInfo := range template.Pods { + for _, podInfo := range newNodeInfo.Pods { pods = append(pods, podInfo.Pod) } - if err := estimator.clusterSnapshot.AddNodeWithPods(newNode, pods); err != nil { + if err := estimator.clusterSnapshot.AddNodeWithPods(newNodeInfo.Node(), pods); err != nil { return "", err } - return newNode.Name, nil + return newNodeInfo.Node().Name, nil } // Calculates score for all pods and returns podInfo structure. diff --git a/cluster-autoscaler/expander/expander.go b/cluster-autoscaler/expander/expander.go index e38396239494..57a91cfa78e7 100644 --- a/cluster-autoscaler/expander/expander.go +++ b/cluster-autoscaler/expander/expander.go @@ -24,7 +24,7 @@ import ( var ( // AvailableExpanders is a list of available expander options - AvailableExpanders = []string{RandomExpanderName, MostPodsExpanderName, LeastWasteExpanderName, PriceBasedExpanderName, PriorityBasedExpanderName} + AvailableExpanders = []string{RandomExpanderName, MostPodsExpanderName, LeastWasteExpanderName, PriceBasedExpanderName, PriorityBasedExpanderName, GRPCExpanderName} // RandomExpanderName selects a node group at random RandomExpanderName = "random" // MostPodsExpanderName selects a node group that fits the most pods @@ -36,6 +36,8 @@ var ( PriceBasedExpanderName = "price" // PriorityBasedExpanderName selects a node group based on a user-configured priorities assigned to group names PriorityBasedExpanderName = "priority" + // GRPCExpanderName uses the gRPC client expander to call to an external gRPC server to select a node group for scale up + GRPCExpanderName = "grpc" ) // Option describes an option to expand the cluster. @@ -50,3 +52,8 @@ type Option struct { type Strategy interface { BestOption(options []Option, nodeInfo map[string]*schedulerframework.NodeInfo) *Option } + +// Filter describes an interface for filtering to equally good options according to some criteria +type Filter interface { + BestOptions(options []Option, nodeInfo map[string]*schedulerframework.NodeInfo) []Option +} diff --git a/cluster-autoscaler/expander/factory/chain.go b/cluster-autoscaler/expander/factory/chain.go new file mode 100644 index 000000000000..eec2ec91a311 --- /dev/null +++ b/cluster-autoscaler/expander/factory/chain.go @@ -0,0 +1,46 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package factory + +import ( + "k8s.io/autoscaler/cluster-autoscaler/expander" + + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" +) + +type chainStrategy struct { + filters []expander.Filter + fallback expander.Strategy +} + +func newChainStrategy(filters []expander.Filter, fallback expander.Strategy) expander.Strategy { + return &chainStrategy{ + filters: filters, + fallback: fallback, + } +} + +func (c *chainStrategy) BestOption(options []expander.Option, nodeInfo map[string]*schedulerframework.NodeInfo) *expander.Option { + filteredOptions := options + for _, filter := range c.filters { + filteredOptions = filter.BestOptions(filteredOptions, nodeInfo) + if len(filteredOptions) == 1 { + return &filteredOptions[0] + } + } + return c.fallback.BestOption(filteredOptions, nodeInfo) +} diff --git a/cluster-autoscaler/expander/factory/chain_test.go b/cluster-autoscaler/expander/factory/chain_test.go new file mode 100644 index 000000000000..b6039269752a --- /dev/null +++ b/cluster-autoscaler/expander/factory/chain_test.go @@ -0,0 +1,133 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package factory + +import ( + "k8s.io/autoscaler/cluster-autoscaler/expander" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" +) + +type substringTestFilterStrategy struct { + substring string +} + +func newSubstringTestFilterStrategy(substring string) *substringTestFilterStrategy { + return &substringTestFilterStrategy{ + substring: substring, + } +} + +func (s *substringTestFilterStrategy) BestOptions(expansionOptions []expander.Option, nodeInfo map[string]*schedulerframework.NodeInfo) []expander.Option { + var ret []expander.Option + for _, option := range expansionOptions { + if strings.Contains(option.Debug, s.substring) { + ret = append(ret, option) + } + } + return ret + +} + +func (s *substringTestFilterStrategy) BestOption(expansionOptions []expander.Option, nodeInfo map[string]*schedulerframework.NodeInfo) *expander.Option { + ret := s.BestOptions(expansionOptions, nodeInfo) + if len(ret) == 0 { + return nil + } + return &ret[0] +} + +func TestChainStrategy_BestOption(t *testing.T) { + for name, tc := range map[string]struct { + filters []expander.Filter + fallback expander.Strategy + options []expander.Option + expected *expander.Option + }{ + "selects with no filters": { + filters: []expander.Filter{}, + fallback: newSubstringTestFilterStrategy("a"), + options: []expander.Option{ + *newOption("b"), + *newOption("a"), + }, + expected: newOption("a"), + }, + "filters with one filter": { + filters: []expander.Filter{ + newSubstringTestFilterStrategy("a"), + }, + fallback: newSubstringTestFilterStrategy("b"), + options: []expander.Option{ + *newOption("ab"), + *newOption("b"), + }, + expected: newOption("ab"), + }, + "filters with multiple filters": { + filters: []expander.Filter{ + newSubstringTestFilterStrategy("a"), + newSubstringTestFilterStrategy("b"), + }, + fallback: newSubstringTestFilterStrategy("x"), + options: []expander.Option{ + *newOption("xab"), + *newOption("xa"), + *newOption("x"), + }, + expected: newOption("xab"), + }, + "selects from multiple after filters": { + filters: []expander.Filter{ + newSubstringTestFilterStrategy("x"), + }, + fallback: newSubstringTestFilterStrategy("a"), + options: []expander.Option{ + *newOption("xc"), + *newOption("xaa"), + *newOption("xab"), + }, + expected: newOption("xaa"), + }, + "short circuits": { + filters: []expander.Filter{ + newSubstringTestFilterStrategy("a"), + newSubstringTestFilterStrategy("b"), + }, + fallback: newSubstringTestFilterStrategy("x"), + options: []expander.Option{ + *newOption("a"), + }, + expected: newOption("a"), + }, + } { + t.Run(name, func(t *testing.T) { + subject := newChainStrategy(tc.filters, tc.fallback) + actual := subject.BestOption(tc.options, nil) + assert.Equal(t, tc.expected, actual) + }) + } +} + +func newOption(debug string) *expander.Option { + return &expander.Option{ + Debug: debug, + } +} diff --git a/cluster-autoscaler/expander/factory/expander_factory.go b/cluster-autoscaler/expander/factory/expander_factory.go index 9c26520629db..a0e0b7fe0d5a 100644 --- a/cluster-autoscaler/expander/factory/expander_factory.go +++ b/cluster-autoscaler/expander/factory/expander_factory.go @@ -20,6 +20,7 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/autoscaler/cluster-autoscaler/context" "k8s.io/autoscaler/cluster-autoscaler/expander" + "k8s.io/autoscaler/cluster-autoscaler/expander/grpcplugin" "k8s.io/autoscaler/cluster-autoscaler/expander/mostpods" "k8s.io/autoscaler/cluster-autoscaler/expander/price" "k8s.io/autoscaler/cluster-autoscaler/expander/priority" @@ -27,34 +28,54 @@ import ( "k8s.io/autoscaler/cluster-autoscaler/expander/waste" "k8s.io/autoscaler/cluster-autoscaler/utils/errors" "k8s.io/autoscaler/cluster-autoscaler/utils/kubernetes" - kube_client "k8s.io/client-go/kubernetes" ) -// ExpanderStrategyFromString creates an expander.Strategy according to its name -func ExpanderStrategyFromString(expanderFlag string, cloudProvider cloudprovider.CloudProvider, +// ExpanderStrategyFromStrings creates an expander.Strategy according to the names of the expanders passed in +// take in whole opts and access stuff here +func ExpanderStrategyFromStrings(expanderFlags []string, cloudProvider cloudprovider.CloudProvider, autoscalingKubeClients *context.AutoscalingKubeClients, kubeClient kube_client.Interface, - configNamespace string) (expander.Strategy, errors.AutoscalerError) { - switch expanderFlag { - case expander.RandomExpanderName: - return random.NewStrategy(), nil - case expander.MostPodsExpanderName: - return mostpods.NewStrategy(), nil - case expander.LeastWasteExpanderName: - return waste.NewStrategy(), nil - case expander.PriceBasedExpanderName: - if _, err := cloudProvider.Pricing(); err != nil { - return nil, err + configNamespace string, GRPCExpanderCert string, GRPCExpanderURL string) (expander.Strategy, errors.AutoscalerError) { + var filters []expander.Filter + seenExpanders := map[string]struct{}{} + strategySeen := false + for i, expanderFlag := range expanderFlags { + if _, ok := seenExpanders[expanderFlag]; ok { + return nil, errors.NewAutoscalerError(errors.InternalError, "Expander %s was specified multiple times, each expander must not be specified more than once", expanderFlag) + } + if strategySeen { + return nil, errors.NewAutoscalerError(errors.InternalError, "Expander %s came after an expander %s that will always return only one result, this is not allowed since %s will never be used", expanderFlag, expanderFlags[i-1], expanderFlag) + } + seenExpanders[expanderFlag] = struct{}{} + + switch expanderFlag { + case expander.RandomExpanderName: + filters = append(filters, random.NewFilter()) + case expander.MostPodsExpanderName: + filters = append(filters, mostpods.NewFilter()) + case expander.LeastWasteExpanderName: + filters = append(filters, waste.NewFilter()) + case expander.PriceBasedExpanderName: + if _, err := cloudProvider.Pricing(); err != nil { + return nil, err + } + filters = append(filters, price.NewFilter(cloudProvider, + price.NewSimplePreferredNodeProvider(autoscalingKubeClients.AllNodeLister()), + price.SimpleNodeUnfitness)) + case expander.PriorityBasedExpanderName: + // It seems other listers do the same here - they never receive the termination msg on the ch. + // This should be currently OK. + stopChannel := make(chan struct{}) + lister := kubernetes.NewConfigMapListerForNamespace(kubeClient, stopChannel, configNamespace) + filters = append(filters, priority.NewFilter(lister.ConfigMaps(configNamespace), autoscalingKubeClients.Recorder)) + case expander.GRPCExpanderName: + filters = append(filters, grpcplugin.NewFilter(GRPCExpanderCert, GRPCExpanderURL)) + default: + return nil, errors.NewAutoscalerError(errors.InternalError, "Expander %s not supported", expanderFlag) + } + if _, ok := filters[len(filters)-1].(expander.Strategy); ok { + strategySeen = true } - return price.NewStrategy(cloudProvider, - price.NewSimplePreferredNodeProvider(autoscalingKubeClients.AllNodeLister()), - price.SimpleNodeUnfitness), nil - case expander.PriorityBasedExpanderName: - // It seems other listers do the same here - they never receive the termination msg on the ch. - // This should be currently OK. - stopChannel := make(chan struct{}) - lister := kubernetes.NewConfigMapListerForNamespace(kubeClient, stopChannel, configNamespace) - return priority.NewStrategy(lister.ConfigMaps(configNamespace), autoscalingKubeClients.Recorder) } - return nil, errors.NewAutoscalerError(errors.InternalError, "Expander %s not supported", expanderFlag) + return newChainStrategy(filters, random.NewStrategy()), nil } diff --git a/cluster-autoscaler/expander/grpcplugin/README.md b/cluster-autoscaler/expander/grpcplugin/README.md new file mode 100644 index 000000000000..4fd24408fd21 --- /dev/null +++ b/cluster-autoscaler/expander/grpcplugin/README.md @@ -0,0 +1,41 @@ +# gRPC Expander for Cluster Autoscaler + +## Introduction +This expander functions as a gRPC client, and will pass expansion options to an external gRPC server. +The external server will use this information to make a decision on which Node Group to expand, and return an option to expand. + +## Motivation + +This expander gives users very fine grained control over which option they'd like to expand. +The gRPC server must be implemented by the user, but the logic can be developed out of band with Cluster Autoscaler. +There are a wide variety of use cases here. Some examples are as follows: +* A tiered weighted random strategy can be implemented, instead of a static priority ladder offered by the priority expander. +* A strategy to encapsulate business logic specific to a user but not all users of Cluster Autoscaler +* A strategy to take into account the dynamic fluctuating prices of the spot instance market + +## Configuration options +As using this expander requires communication with another service, users must specify a few options as CLI arguments. + +```yaml +--grpcExpanderUrl +``` +URL of the gRPC Expander server, for CA to communicate with. +```yaml +--grpcExpanderCert +``` +Location of the volume mounted certificate of the gRPC server if it is configured to communicate over TLS + +## gRPC Expander Server Setup +The gRPC server can be set up in many ways, but a simple example is described below. +An example of a barebones gRPC Exapnder Server can be found in the `example` directory under `fake_grpc_server.go` file. This is meant to be copied elsewhere and deployed as a separate +service. Note that the `protos/expander.pb.go` generated protobuf code will also need to be copied and used to serialize/deserizle the Options passed from CA. +Communication between Cluster Autoscaler and the gRPC Server will occur over native kube-proxy. To use this, note the Service and Namespace the gRPC server is deployed in. + +Deploy the gRPC Expander Server as a separate app, listening on a specifc port number. +Start Cluster Autoscaler with the `--grpcExapnderURl=SERVICE_NAME.NAMESPACE_NAME.svc.cluster.local:PORT_NUMBER` flag, as well as `--grpcExpanderCert` pointed at the location of the volume mounted certificate of the gRPC server. + +## Details + +The gRPC client currently transforms nodeInfo objects passed into the expander to v1.Node objects to save rpc call throughput. As such, the gRPC server will not have access to daemonsets and static pods running on each node. + + diff --git a/cluster-autoscaler/expander/grpcplugin/example/fake_grpc_server.go b/cluster-autoscaler/expander/grpcplugin/example/fake_grpc_server.go new file mode 100644 index 000000000000..05fd5f47f06b --- /dev/null +++ b/cluster-autoscaler/expander/grpcplugin/example/fake_grpc_server.go @@ -0,0 +1,104 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package example + +import ( + "context" + "fmt" + "log" + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "k8s.io/autoscaler/cluster-autoscaler/expander/grpcplugin/protos" +) + +// This code is meant to be used as starter code, deployed as a separate app, not in Cluster Autoscaler. +// This serves as the gRPC Expander Server counterpart to the client which lives in this repo +// main.go of said application should simply pass in paths to (optional)cert, (optional)private key, and port, and call Serve to start listening +// copy the protos/expander.pb.go to your other application's repo, so it has access to the protobuf definitions + +// Serve should be called by the main() function in main.go of the Expander Server repo to start serving +func Serve(certPath string, keyPath string, port uint) { + + var grpcServer *grpc.Server + + // If credentials are passed in, use them + if certPath != "" && keyPath != "" { + log.Printf("Using certFile: %v and keyFile: %v", certPath, keyPath) + tlsCredentials, err := credentials.NewServerTLSFromFile(certPath, keyPath) + if err != nil { + log.Fatal("cannot load TLS credentials: ", err) + } + grpcServer = grpc.NewServer(grpc.Creds(tlsCredentials)) + } else { + grpcServer = grpc.NewServer() + } + + netListener := getNetListener(port) + + expanderServerImpl := NewExpanderServerImpl() + + protos.RegisterExpanderServer(grpcServer, expanderServerImpl) + + // start the server + log.Println("Starting server on port ", port) + if err := grpcServer.Serve(netListener); err != nil { + log.Fatalf("failed to serve: %s", err) + } +} + +func getNetListener(port uint) net.Listener { + lis, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + log.Fatalf("failed to listen: %v", err) + panic(fmt.Sprintf("failed to listen: %v", err)) + } + + return lis +} + +// ExpanderServerImpl is an implementation of Expander Server from proto definition +type ExpanderServerImpl struct{} + +// NewExpanderServerImpl is this Expander's implementation of the server +func NewExpanderServerImpl() *ExpanderServerImpl { + return &ExpanderServerImpl{} +} + +// BestOptions method filters out the best options of all options passed from the gRPC Client in CA, according to the defined strategy. +func (ServerImpl *ExpanderServerImpl) BestOptions(ctx context.Context, req *protos.BestOptionsRequest) (*protos.BestOptionsResponse, error) { + opts := req.GetOptions() + log.Printf("Received BestOption Request with %v options", len(opts)) + + // This strategy simply chooses the Option with the longest NodeGroupID name, but can be replaced with any arbitrary logic + longest := 0 + var choice *protos.Option + for _, opt := range opts { + log.Println(opt.NodeGroupId) + if len(opt.NodeGroupId) > longest { + choice = opt + } + } + + log.Print("returned bestOptions with option: ", choice.NodeGroupId) + + // Return just one option for now + return &protos.BestOptionsResponse{ + Options: []*protos.Option{choice}, + }, nil +} diff --git a/cluster-autoscaler/expander/grpcplugin/example/main.go b/cluster-autoscaler/expander/grpcplugin/example/main.go new file mode 100644 index 000000000000..5401416baa01 --- /dev/null +++ b/cluster-autoscaler/expander/grpcplugin/example/main.go @@ -0,0 +1,30 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package example + +import "flag" + +func main() { + + certPath := flag.String("cert-path", "", "Path to cert file for gRPC Expander Server") + keyPath := flag.String("key-path", "", "Path to private key for gRPC Expander Server") + port := flag.Uint("port", 7000, "Port number for server to listen on") + + flag.Parse() + + Serve(*certPath, *keyPath, *port) +} diff --git a/cluster-autoscaler/expander/grpcplugin/grpc_client.go b/cluster-autoscaler/expander/grpcplugin/grpc_client.go new file mode 100644 index 000000000000..7800bd270136 --- /dev/null +++ b/cluster-autoscaler/expander/grpcplugin/grpc_client.go @@ -0,0 +1,143 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package grpcplugin + +import ( + "context" + "log" + "time" + + v1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/cluster-autoscaler/expander" + "k8s.io/autoscaler/cluster-autoscaler/expander/grpcplugin/protos" + "k8s.io/klog/v2" + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +const gRPCTimeout = 5 * time.Second + +type grpcclientstrategy struct { + grpcClient protos.ExpanderClient +} + +// NewFilter returns an expansion filter that creates a gRPC client, and calls out to a gRPC server +func NewFilter(expanderCert string, expanderUrl string) expander.Filter { + client := createGRPCClient(expanderCert, expanderUrl) + if client == nil { + return &grpcclientstrategy{grpcClient: nil} + } + return &grpcclientstrategy{grpcClient: client} +} + +func createGRPCClient(expanderCert string, expanderUrl string) protos.ExpanderClient { + var dialOpt grpc.DialOption + + if expanderCert == "" { + log.Fatalf("GRPC Expander Cert not specified, insecure connections not allowed") + return nil + } + creds, err := credentials.NewClientTLSFromFile(expanderCert, "") + if err != nil { + log.Fatalf("Failed to create TLS credentials %v", err) + return nil + } + dialOpt = grpc.WithTransportCredentials(creds) + klog.V(2).Infof("Dialing: %s with dialopt: %v", expanderUrl, dialOpt) + conn, err := grpc.Dial(expanderUrl, dialOpt) + if err != nil { + log.Fatalf("Fail to dial server: %v", err) + return nil + } + return protos.NewExpanderClient(conn) +} + +func (g *grpcclientstrategy) BestOptions(expansionOptions []expander.Option, nodeInfo map[string]*schedulerframework.NodeInfo) []expander.Option { + if g.grpcClient == nil { + klog.Errorf("Incorrect gRPC client config, filtering no options") + return expansionOptions + } + + // Transform inputs to gRPC inputs + grpcOptionsSlice, nodeGroupIDOptionMap := populateOptionsForGRPC(expansionOptions) + grpcNodeMap := populateNodeInfoForGRPC(nodeInfo) + + // call gRPC server to get BestOption + klog.V(2).Infof("GPRC call of best options to server with %v options", len(nodeGroupIDOptionMap)) + ctx, cancel := context.WithTimeout(context.Background(), gRPCTimeout) + defer cancel() + bestOptionsResponse, err := g.grpcClient.BestOptions(ctx, &protos.BestOptionsRequest{Options: grpcOptionsSlice, NodeMap: grpcNodeMap}) + if err != nil { + klog.V(4).Info("GRPC call timed out, no options filtered") + return expansionOptions + } + + if bestOptionsResponse == nil || bestOptionsResponse.Options == nil { + klog.V(4).Info("GRPC returned nil bestOptions, no options filtered") + return expansionOptions + } + // Transform back options slice + options := transformAndSanitizeOptionsFromGRPC(bestOptionsResponse.Options, nodeGroupIDOptionMap) + if options == nil { + klog.V(4).Info("Unable to sanitize GPRC returned bestOptions, no options filtered") + return expansionOptions + } + return options +} + +// populateOptionsForGRPC creates a map of nodegroup ID and options, as well as a slice of Options objects for the gRPC call +func populateOptionsForGRPC(expansionOptions []expander.Option) ([]*protos.Option, map[string]expander.Option) { + grpcOptionsSlice := []*protos.Option{} + nodeGroupIDOptionMap := make(map[string]expander.Option) + for _, option := range expansionOptions { + nodeGroupIDOptionMap[option.NodeGroup.Id()] = option + grpcOptionsSlice = append(grpcOptionsSlice, newOptionMessage(option.NodeGroup.Id(), int32(option.NodeCount), option.Debug, option.Pods)) + } + return grpcOptionsSlice, nodeGroupIDOptionMap +} + +// populateNodeInfoForGRPC looks at the corresponding v1.Node object per NodeInfo object, and populates the grpcNodeInfoMap with these to pass over grpc +func populateNodeInfoForGRPC(nodeInfos map[string]*schedulerframework.NodeInfo) map[string]*v1.Node { + grpcNodeInfoMap := make(map[string]*v1.Node) + for nodeId, nodeInfo := range nodeInfos { + grpcNodeInfoMap[nodeId] = nodeInfo.Node() + } + return grpcNodeInfoMap +} + +func transformAndSanitizeOptionsFromGRPC(bestOptionsResponseOptions []*protos.Option, nodeGroupIDOptionMap map[string]expander.Option) []expander.Option { + var options []expander.Option + for _, option := range bestOptionsResponseOptions { + if option == nil { + klog.Errorf("GRPC server returned nil Option") + continue + } + if _, ok := nodeGroupIDOptionMap[option.NodeGroupId]; ok { + options = append(options, nodeGroupIDOptionMap[option.NodeGroupId]) + } else { + klog.Errorf("GRPC server returned invalid nodeGroup ID: ", option.NodeGroupId) + continue + } + } + return options +} + +func newOptionMessage(nodeGroupId string, nodeCount int32, debug string, pods []*v1.Pod) *protos.Option { + return &protos.Option{NodeGroupId: nodeGroupId, NodeCount: nodeCount, Debug: debug, Pod: pods} +} diff --git a/cluster-autoscaler/expander/grpcplugin/grpc_client_test.go b/cluster-autoscaler/expander/grpcplugin/grpc_client_test.go new file mode 100644 index 000000000000..65b94a17d54b --- /dev/null +++ b/cluster-autoscaler/expander/grpcplugin/grpc_client_test.go @@ -0,0 +1,276 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package grpcplugin + +import ( + "errors" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/cluster-autoscaler/expander/grpcplugin/protos" + "k8s.io/autoscaler/cluster-autoscaler/expander/mocks" + . "k8s.io/autoscaler/cluster-autoscaler/utils/test" + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" + + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/test" + "k8s.io/autoscaler/cluster-autoscaler/expander" + + _ "github.com/golang/mock/mockgen/model" +) + +var ( + nodes = []*v1.Node{ + BuildTestNode("n1", 1000, 1000), + BuildTestNode("n2", 1000, 1000), + BuildTestNode("n3", 1000, 1000), + BuildTestNode("n4", 1000, 1000), + } + + eoT2Micro = expander.Option{ + Debug: "t2.micro", + NodeGroup: test.NewTestNodeGroup("my-asg.t2.micro", 10, 1, 1, true, false, "t2.micro", nil, nil), + } + eoT2Large = expander.Option{ + Debug: "t2.large", + NodeGroup: test.NewTestNodeGroup("my-asg.t2.large", 10, 1, 1, true, false, "t2.large", nil, nil), + } + eoT3Large = expander.Option{ + Debug: "t3.large", + NodeGroup: test.NewTestNodeGroup("my-asg.t3.large", 10, 1, 1, true, false, "t3.large", nil, nil), + } + eoM44XLarge = expander.Option{ + Debug: "m4.4xlarge", + NodeGroup: test.NewTestNodeGroup("my-asg.m4.4xlarge", 10, 1, 1, true, false, "m4.4xlarge", nil, nil), + } + options = []expander.Option{eoT2Micro, eoT2Large, eoT3Large, eoM44XLarge} + + grpcEoT2Micro = protos.Option{ + NodeGroupId: eoT2Micro.NodeGroup.Id(), + NodeCount: int32(eoT2Micro.NodeCount), + Debug: eoT2Micro.Debug, + Pod: eoT2Micro.Pods, + } + grpcEoT2Large = protos.Option{ + NodeGroupId: eoT2Large.NodeGroup.Id(), + NodeCount: int32(eoT2Large.NodeCount), + Debug: eoT2Large.Debug, + Pod: eoT2Large.Pods, + } + grpcEoT3Large = protos.Option{ + NodeGroupId: eoT3Large.NodeGroup.Id(), + NodeCount: int32(eoT3Large.NodeCount), + Debug: eoT3Large.Debug, + Pod: eoT3Large.Pods, + } + grpcEoM44XLarge = protos.Option{ + NodeGroupId: eoM44XLarge.NodeGroup.Id(), + NodeCount: int32(eoM44XLarge.NodeCount), + Debug: eoM44XLarge.Debug, + Pod: eoM44XLarge.Pods, + } +) + +func TestPopulateOptionsForGrpc(t *testing.T) { + testCases := []struct { + desc string + opts []expander.Option + expectedOpts []*protos.Option + expectedMap map[string]expander.Option + }{ + { + desc: "empty options", + opts: []expander.Option{}, + expectedOpts: []*protos.Option{}, + expectedMap: map[string]expander.Option{}, + }, + { + desc: "one option", + opts: []expander.Option{eoT2Micro}, + expectedOpts: []*protos.Option{&grpcEoT2Micro}, + expectedMap: map[string]expander.Option{eoT2Micro.NodeGroup.Id(): eoT2Micro}, + }, + { + desc: "many options", + opts: options, + expectedOpts: []*protos.Option{&grpcEoT2Micro, &grpcEoT2Large, &grpcEoT3Large, &grpcEoM44XLarge}, + expectedMap: map[string]expander.Option{ + eoT2Micro.NodeGroup.Id(): eoT2Micro, + eoT2Large.NodeGroup.Id(): eoT2Large, + eoT3Large.NodeGroup.Id(): eoT3Large, + eoM44XLarge.NodeGroup.Id(): eoM44XLarge, + }, + }, + } + for _, tc := range testCases { + grpcOptionsSlice, nodeGroupIDOptionMap := populateOptionsForGRPC(tc.opts) + assert.Equal(t, tc.expectedOpts, grpcOptionsSlice) + assert.Equal(t, tc.expectedMap, nodeGroupIDOptionMap) + } +} + +func makeFakeNodeInfos() map[string]*schedulerframework.NodeInfo { + nodeInfos := make(map[string]*schedulerframework.NodeInfo) + for i, opt := range options { + nodeInfo := schedulerframework.NewNodeInfo() + nodeInfo.SetNode(nodes[i]) + nodeInfos[opt.NodeGroup.Id()] = nodeInfo + } + return nodeInfos +} + +func TestPopulateNodeInfoForGRPC(t *testing.T) { + nodeInfos := makeFakeNodeInfos() + grpcNodeInfoMap := populateNodeInfoForGRPC(nodeInfos) + + expectedGrpcNodeInfoMap := make(map[string]*v1.Node) + for i, opt := range options { + expectedGrpcNodeInfoMap[opt.NodeGroup.Id()] = nodes[i] + } + assert.Equal(t, expectedGrpcNodeInfoMap, grpcNodeInfoMap) +} + +func TestValidTransformAndSanitizeOptionsFromGRPC(t *testing.T) { + responseOptionsSlice := []*protos.Option{&grpcEoT2Micro, &grpcEoT3Large, &grpcEoM44XLarge} + nodeGroupIDOptionMap := map[string]expander.Option{ + eoT2Micro.NodeGroup.Id(): eoT2Micro, + eoT2Large.NodeGroup.Id(): eoT2Large, + eoT3Large.NodeGroup.Id(): eoT3Large, + eoM44XLarge.NodeGroup.Id(): eoM44XLarge, + } + + expectedOptions := []expander.Option{eoT2Micro, eoT3Large, eoM44XLarge} + + ret := transformAndSanitizeOptionsFromGRPC(responseOptionsSlice, nodeGroupIDOptionMap) + assert.Equal(t, expectedOptions, ret) +} + +func TestAnInvalidTransformAndSanitizeOptionsFromGRPC(t *testing.T) { + responseOptionsSlice := []*protos.Option{&grpcEoT2Micro, &grpcEoT3Large, &grpcEoM44XLarge} + nodeGroupIDOptionMap := map[string]expander.Option{ + eoT2Micro.NodeGroup.Id(): eoT2Micro, + eoT2Large.NodeGroup.Id(): eoT2Large, + eoT3Large.NodeGroup.Id(): eoT3Large, + } + + ret := transformAndSanitizeOptionsFromGRPC(responseOptionsSlice, nodeGroupIDOptionMap) + assert.Equal(t, []expander.Option{eoT2Micro, eoT3Large}, ret) +} + +func TestBestOptionsValid(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockClient := mocks.NewMockExpanderClient(ctrl) + g := &grpcclientstrategy{mockClient} + + nodeInfos := makeFakeNodeInfos() + grpcNodeInfoMap := make(map[string]*v1.Node) + for i, opt := range options { + grpcNodeInfoMap[opt.NodeGroup.Id()] = nodes[i] + } + expectedBestOptionsReq := &protos.BestOptionsRequest{ + Options: []*protos.Option{&grpcEoT2Micro, &grpcEoT2Large, &grpcEoT3Large, &grpcEoM44XLarge}, + NodeMap: grpcNodeInfoMap, + } + + mockClient.EXPECT().BestOptions( + gomock.Any(), gomock.Eq(expectedBestOptionsReq), + ).Return(&protos.BestOptionsResponse{Options: []*protos.Option{&grpcEoT3Large}}, nil) + + resp := g.BestOptions(options, nodeInfos) + + assert.Equal(t, resp, []expander.Option{eoT3Large}) +} + +// All test cases should error, and no options should be filtered +func TestBestOptionsErrors(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockClient := mocks.NewMockExpanderClient(ctrl) + g := grpcclientstrategy{mockClient} + + badProtosOption := protos.Option{ + NodeGroupId: "badID", + NodeCount: int32(eoM44XLarge.NodeCount), + Debug: eoM44XLarge.Debug, + Pod: eoM44XLarge.Pods, + } + + testCases := []struct { + desc string + client grpcclientstrategy + nodeInfo map[string]*schedulerframework.NodeInfo + mockResponse protos.BestOptionsResponse + errResponse error + }{ + { + desc: "Bad gRPC client config", + client: grpcclientstrategy{nil}, + nodeInfo: makeFakeNodeInfos(), + mockResponse: protos.BestOptionsResponse{}, + errResponse: nil, + }, + { + desc: "gRPC error response", + client: g, + nodeInfo: makeFakeNodeInfos(), + mockResponse: protos.BestOptionsResponse{}, + errResponse: errors.New("timeout error"), + }, + { + desc: "bad bestOptions response", + client: g, + nodeInfo: makeFakeNodeInfos(), + mockResponse: protos.BestOptionsResponse{}, + errResponse: nil, + }, + { + desc: "bad bestOptions response, options nil", + client: g, + nodeInfo: makeFakeNodeInfos(), + mockResponse: protos.BestOptionsResponse{Options: nil}, + errResponse: nil, + }, + { + desc: "bad bestOptions response, options invalid - nil", + client: g, + nodeInfo: makeFakeNodeInfos(), + mockResponse: protos.BestOptionsResponse{Options: []*protos.Option{&grpcEoT2Micro, nil, &grpcEoT2Large, &grpcEoT3Large, &grpcEoM44XLarge}}, + errResponse: nil, + }, + { + desc: "bad bestOptions response, options invalid - nonExistent nodeID", + client: g, + nodeInfo: makeFakeNodeInfos(), + mockResponse: protos.BestOptionsResponse{Options: []*protos.Option{&grpcEoT2Micro, &badProtosOption, &grpcEoT2Large, &grpcEoT3Large, &grpcEoM44XLarge}}, + errResponse: nil, + }, + } + for _, tc := range testCases { + grpcNodeInfoMap := populateNodeInfoForGRPC(tc.nodeInfo) + mockClient.EXPECT().BestOptions( + gomock.Any(), gomock.Eq( + &protos.BestOptionsRequest{ + Options: []*protos.Option{&grpcEoT2Micro, &grpcEoT2Large, &grpcEoT3Large, &grpcEoM44XLarge}, + NodeMap: grpcNodeInfoMap, + })).Return(&tc.mockResponse, tc.errResponse) + resp := g.BestOptions(options, tc.nodeInfo) + + assert.Equal(t, resp, options) + } +} diff --git a/cluster-autoscaler/expander/grpcplugin/protos/expander.pb.go b/cluster-autoscaler/expander/grpcplugin/protos/expander.pb.go new file mode 100644 index 000000000000..3c071d2f37d7 --- /dev/null +++ b/cluster-autoscaler/expander/grpcplugin/protos/expander.pb.go @@ -0,0 +1,440 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protos + +import ( + context "context" + reflect "reflect" + sync "sync" + + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + v1 "k8s.io/api/core/v1" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type BestOptionsRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Options []*Option `protobuf:"bytes,1,rep,name=options,proto3" json:"options,omitempty"` + // key is node id from options + NodeMap map[string]*v1.Node `protobuf:"bytes,2,rep,name=nodeMap,proto3" json:"nodeMap,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *BestOptionsRequest) Reset() { + *x = BestOptionsRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BestOptionsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BestOptionsRequest) ProtoMessage() {} + +func (x *BestOptionsRequest) ProtoReflect() protoreflect.Message { + mi := &file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BestOptionsRequest.ProtoReflect.Descriptor instead. +func (*BestOptionsRequest) Descriptor() ([]byte, []int) { + return file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_rawDescGZIP(), []int{0} +} + +func (x *BestOptionsRequest) GetOptions() []*Option { + if x != nil { + return x.Options + } + return nil +} + +func (x *BestOptionsRequest) GetNodeMap() map[string]*v1.Node { + if x != nil { + return x.NodeMap + } + return nil +} + +type BestOptionsResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Options []*Option `protobuf:"bytes,1,rep,name=options,proto3" json:"options,omitempty"` +} + +func (x *BestOptionsResponse) Reset() { + *x = BestOptionsResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BestOptionsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BestOptionsResponse) ProtoMessage() {} + +func (x *BestOptionsResponse) ProtoReflect() protoreflect.Message { + mi := &file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BestOptionsResponse.ProtoReflect.Descriptor instead. +func (*BestOptionsResponse) Descriptor() ([]byte, []int) { + return file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_rawDescGZIP(), []int{1} +} + +func (x *BestOptionsResponse) GetOptions() []*Option { + if x != nil { + return x.Options + } + return nil +} + +type Option struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // only need the ID of node to uniquely identify the nodeGroup, used in the nodeInfo map. + NodeGroupId string `protobuf:"bytes,1,opt,name=nodeGroupId,proto3" json:"nodeGroupId,omitempty"` + NodeCount int32 `protobuf:"varint,2,opt,name=nodeCount,proto3" json:"nodeCount,omitempty"` + Debug string `protobuf:"bytes,3,opt,name=debug,proto3" json:"debug,omitempty"` + Pod []*v1.Pod `protobuf:"bytes,4,rep,name=pod,proto3" json:"pod,omitempty"` +} + +func (x *Option) Reset() { + *x = Option{} + if protoimpl.UnsafeEnabled { + mi := &file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Option) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Option) ProtoMessage() {} + +func (x *Option) ProtoReflect() protoreflect.Message { + mi := &file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Option.ProtoReflect.Descriptor instead. +func (*Option) Descriptor() ([]byte, []int) { + return file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_rawDescGZIP(), []int{2} +} + +func (x *Option) GetNodeGroupId() string { + if x != nil { + return x.NodeGroupId + } + return "" +} + +func (x *Option) GetNodeCount() int32 { + if x != nil { + return x.NodeCount + } + return 0 +} + +func (x *Option) GetDebug() string { + if x != nil { + return x.Debug + } + return "" +} + +func (x *Option) GetPod() []*v1.Pod { + if x != nil { + return x.Pod + } + return nil +} + +var File_cluster_autoscaler_expander_grpcplugin_protos_expander_proto protoreflect.FileDescriptor + +var file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_rawDesc = []byte{ + 0x0a, 0x3c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2d, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, + 0x61, 0x6c, 0x65, 0x72, 0x2f, 0x65, 0x78, 0x70, 0x61, 0x6e, 0x64, 0x65, 0x72, 0x2f, 0x67, 0x72, + 0x70, 0x63, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x2f, + 0x65, 0x78, 0x70, 0x61, 0x6e, 0x64, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, + 0x67, 0x72, 0x70, 0x63, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x1a, 0x22, 0x6b, 0x38, 0x73, 0x2e, + 0x69, 0x6f, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x76, 0x31, 0x2f, 0x67, + 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xdf, + 0x01, 0x0a, 0x12, 0x42, 0x65, 0x73, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2c, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x70, 0x6c, 0x75, + 0x67, 0x69, 0x6e, 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x12, 0x45, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4d, 0x61, 0x70, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x70, 0x6c, 0x75, 0x67, 0x69, + 0x6e, 0x2e, 0x42, 0x65, 0x73, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x52, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x4d, 0x61, 0x70, 0x1a, 0x54, 0x0a, 0x0c, 0x4e, 0x6f, + 0x64, 0x65, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2e, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x6b, 0x38, + 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0x43, 0x0a, 0x13, 0x42, 0x65, 0x73, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x72, 0x70, 0x63, 0x70, + 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x6f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x89, 0x01, 0x0a, 0x06, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x20, 0x0a, 0x0b, 0x6e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x49, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6e, 0x6f, 0x64, 0x65, 0x47, 0x72, 0x6f, 0x75, 0x70, + 0x49, 0x64, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x6f, 0x64, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x6e, 0x6f, 0x64, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, + 0x12, 0x14, 0x0a, 0x05, 0x64, 0x65, 0x62, 0x75, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x64, 0x65, 0x62, 0x75, 0x67, 0x12, 0x29, 0x0a, 0x03, 0x70, 0x6f, 0x64, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6b, 0x38, 0x73, 0x2e, 0x69, 0x6f, 0x2e, 0x61, 0x70, 0x69, + 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x6f, 0x64, 0x52, 0x03, 0x70, 0x6f, + 0x64, 0x32, 0x5c, 0x0a, 0x08, 0x45, 0x78, 0x70, 0x61, 0x6e, 0x64, 0x65, 0x72, 0x12, 0x50, 0x0a, + 0x0b, 0x42, 0x65, 0x73, 0x74, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1e, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2e, 0x42, 0x65, 0x73, 0x74, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x67, + 0x72, 0x70, 0x63, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2e, 0x42, 0x65, 0x73, 0x74, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, + 0x2f, 0x5a, 0x2d, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2d, 0x61, 0x75, 0x74, 0x6f, 0x73, + 0x63, 0x61, 0x6c, 0x65, 0x72, 0x2f, 0x65, 0x78, 0x70, 0x61, 0x6e, 0x64, 0x65, 0x72, 0x2f, 0x67, + 0x72, 0x70, 0x63, 0x70, 0x6c, 0x75, 0x67, 0x69, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_rawDescOnce sync.Once + file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_rawDescData = file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_rawDesc +) + +func file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_rawDescGZIP() []byte { + file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_rawDescOnce.Do(func() { + file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_rawDescData = protoimpl.X.CompressGZIP(file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_rawDescData) + }) + return file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_rawDescData +} + +var file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_goTypes = []interface{}{ + (*BestOptionsRequest)(nil), // 0: grpcplugin.BestOptionsRequest + (*BestOptionsResponse)(nil), // 1: grpcplugin.BestOptionsResponse + (*Option)(nil), // 2: grpcplugin.Option + nil, // 3: grpcplugin.BestOptionsRequest.NodeMapEntry + (*v1.Pod)(nil), // 4: k8s.io.api.core.v1.Pod + (*v1.Node)(nil), // 5: k8s.io.api.core.v1.Node +} +var file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_depIdxs = []int32{ + 2, // 0: grpcplugin.BestOptionsRequest.options:type_name -> grpcplugin.Option + 3, // 1: grpcplugin.BestOptionsRequest.nodeMap:type_name -> grpcplugin.BestOptionsRequest.NodeMapEntry + 2, // 2: grpcplugin.BestOptionsResponse.options:type_name -> grpcplugin.Option + 4, // 3: grpcplugin.Option.pod:type_name -> k8s.io.api.core.v1.Pod + 5, // 4: grpcplugin.BestOptionsRequest.NodeMapEntry.value:type_name -> k8s.io.api.core.v1.Node + 0, // 5: grpcplugin.Expander.BestOptions:input_type -> grpcplugin.BestOptionsRequest + 1, // 6: grpcplugin.Expander.BestOptions:output_type -> grpcplugin.BestOptionsResponse + 6, // [6:7] is the sub-list for method output_type + 5, // [5:6] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_init() } +func file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_init() { + if File_cluster_autoscaler_expander_grpcplugin_protos_expander_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BestOptionsRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BestOptionsResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Option); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_goTypes, + DependencyIndexes: file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_depIdxs, + MessageInfos: file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_msgTypes, + }.Build() + File_cluster_autoscaler_expander_grpcplugin_protos_expander_proto = out.File + file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_rawDesc = nil + file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_goTypes = nil + file_cluster_autoscaler_expander_grpcplugin_protos_expander_proto_depIdxs = nil +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConnInterface + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion6 + +// ExpanderClient is the client API for Expander service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type ExpanderClient interface { + BestOptions(ctx context.Context, in *BestOptionsRequest, opts ...grpc.CallOption) (*BestOptionsResponse, error) +} + +type expanderClient struct { + cc grpc.ClientConnInterface +} + +func NewExpanderClient(cc grpc.ClientConnInterface) ExpanderClient { + return &expanderClient{cc} +} + +func (c *expanderClient) BestOptions(ctx context.Context, in *BestOptionsRequest, opts ...grpc.CallOption) (*BestOptionsResponse, error) { + out := new(BestOptionsResponse) + err := c.cc.Invoke(ctx, "/grpcplugin.Expander/BestOptions", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ExpanderServer is the server API for Expander service. +type ExpanderServer interface { + BestOptions(context.Context, *BestOptionsRequest) (*BestOptionsResponse, error) +} + +// UnimplementedExpanderServer can be embedded to have forward compatible implementations. +type UnimplementedExpanderServer struct { +} + +func (*UnimplementedExpanderServer) BestOptions(context.Context, *BestOptionsRequest) (*BestOptionsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method BestOptions not implemented") +} + +func RegisterExpanderServer(s *grpc.Server, srv ExpanderServer) { + s.RegisterService(&_Expander_serviceDesc, srv) +} + +func _Expander_BestOptions_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(BestOptionsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExpanderServer).BestOptions(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/grpcplugin.Expander/BestOptions", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExpanderServer).BestOptions(ctx, req.(*BestOptionsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +var _Expander_serviceDesc = grpc.ServiceDesc{ + ServiceName: "grpcplugin.Expander", + HandlerType: (*ExpanderServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "BestOptions", + Handler: _Expander_BestOptions_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "cluster-autoscaler/expander/grpcplugin/protos/expander.proto", +} diff --git a/cluster-autoscaler/expander/grpcplugin/protos/expander.proto b/cluster-autoscaler/expander/grpcplugin/protos/expander.proto new file mode 100644 index 000000000000..5a08e8ff301b --- /dev/null +++ b/cluster-autoscaler/expander/grpcplugin/protos/expander.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package grpcplugin; +import "k8s.io/api/core/v1/generated.proto"; +option go_package = "cluster-autoscaler/expander/grpcplugin/protos"; + + + +// Interface for Expander +service Expander { + + rpc BestOptions (BestOptionsRequest) + returns (BestOptionsResponse) {} +} + +message BestOptionsRequest { + repeated Option options = 1; + // key is node id from options + map nodeMap = 2; +} +message BestOptionsResponse { + repeated Option options = 1; +} +message Option { + // only need the ID of node to uniquely identify the nodeGroup, used in the nodeInfo map. + string nodeGroupId = 1; + int32 nodeCount = 2; + string debug = 3; + repeated k8s.io.api.core.v1.Pod pod = 4; +} diff --git a/cluster-autoscaler/expander/mocks/GRPCPluginExpander.go b/cluster-autoscaler/expander/mocks/GRPCPluginExpander.go new file mode 100644 index 000000000000..358fcfcc857e --- /dev/null +++ b/cluster-autoscaler/expander/mocks/GRPCPluginExpander.go @@ -0,0 +1,107 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + grpc "google.golang.org/grpc" + "k8s.io/autoscaler/cluster-autoscaler/expander/grpcplugin/protos" +) + +// MockExpanderClient is a mock of ExpanderClient interface. +type MockExpanderClient struct { + ctrl *gomock.Controller + recorder *MockExpanderClientMockRecorder +} + +// MockExpanderClientMockRecorder is the mock recorder for MockExpanderClient. +type MockExpanderClientMockRecorder struct { + mock *MockExpanderClient +} + +// NewMockExpanderClient creates a new mock instance. +func NewMockExpanderClient(ctrl *gomock.Controller) *MockExpanderClient { + mock := &MockExpanderClient{ctrl: ctrl} + mock.recorder = &MockExpanderClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockExpanderClient) EXPECT() *MockExpanderClientMockRecorder { + return m.recorder +} + +// BestOptions mocks base method. +func (m *MockExpanderClient) BestOptions(ctx context.Context, in *protos.BestOptionsRequest, opts ...grpc.CallOption) (*protos.BestOptionsResponse, error) { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, in} + for _, a := range opts { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "BestOptions", varargs...) + ret0, _ := ret[0].(*protos.BestOptionsResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// BestOptions indicates an expected call of BestOptions. +func (mr *MockExpanderClientMockRecorder) BestOptions(ctx, in interface{}, opts ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, in}, opts...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BestOptions", reflect.TypeOf((*MockExpanderClient)(nil).BestOptions), varargs...) +} + +// MockExpanderServer is a mock of ExpanderServer interface. +type MockExpanderServer struct { + ctrl *gomock.Controller + recorder *MockExpanderServerMockRecorder +} + +// MockExpanderServerMockRecorder is the mock recorder for MockExpanderServer. +type MockExpanderServerMockRecorder struct { + mock *MockExpanderServer +} + +// NewMockExpanderServer creates a new mock instance. +func NewMockExpanderServer(ctrl *gomock.Controller) *MockExpanderServer { + mock := &MockExpanderServer{ctrl: ctrl} + mock.recorder = &MockExpanderServerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockExpanderServer) EXPECT() *MockExpanderServerMockRecorder { + return m.recorder +} + +// BestOptions mocks base method. +func (m *MockExpanderServer) BestOptions(arg0 context.Context, arg1 *protos.BestOptionsRequest) (*protos.BestOptionsResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BestOptions", arg0, arg1) + ret0, _ := ret[0].(*protos.BestOptionsResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// BestOptions indicates an expected call of BestOptions. +func (mr *MockExpanderServerMockRecorder) BestOptions(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BestOptions", reflect.TypeOf((*MockExpanderServer)(nil).BestOptions), arg0, arg1) +} diff --git a/cluster-autoscaler/expander/mostpods/mostpods.go b/cluster-autoscaler/expander/mostpods/mostpods.go index ec2fc6c012b0..9c5ea375b94e 100644 --- a/cluster-autoscaler/expander/mostpods/mostpods.go +++ b/cluster-autoscaler/expander/mostpods/mostpods.go @@ -18,21 +18,19 @@ package mostpods import ( "k8s.io/autoscaler/cluster-autoscaler/expander" - "k8s.io/autoscaler/cluster-autoscaler/expander/random" schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" ) type mostpods struct { - fallbackStrategy expander.Strategy } -// NewStrategy returns a scale up strategy (expander) that picks the node group that can schedule the most pods -func NewStrategy() expander.Strategy { - return &mostpods{random.NewStrategy()} +// NewFilter returns a scale up filter that picks the node group that can schedule the most pods +func NewFilter() expander.Filter { + return &mostpods{} } // BestOption Selects the expansion option that schedules the most pods -func (m *mostpods) BestOption(expansionOptions []expander.Option, nodeInfo map[string]*schedulerframework.NodeInfo) *expander.Option { +func (m *mostpods) BestOptions(expansionOptions []expander.Option, nodeInfo map[string]*schedulerframework.NodeInfo) []expander.Option { var maxPods int var maxOptions []expander.Option @@ -51,5 +49,5 @@ func (m *mostpods) BestOption(expansionOptions []expander.Option, nodeInfo map[s return nil } - return m.fallbackStrategy.BestOption(maxOptions, nodeInfo) + return maxOptions } diff --git a/cluster-autoscaler/expander/mostpods/mostpods_test.go b/cluster-autoscaler/expander/mostpods/mostpods_test.go index a861f755d2b6..de7c8492ad1c 100644 --- a/cluster-autoscaler/expander/mostpods/mostpods_test.go +++ b/cluster-autoscaler/expander/mostpods/mostpods_test.go @@ -26,18 +26,18 @@ import ( ) func TestMostPods(t *testing.T) { - e := NewStrategy() + e := NewFilter() eo0 := expander.Option{Debug: "EO0"} - ret := e.BestOption([]expander.Option{eo0}, nil) - assert.Equal(t, *ret, eo0) + ret := e.BestOptions([]expander.Option{eo0}, nil) + assert.Equal(t, ret, []expander.Option{eo0}) eo1 := expander.Option{Debug: "EO1", Pods: []*apiv1.Pod{nil}} - ret = e.BestOption([]expander.Option{eo0, eo1}, nil) - assert.Equal(t, *ret, eo1) + ret = e.BestOptions([]expander.Option{eo0, eo1}, nil) + assert.Equal(t, ret, []expander.Option{eo1}) eo1b := expander.Option{Debug: "EO1b", Pods: []*apiv1.Pod{nil}} - ret = e.BestOption([]expander.Option{eo0, eo1, eo1b}, nil) - assert.NotEqual(t, *ret, eo0) - assert.True(t, assert.ObjectsAreEqual(*ret, eo1) || assert.ObjectsAreEqual(*ret, eo1b)) + ret = e.BestOptions([]expander.Option{eo0, eo1, eo1b}, nil) + assert.NotEqual(t, ret, []expander.Option{eo0}) + assert.ObjectsAreEqual(ret, []expander.Option{eo1, eo1b}) } diff --git a/cluster-autoscaler/expander/price/price.go b/cluster-autoscaler/expander/price/price.go index 0c5beaf2526e..8a1297cc2c05 100644 --- a/cluster-autoscaler/expander/price/price.go +++ b/cluster-autoscaler/expander/price/price.go @@ -74,11 +74,11 @@ var ( gpuUnfitnessOverride = 1000.0 ) -// NewStrategy returns an expansion strategy that picks nodes based on price and preferred node type. -func NewStrategy(cloudProvider cloudprovider.CloudProvider, +// NewFilter returns an expansion filter that picks nodes based on price and preferred node type. +func NewFilter(cloudProvider cloudprovider.CloudProvider, preferredNodeProvider PreferredNodeProvider, nodeUnfitness NodeUnfitness, -) expander.Strategy { +) expander.Filter { return &priceBased{ cloudProvider: cloudProvider, preferredNodeProvider: preferredNodeProvider, @@ -87,8 +87,8 @@ func NewStrategy(cloudProvider cloudprovider.CloudProvider, } // BestOption selects option based on cost and preferred node type. -func (p *priceBased) BestOption(expansionOptions []expander.Option, nodeInfos map[string]*schedulerframework.NodeInfo) *expander.Option { - var bestOption *expander.Option +func (p *priceBased) BestOptions(expansionOptions []expander.Option, nodeInfos map[string]*schedulerframework.NodeInfo) []expander.Option { + var bestOptions []expander.Option bestOptionScore := 0.0 now := time.Now() then := now.Add(time.Hour) @@ -169,17 +169,21 @@ nextoption: klog.V(5).Infof("Price expander for %s: %s", option.NodeGroup.Id(), debug) - if bestOption == nil || bestOptionScore > optionScore { - bestOption = &expander.Option{ - NodeGroup: option.NodeGroup, - NodeCount: option.NodeCount, - Debug: fmt.Sprintf("%s | price-expander: %s", option.Debug, debug), - Pods: option.Pods, - } + maybeBestOption := expander.Option{ + NodeGroup: option.NodeGroup, + NodeCount: option.NodeCount, + Debug: fmt.Sprintf("%s | price-expander: %s", option.Debug, debug), + Pods: option.Pods, + } + if len(bestOptions) == 0 || bestOptionScore == optionScore { + bestOptions = append(bestOptions, maybeBestOption) + bestOptionScore = optionScore + } else if bestOptionScore > optionScore { + bestOptions = []expander.Option{maybeBestOption} bestOptionScore = optionScore } } - return bestOption + return bestOptions } // buildPod creates a pod with specified resources. diff --git a/cluster-autoscaler/expander/price/price_test.go b/cluster-autoscaler/expander/price/price_test.go index 0e56a0c611e9..90d2d9982938 100644 --- a/cluster-autoscaler/expander/price/price_test.go +++ b/cluster-autoscaler/expander/price/price_test.go @@ -18,6 +18,7 @@ package price import ( "fmt" + "strings" "testing" "time" @@ -60,6 +61,18 @@ func (tpnp *testPreferredNodeProvider) Node() (*apiv1.Node, error) { return tpnp.preferred, nil } +func optionsToDebug(options []expander.Option) []string { + var ret []string + for _, option := range options { + s := strings.Split(option.Debug, " ") + if len(s) == 0 { + s = append(s, "") + } + ret = append(ret, s[0]) + } + return ret +} + func TestPriceExpander(t *testing.T) { n1 := BuildTestNode("n1", 1000, 1000) n2 := BuildTestNode("n2", 4000, 1000) @@ -117,13 +130,13 @@ func TestPriceExpander(t *testing.T) { }, } provider.SetPricingModel(pricingModel) - assert.Contains(t, NewStrategy( + assert.Equal(t, optionsToDebug(NewFilter( provider, &testPreferredNodeProvider{ preferred: buildNode(2000, units.GiB), }, SimpleNodeUnfitness, - ).BestOption(options, nodeInfosForGroups).Debug, "ng1") + ).BestOptions(options, nodeInfosForGroups)), []string{"ng1"}) // First node group is cheaper, however, the second one is preferred. pricingModel = &testPricingModel{ @@ -138,13 +151,13 @@ func TestPriceExpander(t *testing.T) { }, } provider.SetPricingModel(pricingModel) - assert.Contains(t, NewStrategy( + assert.Equal(t, optionsToDebug(NewFilter( provider, &testPreferredNodeProvider{ preferred: buildNode(4000, units.GiB), }, SimpleNodeUnfitness, - ).BestOption(options, nodeInfosForGroups).Debug, "ng2") + ).BestOptions(options, nodeInfosForGroups)), []string{"ng2"}) // All node groups accept the same set of pods. Lots of nodes. options1b := []expander.Option{ @@ -175,14 +188,14 @@ func TestPriceExpander(t *testing.T) { }, } provider.SetPricingModel(pricingModel) - assert.Contains(t, NewStrategy( + assert.Equal(t, optionsToDebug(NewFilter( provider, &testPreferredNodeProvider{ preferred: buildNode(4000, units.GiB), }, SimpleNodeUnfitness, - ).BestOption(options1b, nodeInfosForGroups).Debug, "ng1") + ).BestOptions(options1b, nodeInfosForGroups)), []string{"ng1"}) // Second node group is cheaper pricingModel = &testPricingModel{ @@ -197,13 +210,13 @@ func TestPriceExpander(t *testing.T) { }, } provider.SetPricingModel(pricingModel) - assert.Contains(t, NewStrategy( + assert.Equal(t, optionsToDebug(NewFilter( provider, &testPreferredNodeProvider{ preferred: buildNode(2000, units.GiB), }, SimpleNodeUnfitness, - ).BestOption(options, nodeInfosForGroups).Debug, "ng2") + ).BestOptions(options, nodeInfosForGroups)), []string{"ng2"}) // First group accept 1 pod and second accepts 2. options2 := []expander.Option{ @@ -234,13 +247,13 @@ func TestPriceExpander(t *testing.T) { provider.SetPricingModel(pricingModel) // Both node groups are equally expensive. However 2 // accept two pods. - assert.Contains(t, NewStrategy( + assert.Equal(t, optionsToDebug(NewFilter( provider, &testPreferredNodeProvider{ preferred: buildNode(2000, units.GiB), }, SimpleNodeUnfitness, - ).BestOption(options2, nodeInfosForGroups).Debug, "ng2") + ).BestOptions(options2, nodeInfosForGroups)), []string{"ng2"}) // Errors are expected pricingModel = &testPricingModel{ @@ -248,13 +261,13 @@ func TestPriceExpander(t *testing.T) { nodePrice: map[string]float64{}, } provider.SetPricingModel(pricingModel) - assert.Nil(t, NewStrategy( + assert.Empty(t, NewFilter( provider, &testPreferredNodeProvider{ preferred: buildNode(2000, units.GiB), }, SimpleNodeUnfitness, - ).BestOption(options2, nodeInfosForGroups)) + ).BestOptions(options2, nodeInfosForGroups)) // Add node info for autoprovisioned group. nodeInfosForGroups["autoprovisioned-MT1"] = ni3 @@ -293,13 +306,13 @@ func TestPriceExpander(t *testing.T) { }, } provider.SetPricingModel(pricingModel) - assert.Contains(t, NewStrategy( + assert.Equal(t, optionsToDebug(NewFilter( provider, &testPreferredNodeProvider{ preferred: buildNode(2000, units.GiB), }, SimpleNodeUnfitness, - ).BestOption(options3, nodeInfosForGroups).Debug, "ng2") + ).BestOptions(options3, nodeInfosForGroups)), []string{"ng2"}) // Choose non-existing group when non-existing is cheaper. pricingModel = &testPricingModel{ @@ -315,11 +328,11 @@ func TestPriceExpander(t *testing.T) { }, } provider.SetPricingModel(pricingModel) - assert.Contains(t, NewStrategy( + assert.Equal(t, optionsToDebug(NewFilter( provider, &testPreferredNodeProvider{ preferred: buildNode(2000, units.GiB), }, SimpleNodeUnfitness, - ).BestOption(options3, nodeInfosForGroups).Debug, "ng3") + ).BestOptions(options3, nodeInfosForGroups)), []string{"ng3"}) } diff --git a/cluster-autoscaler/expander/priority/priority.go b/cluster-autoscaler/expander/priority/priority.go index 12e5bad541b3..7d6f63b849cf 100644 --- a/cluster-autoscaler/expander/priority/priority.go +++ b/cluster-autoscaler/expander/priority/priority.go @@ -24,8 +24,6 @@ import ( "gopkg.in/yaml.v2" "k8s.io/autoscaler/cluster-autoscaler/expander" - "k8s.io/autoscaler/cluster-autoscaler/expander/random" - caserrors "k8s.io/autoscaler/cluster-autoscaler/utils/errors" apiv1 "k8s.io/api/core/v1" v1lister "k8s.io/client-go/listers/core/v1" @@ -45,21 +43,19 @@ type priorities map[int][]*regexp.Regexp type priority struct { logRecorder record.EventRecorder - fallbackStrategy expander.Strategy okConfigUpdates int badConfigUpdates int configMapLister v1lister.ConfigMapNamespaceLister } -// NewStrategy returns an expansion strategy that picks node groups based on user-defined priorities -func NewStrategy(configMapLister v1lister.ConfigMapNamespaceLister, - logRecorder record.EventRecorder) (expander.Strategy, caserrors.AutoscalerError) { +// NewFilter returns an expansion filter that picks node groups based on user-defined priorities +func NewFilter(configMapLister v1lister.ConfigMapNamespaceLister, + logRecorder record.EventRecorder) expander.Filter { res := &priority{ - logRecorder: logRecorder, - fallbackStrategy: random.NewStrategy(), - configMapLister: configMapLister, + logRecorder: logRecorder, + configMapLister: configMapLister, } - return res, nil + return res } func (p *priority) reloadConfigMap() (priorities, *apiv1.ConfigMap, error) { @@ -120,7 +116,7 @@ func (p *priority) parsePrioritiesYAMLString(prioritiesYAML string) (priorities, return newPriorities, nil } -func (p *priority) BestOption(expansionOptions []expander.Option, nodeInfo map[string]*schedulerframework.NodeInfo) *expander.Option { +func (p *priority) BestOptions(expansionOptions []expander.Option, nodeInfo map[string]*schedulerframework.NodeInfo) []expander.Option { if len(expansionOptions) <= 0 { return nil } @@ -157,15 +153,15 @@ func (p *priority) BestOption(expansionOptions []expander.Option, nodeInfo map[s } if len(best) == 0 { - msg := "Priority expander: no priorities info found for any of the expansion options. Falling back to random choice." + msg := "Priority expander: no priorities info found for any of the expansion options. No options filtered." p.logConfigWarning(cm, "PriorityConfigMapNoGroupMatched", msg) - return p.fallbackStrategy.BestOption(expansionOptions, nodeInfo) + return expansionOptions } for _, opt := range best { klog.V(2).Infof("priority expander: %s chosen as the highest available", opt.NodeGroup.Id()) } - return p.fallbackStrategy.BestOption(best, nodeInfo) + return best } func (p *priority) groupIDMatchesList(id string, nameRegexpList []*regexp.Regexp) bool { diff --git a/cluster-autoscaler/expander/priority/priority_test.go b/cluster-autoscaler/expander/priority/priority_test.go index 1796ef7a422d..c4cbf2dad71b 100644 --- a/cluster-autoscaler/expander/priority/priority_test.go +++ b/cluster-autoscaler/expander/priority/priority_test.go @@ -87,7 +87,7 @@ var ( } ) -func getStrategyInstance(t *testing.T, config string) (expander.Strategy, *record.FakeRecorder, *apiv1.ConfigMap, error) { +func getFilterInstance(t *testing.T, config string) (expander.Filter, *record.FakeRecorder, *apiv1.ConfigMap, error) { cm := &apiv1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Namespace: testNamespace, @@ -100,50 +100,44 @@ func getStrategyInstance(t *testing.T, config string) (expander.Strategy, *recor lister, err := kubernetes.NewTestConfigMapLister([]*apiv1.ConfigMap{cm}) assert.Nil(t, err) r := record.NewFakeRecorder(100) - s, err := NewStrategy(lister.ConfigMaps(testNamespace), r) + s := NewFilter(lister.ConfigMaps(testNamespace), r) return s, r, cm, err } -func TestPriorityExpanderCorrecltySelectsSingleMatchingOptionOutOfOne(t *testing.T) { - s, _, _, _ := getStrategyInstance(t, config) - ret := s.BestOption([]expander.Option{eoT2Large}, nil) - assert.Equal(t, *ret, eoT2Large) +func TestPriorityExpanderCorrecltyFiltersSingleMatchingOptionOutOfOne(t *testing.T) { + s, _, _, _ := getFilterInstance(t, config) + ret := s.BestOptions([]expander.Option{eoT2Large}, nil) + assert.Equal(t, ret, []expander.Option{eoT2Large}) } -func TestPriorityExpanderCorrecltySelectsSingleMatchingOptionOutOfMany(t *testing.T) { - s, _, _, _ := getStrategyInstance(t, config) - ret := s.BestOption([]expander.Option{eoT2Large, eoM44XLarge}, nil) - assert.Equal(t, *ret, eoM44XLarge) +func TestPriorityExpanderCorrecltyFiltersSingleMatchingOptionOutOfMany(t *testing.T) { + s, _, _, _ := getFilterInstance(t, config) + ret := s.BestOptions([]expander.Option{eoT2Large, eoM44XLarge}, nil) + assert.Equal(t, ret, []expander.Option{eoM44XLarge}) } -func TestPriorityExpanderDoesNotFallBackToRandomWhenHigherPriorityMatches(t *testing.T) { - s, _, _, _ := getStrategyInstance(t, wildcardMatchConfig) - for i := 0; i < 10; i++ { - ret := s.BestOption([]expander.Option{eoT2Large, eoT2Micro}, nil) - assert.Equal(t, *ret, eoT2Large) - } +func TestPriorityExpanderFiltersToHigherPriorityMatch(t *testing.T) { + s, _, _, _ := getFilterInstance(t, wildcardMatchConfig) + ret := s.BestOptions([]expander.Option{eoT2Large, eoT2Micro}, nil) + assert.Equal(t, ret, []expander.Option{eoT2Large}) } -func TestPriorityExpanderCorrecltySelectsOneOfTwoMatchingOptionsOutOfMany(t *testing.T) { - s, _, _, _ := getStrategyInstance(t, config) - for i := 0; i < 10; i++ { - ret := s.BestOption([]expander.Option{eoT2Large, eoT3Large, eoT2Micro}, nil) - assert.True(t, ret.NodeGroup.Id() == eoT2Large.NodeGroup.Id() || ret.NodeGroup.Id() == eoT3Large.NodeGroup.Id()) - } +func TestPriorityExpanderCorrecltyFiltersTwoMatchingOptionsOutOfMany(t *testing.T) { + s, _, _, _ := getFilterInstance(t, config) + ret := s.BestOptions([]expander.Option{eoT2Large, eoT3Large, eoT2Micro}, nil) + assert.Equal(t, ret, []expander.Option{eoT2Large, eoT3Large}) } -func TestPriorityExpanderCorrecltyFallsBackToRandomWhenNoMatches(t *testing.T) { - s, _, _, _ := getStrategyInstance(t, config) - for i := 0; i < 10; i++ { - ret := s.BestOption([]expander.Option{eoT2Large, eoT3Large}, nil) - assert.True(t, ret.NodeGroup.Id() == eoT2Large.NodeGroup.Id() || ret.NodeGroup.Id() == eoT3Large.NodeGroup.Id()) - } +func TestPriorityExpanderCorrecltyFallsBackToAllWhenNoMatches(t *testing.T) { + s, _, _, _ := getFilterInstance(t, config) + ret := s.BestOptions([]expander.Option{eoT2Large, eoT3Large}, nil) + assert.Equal(t, ret, []expander.Option{eoT2Large, eoT3Large}) } func TestPriorityExpanderCorrecltyHandlesConfigUpdate(t *testing.T) { - s, r, cm, _ := getStrategyInstance(t, oneEntryConfig) - ret := s.BestOption([]expander.Option{eoT2Large, eoT3Large, eoM44XLarge}, nil) - assert.Equal(t, *ret, eoT2Large) + s, r, cm, _ := getFilterInstance(t, oneEntryConfig) + ret := s.BestOptions([]expander.Option{eoT2Large, eoT3Large, eoM44XLarge}, nil) + assert.Equal(t, ret, []expander.Option{eoT2Large}) var event string for _, group := range []string{eoT3Large.NodeGroup.Id(), eoM44XLarge.NodeGroup.Id()} { @@ -152,24 +146,24 @@ func TestPriorityExpanderCorrecltyHandlesConfigUpdate(t *testing.T) { } cm.Data[ConfigMapKey] = config - ret = s.BestOption([]expander.Option{eoT2Large, eoT3Large, eoM44XLarge}, nil) + ret = s.BestOptions([]expander.Option{eoT2Large, eoT3Large, eoM44XLarge}, nil) priority := s.(*priority) assert.Equal(t, 2, priority.okConfigUpdates) - assert.Equal(t, *ret, eoM44XLarge) + assert.Equal(t, ret, []expander.Option{eoM44XLarge}) } func TestPriorityExpanderCorrecltySkipsBadChangeConfig(t *testing.T) { - s, r, cm, _ := getStrategyInstance(t, oneEntryConfig) + s, r, cm, _ := getFilterInstance(t, oneEntryConfig) priority := s.(*priority) assert.Equal(t, 0, priority.okConfigUpdates) cm.Data[ConfigMapKey] = "" - ret := s.BestOption([]expander.Option{eoT2Large, eoT3Large, eoM44XLarge}, nil) + ret := s.BestOptions([]expander.Option{eoT2Large, eoT3Large, eoM44XLarge}, nil) assert.Equal(t, 1, priority.badConfigUpdates) event := <-r.Events assert.EqualValues(t, configWarnConfigMapEmpty, event) - assert.Nil(t, ret) + assert.Empty(t, ret) } diff --git a/cluster-autoscaler/expander/random/random.go b/cluster-autoscaler/expander/random/random.go index 6120589fcafe..a789a01b9260 100644 --- a/cluster-autoscaler/expander/random/random.go +++ b/cluster-autoscaler/expander/random/random.go @@ -26,12 +26,26 @@ import ( type random struct { } +// NewFilter returns an expansion filter that randomly picks between node groups +func NewFilter() expander.Filter { + return &random{} +} + // NewStrategy returns an expansion strategy that randomly picks between node groups func NewStrategy() expander.Strategy { return &random{} } -// RandomExpansion Selects from the expansion options at random +// BestOptions selects from the expansion options at random +func (r *random) BestOptions(expansionOptions []expander.Option, nodeInfo map[string]*schedulerframework.NodeInfo) []expander.Option { + best := r.BestOption(expansionOptions, nodeInfo) + if best == nil { + return nil + } + return []expander.Option{*best} +} + +// BestOption selects from the expansion options at random func (r *random) BestOption(expansionOptions []expander.Option, nodeInfo map[string]*schedulerframework.NodeInfo) *expander.Option { if len(expansionOptions) <= 0 { return nil diff --git a/cluster-autoscaler/expander/waste/waste.go b/cluster-autoscaler/expander/waste/waste.go index ccb47d15700d..a4a7768b8835 100644 --- a/cluster-autoscaler/expander/waste/waste.go +++ b/cluster-autoscaler/expander/waste/waste.go @@ -20,22 +20,20 @@ import ( apiv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/autoscaler/cluster-autoscaler/expander" - "k8s.io/autoscaler/cluster-autoscaler/expander/random" klog "k8s.io/klog/v2" schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" ) type leastwaste struct { - fallbackStrategy expander.Strategy } -// NewStrategy returns a strategy that selects the best scale up option based on which node group returns the least waste -func NewStrategy() expander.Strategy { - return &leastwaste{random.NewStrategy()} +// NewFilter returns a filter that selects the best scale up option based on which node group returns the least waste +func NewFilter() expander.Filter { + return &leastwaste{} } // BestOption Finds the option that wastes the least fraction of CPU and Memory -func (l *leastwaste) BestOption(expansionOptions []expander.Option, nodeInfo map[string]*schedulerframework.NodeInfo) *expander.Option { +func (l *leastwaste) BestOptions(expansionOptions []expander.Option, nodeInfo map[string]*schedulerframework.NodeInfo) []expander.Option { var leastWastedScore float64 var leastWastedOptions []expander.Option @@ -70,7 +68,7 @@ func (l *leastwaste) BestOption(expansionOptions []expander.Option, nodeInfo map return nil } - return l.fallbackStrategy.BestOption(leastWastedOptions, nodeInfo) + return leastWastedOptions } func resourcesForPods(pods []*apiv1.Pod) (cpu resource.Quantity, memory resource.Quantity) { diff --git a/cluster-autoscaler/expander/waste/waste_test.go b/cluster-autoscaler/expander/waste/waste_test.go index e138e5e3c814..945188d19b4c 100644 --- a/cluster-autoscaler/expander/waste/waste_test.go +++ b/cluster-autoscaler/expander/waste/waste_test.go @@ -77,14 +77,14 @@ func makeNodeInfo(cpu int64, memory int64, pods int64) *schedulerframework.NodeI func TestLeastWaste(t *testing.T) { cpuPerPod := int64(500) memoryPerPod := int64(1000 * 1024 * 1024) - e := NewStrategy() + e := NewFilter() balancedNodeInfo := makeNodeInfo(16*cpuPerPod, 16*memoryPerPod, 100) nodeMap := map[string]*schedulerframework.NodeInfo{"balanced": balancedNodeInfo} balancedOption := expander.Option{NodeGroup: &FakeNodeGroup{"balanced"}, NodeCount: 1} // Test without any pods, one node info - ret := e.BestOption([]expander.Option{balancedOption}, nodeMap) - assert.Equal(t, *ret, balancedOption) + ret := e.BestOptions([]expander.Option{balancedOption}, nodeMap) + assert.Equal(t, ret, []expander.Option{balancedOption}) pod := &apiv1.Pod{ Spec: apiv1.PodSpec{ @@ -103,20 +103,20 @@ func TestLeastWaste(t *testing.T) { // Test with one pod, one node info balancedOption.Pods = []*apiv1.Pod{pod} - ret = e.BestOption([]expander.Option{balancedOption}, nodeMap) - assert.Equal(t, *ret, balancedOption) + ret = e.BestOptions([]expander.Option{balancedOption}, nodeMap) + assert.Equal(t, ret, []expander.Option{balancedOption}) // Test with one pod, two node infos, one that has lots of RAM one that has less highmemNodeInfo := makeNodeInfo(16*cpuPerPod, 32*memoryPerPod, 100) nodeMap["highmem"] = highmemNodeInfo highmemOption := expander.Option{NodeGroup: &FakeNodeGroup{"highmem"}, NodeCount: 1, Pods: []*apiv1.Pod{pod}} - ret = e.BestOption([]expander.Option{balancedOption, highmemOption}, nodeMap) - assert.Equal(t, *ret, balancedOption) + ret = e.BestOptions([]expander.Option{balancedOption, highmemOption}, nodeMap) + assert.Equal(t, ret, []expander.Option{balancedOption}) // Test with one pod, three node infos, one that has lots of RAM one that has less, and one that has less CPU lowcpuNodeInfo := makeNodeInfo(8*cpuPerPod, 16*memoryPerPod, 100) nodeMap["lowcpu"] = lowcpuNodeInfo lowcpuOption := expander.Option{NodeGroup: &FakeNodeGroup{"lowcpu"}, NodeCount: 1, Pods: []*apiv1.Pod{pod}} - ret = e.BestOption([]expander.Option{balancedOption, highmemOption, lowcpuOption}, nodeMap) - assert.Equal(t, *ret, lowcpuOption) + ret = e.BestOptions([]expander.Option{balancedOption, highmemOption, lowcpuOption}, nodeMap) + assert.Equal(t, ret, []expander.Option{lowcpuOption}) } diff --git a/cluster-autoscaler/go.mod b/cluster-autoscaler/go.mod index 014a3a4ed9f5..5ddcfcaabb49 100644 --- a/cluster-autoscaler/go.mod +++ b/cluster-autoscaler/go.mod @@ -24,10 +24,12 @@ require ( github.com/pkg/errors v0.9.1 github.com/satori/go.uuid v1.2.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.6.1 - golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 - golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d + github.com/stretchr/testify v1.7.0 + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 + golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f google.golang.org/api v0.20.0 + google.golang.org/grpc v1.40.0 + google.golang.org/protobuf v1.27.1 gopkg.in/gcfg.v1 v1.2.0 gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.0.0 diff --git a/cluster-autoscaler/main.go b/cluster-autoscaler/main.go index 1c3d87932ffb..b11e8efffa97 100644 --- a/cluster-autoscaler/main.go +++ b/cluster-autoscaler/main.go @@ -150,8 +150,10 @@ var ( estimatorFlag = flag.String("estimator", estimator.BinpackingEstimatorName, "Type of resource estimator to be used in scale up. Available values: ["+strings.Join(estimator.AvailableEstimators, ",")+"]") - expanderFlag = flag.String("expander", expander.RandomExpanderName, - "Type of node group expander to be used in scale up. Available values: ["+strings.Join(expander.AvailableExpanders, ",")+"]") + expanderFlag = flag.String("expander", "", "Type of node group expander to be used in scale up. Available values: ["+strings.Join(expander.AvailableExpanders, ",")+"]. Specifying multiple values separated by commas will call the expanders in succession until there is only one option remaining. Ties still existing after this process are broken randomly.") + + grpcExpanderCert = flag.String("grpc-expander-cert", "", "Path to cert used by gRPC server over TLS") + grpcExpanderURL = flag.String("grpc-expander-url", "", "URL to reach gRPC expander server.") ignoreDaemonSetsUtilization = flag.Bool("ignore-daemonsets-utilization", false, "Should CA ignore DaemonSet pods when calculating resource utilization for scaling down") @@ -203,7 +205,9 @@ func createAutoscalingOptions() config.AutoscalingOptions { OkTotalUnreadyCount: *okTotalUnreadyCount, ScaleUpFromZero: *scaleUpFromZero, EstimatorName: *estimatorFlag, - ExpanderName: *expanderFlag, + ExpanderNames: *expanderFlag, + GRPCExpanderCert: *grpcExpanderCert, + GRPCExpanderURL: *grpcExpanderURL, IgnoreDaemonSetsUtilization: *ignoreDaemonSetsUtilization, IgnoreMirrorPodsUtilization: *ignoreMirrorPodsUtilization, MaxBulkSoftTaintCount: *maxBulkSoftTaintCount, diff --git a/cluster-autoscaler/utils/scheduler/scheduler.go b/cluster-autoscaler/utils/scheduler/scheduler.go index c78e12a9fcfd..f3e4439f6748 100644 --- a/cluster-autoscaler/utils/scheduler/scheduler.go +++ b/cluster-autoscaler/utils/scheduler/scheduler.go @@ -17,7 +17,10 @@ limitations under the License. package scheduler import ( + "fmt" + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/uuid" schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" ) @@ -57,3 +60,25 @@ func CreateNodeNameToInfoMap(pods []*apiv1.Pod, nodes []*apiv1.Node) map[string] return nodeNameToNodeInfo } + +// DeepCopyTemplateNode copies NodeInfo object used as a template. It changes +// names of UIDs of both node and pods running on it, so that copies can be used +// to represent multiple nodes. +func DeepCopyTemplateNode(nodeTemplate *schedulerframework.NodeInfo, index int) *schedulerframework.NodeInfo { + node := nodeTemplate.Node().DeepCopy() + node.Name = fmt.Sprintf("%s-%d", node.Name, index) + node.UID = uuid.NewUUID() + if node.Labels == nil { + node.Labels = make(map[string]string) + } + node.Labels["kubernetes.io/hostname"] = node.Name + nodeInfo := schedulerframework.NewNodeInfo() + nodeInfo.SetNode(node) + for _, podInfo := range nodeTemplate.Pods { + pod := podInfo.Pod.DeepCopy() + pod.Name = fmt.Sprintf("%s-%d", podInfo.Pod.Name, index) + pod.UID = uuid.NewUUID() + nodeInfo.AddPod(pod) + } + return nodeInfo +} diff --git a/cluster-autoscaler/vendor/github.com/golang/mock/mockgen/model/model.go b/cluster-autoscaler/vendor/github.com/golang/mock/mockgen/model/model.go new file mode 100644 index 000000000000..2c6a62ceb268 --- /dev/null +++ b/cluster-autoscaler/vendor/github.com/golang/mock/mockgen/model/model.go @@ -0,0 +1,495 @@ +// Copyright 2012 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package model contains the data model necessary for generating mock implementations. +package model + +import ( + "encoding/gob" + "fmt" + "io" + "reflect" + "strings" +) + +// pkgPath is the importable path for package model +const pkgPath = "github.com/golang/mock/mockgen/model" + +// Package is a Go package. It may be a subset. +type Package struct { + Name string + PkgPath string + Interfaces []*Interface + DotImports []string +} + +// Print writes the package name and its exported interfaces. +func (pkg *Package) Print(w io.Writer) { + _, _ = fmt.Fprintf(w, "package %s\n", pkg.Name) + for _, intf := range pkg.Interfaces { + intf.Print(w) + } +} + +// Imports returns the imports needed by the Package as a set of import paths. +func (pkg *Package) Imports() map[string]bool { + im := make(map[string]bool) + for _, intf := range pkg.Interfaces { + intf.addImports(im) + } + return im +} + +// Interface is a Go interface. +type Interface struct { + Name string + Methods []*Method +} + +// Print writes the interface name and its methods. +func (intf *Interface) Print(w io.Writer) { + _, _ = fmt.Fprintf(w, "interface %s\n", intf.Name) + for _, m := range intf.Methods { + m.Print(w) + } +} + +func (intf *Interface) addImports(im map[string]bool) { + for _, m := range intf.Methods { + m.addImports(im) + } +} + +// AddMethod adds a new method, de-duplicating by method name. +func (intf *Interface) AddMethod(m *Method) { + for _, me := range intf.Methods { + if me.Name == m.Name { + return + } + } + intf.Methods = append(intf.Methods, m) +} + +// Method is a single method of an interface. +type Method struct { + Name string + In, Out []*Parameter + Variadic *Parameter // may be nil +} + +// Print writes the method name and its signature. +func (m *Method) Print(w io.Writer) { + _, _ = fmt.Fprintf(w, " - method %s\n", m.Name) + if len(m.In) > 0 { + _, _ = fmt.Fprintf(w, " in:\n") + for _, p := range m.In { + p.Print(w) + } + } + if m.Variadic != nil { + _, _ = fmt.Fprintf(w, " ...:\n") + m.Variadic.Print(w) + } + if len(m.Out) > 0 { + _, _ = fmt.Fprintf(w, " out:\n") + for _, p := range m.Out { + p.Print(w) + } + } +} + +func (m *Method) addImports(im map[string]bool) { + for _, p := range m.In { + p.Type.addImports(im) + } + if m.Variadic != nil { + m.Variadic.Type.addImports(im) + } + for _, p := range m.Out { + p.Type.addImports(im) + } +} + +// Parameter is an argument or return parameter of a method. +type Parameter struct { + Name string // may be empty + Type Type +} + +// Print writes a method parameter. +func (p *Parameter) Print(w io.Writer) { + n := p.Name + if n == "" { + n = `""` + } + _, _ = fmt.Fprintf(w, " - %v: %v\n", n, p.Type.String(nil, "")) +} + +// Type is a Go type. +type Type interface { + String(pm map[string]string, pkgOverride string) string + addImports(im map[string]bool) +} + +func init() { + gob.Register(&ArrayType{}) + gob.Register(&ChanType{}) + gob.Register(&FuncType{}) + gob.Register(&MapType{}) + gob.Register(&NamedType{}) + gob.Register(&PointerType{}) + + // Call gob.RegisterName to make sure it has the consistent name registered + // for both gob decoder and encoder. + // + // For a non-pointer type, gob.Register will try to get package full path by + // calling rt.PkgPath() for a name to register. If your project has vendor + // directory, it is possible that PkgPath will get a path like this: + // ../../../vendor/github.com/golang/mock/mockgen/model + gob.RegisterName(pkgPath+".PredeclaredType", PredeclaredType("")) +} + +// ArrayType is an array or slice type. +type ArrayType struct { + Len int // -1 for slices, >= 0 for arrays + Type Type +} + +func (at *ArrayType) String(pm map[string]string, pkgOverride string) string { + s := "[]" + if at.Len > -1 { + s = fmt.Sprintf("[%d]", at.Len) + } + return s + at.Type.String(pm, pkgOverride) +} + +func (at *ArrayType) addImports(im map[string]bool) { at.Type.addImports(im) } + +// ChanType is a channel type. +type ChanType struct { + Dir ChanDir // 0, 1 or 2 + Type Type +} + +func (ct *ChanType) String(pm map[string]string, pkgOverride string) string { + s := ct.Type.String(pm, pkgOverride) + if ct.Dir == RecvDir { + return "<-chan " + s + } + if ct.Dir == SendDir { + return "chan<- " + s + } + return "chan " + s +} + +func (ct *ChanType) addImports(im map[string]bool) { ct.Type.addImports(im) } + +// ChanDir is a channel direction. +type ChanDir int + +// Constants for channel directions. +const ( + RecvDir ChanDir = 1 + SendDir ChanDir = 2 +) + +// FuncType is a function type. +type FuncType struct { + In, Out []*Parameter + Variadic *Parameter // may be nil +} + +func (ft *FuncType) String(pm map[string]string, pkgOverride string) string { + args := make([]string, len(ft.In)) + for i, p := range ft.In { + args[i] = p.Type.String(pm, pkgOverride) + } + if ft.Variadic != nil { + args = append(args, "..."+ft.Variadic.Type.String(pm, pkgOverride)) + } + rets := make([]string, len(ft.Out)) + for i, p := range ft.Out { + rets[i] = p.Type.String(pm, pkgOverride) + } + retString := strings.Join(rets, ", ") + if nOut := len(ft.Out); nOut == 1 { + retString = " " + retString + } else if nOut > 1 { + retString = " (" + retString + ")" + } + return "func(" + strings.Join(args, ", ") + ")" + retString +} + +func (ft *FuncType) addImports(im map[string]bool) { + for _, p := range ft.In { + p.Type.addImports(im) + } + if ft.Variadic != nil { + ft.Variadic.Type.addImports(im) + } + for _, p := range ft.Out { + p.Type.addImports(im) + } +} + +// MapType is a map type. +type MapType struct { + Key, Value Type +} + +func (mt *MapType) String(pm map[string]string, pkgOverride string) string { + return "map[" + mt.Key.String(pm, pkgOverride) + "]" + mt.Value.String(pm, pkgOverride) +} + +func (mt *MapType) addImports(im map[string]bool) { + mt.Key.addImports(im) + mt.Value.addImports(im) +} + +// NamedType is an exported type in a package. +type NamedType struct { + Package string // may be empty + Type string +} + +func (nt *NamedType) String(pm map[string]string, pkgOverride string) string { + if pkgOverride == nt.Package { + return nt.Type + } + prefix := pm[nt.Package] + if prefix != "" { + return prefix + "." + nt.Type + } + + return nt.Type +} + +func (nt *NamedType) addImports(im map[string]bool) { + if nt.Package != "" { + im[nt.Package] = true + } +} + +// PointerType is a pointer to another type. +type PointerType struct { + Type Type +} + +func (pt *PointerType) String(pm map[string]string, pkgOverride string) string { + return "*" + pt.Type.String(pm, pkgOverride) +} +func (pt *PointerType) addImports(im map[string]bool) { pt.Type.addImports(im) } + +// PredeclaredType is a predeclared type such as "int". +type PredeclaredType string + +func (pt PredeclaredType) String(map[string]string, string) string { return string(pt) } +func (pt PredeclaredType) addImports(map[string]bool) {} + +// The following code is intended to be called by the program generated by ../reflect.go. + +// InterfaceFromInterfaceType returns a pointer to an interface for the +// given reflection interface type. +func InterfaceFromInterfaceType(it reflect.Type) (*Interface, error) { + if it.Kind() != reflect.Interface { + return nil, fmt.Errorf("%v is not an interface", it) + } + intf := &Interface{} + + for i := 0; i < it.NumMethod(); i++ { + mt := it.Method(i) + // TODO: need to skip unexported methods? or just raise an error? + m := &Method{ + Name: mt.Name, + } + + var err error + m.In, m.Variadic, m.Out, err = funcArgsFromType(mt.Type) + if err != nil { + return nil, err + } + + intf.AddMethod(m) + } + + return intf, nil +} + +// t's Kind must be a reflect.Func. +func funcArgsFromType(t reflect.Type) (in []*Parameter, variadic *Parameter, out []*Parameter, err error) { + nin := t.NumIn() + if t.IsVariadic() { + nin-- + } + var p *Parameter + for i := 0; i < nin; i++ { + p, err = parameterFromType(t.In(i)) + if err != nil { + return + } + in = append(in, p) + } + if t.IsVariadic() { + p, err = parameterFromType(t.In(nin).Elem()) + if err != nil { + return + } + variadic = p + } + for i := 0; i < t.NumOut(); i++ { + p, err = parameterFromType(t.Out(i)) + if err != nil { + return + } + out = append(out, p) + } + return +} + +func parameterFromType(t reflect.Type) (*Parameter, error) { + tt, err := typeFromType(t) + if err != nil { + return nil, err + } + return &Parameter{Type: tt}, nil +} + +var errorType = reflect.TypeOf((*error)(nil)).Elem() + +var byteType = reflect.TypeOf(byte(0)) + +func typeFromType(t reflect.Type) (Type, error) { + // Hack workaround for https://golang.org/issue/3853. + // This explicit check should not be necessary. + if t == byteType { + return PredeclaredType("byte"), nil + } + + if imp := t.PkgPath(); imp != "" { + return &NamedType{ + Package: impPath(imp), + Type: t.Name(), + }, nil + } + + // only unnamed or predeclared types after here + + // Lots of types have element types. Let's do the parsing and error checking for all of them. + var elemType Type + switch t.Kind() { + case reflect.Array, reflect.Chan, reflect.Map, reflect.Ptr, reflect.Slice: + var err error + elemType, err = typeFromType(t.Elem()) + if err != nil { + return nil, err + } + } + + switch t.Kind() { + case reflect.Array: + return &ArrayType{ + Len: t.Len(), + Type: elemType, + }, nil + case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128, reflect.String: + return PredeclaredType(t.Kind().String()), nil + case reflect.Chan: + var dir ChanDir + switch t.ChanDir() { + case reflect.RecvDir: + dir = RecvDir + case reflect.SendDir: + dir = SendDir + } + return &ChanType{ + Dir: dir, + Type: elemType, + }, nil + case reflect.Func: + in, variadic, out, err := funcArgsFromType(t) + if err != nil { + return nil, err + } + return &FuncType{ + In: in, + Out: out, + Variadic: variadic, + }, nil + case reflect.Interface: + // Two special interfaces. + if t.NumMethod() == 0 { + return PredeclaredType("interface{}"), nil + } + if t == errorType { + return PredeclaredType("error"), nil + } + case reflect.Map: + kt, err := typeFromType(t.Key()) + if err != nil { + return nil, err + } + return &MapType{ + Key: kt, + Value: elemType, + }, nil + case reflect.Ptr: + return &PointerType{ + Type: elemType, + }, nil + case reflect.Slice: + return &ArrayType{ + Len: -1, + Type: elemType, + }, nil + case reflect.Struct: + if t.NumField() == 0 { + return PredeclaredType("struct{}"), nil + } + } + + // TODO: Struct, UnsafePointer + return nil, fmt.Errorf("can't yet turn %v (%v) into a model.Type", t, t.Kind()) +} + +// impPath sanitizes the package path returned by `PkgPath` method of a reflect Type so that +// it is importable. PkgPath might return a path that includes "vendor". These paths do not +// compile, so we need to remove everything up to and including "/vendor/". +// See https://github.com/golang/go/issues/12019. +func impPath(imp string) string { + if strings.HasPrefix(imp, "vendor/") { + imp = "/" + imp + } + if i := strings.LastIndex(imp, "/vendor/"); i != -1 { + imp = imp[i+len("/vendor/"):] + } + return imp +} + +// ErrorInterface represent built-in error interface. +var ErrorInterface = Interface{ + Name: "error", + Methods: []*Method{ + { + Name: "Error", + Out: []*Parameter{ + { + Name: "", + Type: PredeclaredType("string"), + }, + }, + }, + }, +} diff --git a/cluster-autoscaler/vendor/modules.txt b/cluster-autoscaler/vendor/modules.txt index 4c6e6bbeebc8..6e86206d1c78 100644 --- a/cluster-autoscaler/vendor/modules.txt +++ b/cluster-autoscaler/vendor/modules.txt @@ -265,7 +265,14 @@ github.com/golang/groupcache/lru # github.com/golang/mock v1.4.1 => github.com/golang/mock v1.4.1 ## explicit github.com/golang/mock/gomock +<<<<<<< HEAD # github.com/golang/protobuf v1.4.3 => github.com/golang/protobuf v1.4.3 +======= +github.com/golang/mock/mockgen/model +# github.com/golang/protobuf v1.5.2 +github.com/golang/protobuf/descriptor +github.com/golang/protobuf/jsonpb +>>>>>>> 0123869b7 (Merge pull request #4452 from airbnb/es--grpc-expander-plugin) github.com/golang/protobuf/proto github.com/golang/protobuf/protoc-gen-go/descriptor github.com/golang/protobuf/ptypes @@ -700,7 +707,13 @@ google.golang.org/appengine/internal/urlfetch google.golang.org/appengine/urlfetch # google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a => google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a google.golang.org/genproto/googleapis/rpc/status +<<<<<<< HEAD # google.golang.org/grpc v1.27.1 => google.golang.org/grpc v1.27.1 +======= +google.golang.org/genproto/protobuf/field_mask +# google.golang.org/grpc v1.40.0 +## explicit +>>>>>>> 0123869b7 (Merge pull request #4452 from airbnb/es--grpc-expander-plugin) google.golang.org/grpc google.golang.org/grpc/attributes google.golang.org/grpc/backoff @@ -739,7 +752,13 @@ google.golang.org/grpc/serviceconfig google.golang.org/grpc/stats google.golang.org/grpc/status google.golang.org/grpc/tap +<<<<<<< HEAD # google.golang.org/protobuf v1.25.0 => google.golang.org/protobuf v1.25.0 +======= +# google.golang.org/protobuf v1.27.1 +## explicit +google.golang.org/protobuf/encoding/protojson +>>>>>>> 0123869b7 (Merge pull request #4452 from airbnb/es--grpc-expander-plugin) google.golang.org/protobuf/encoding/prototext google.golang.org/protobuf/encoding/protowire google.golang.org/protobuf/internal/descfmt diff --git a/hack/verify-golint.sh b/hack/verify-golint.sh index 8b5d50304d17..ae2593f4a115 100755 --- a/hack/verify-golint.sh +++ b/hack/verify-golint.sh @@ -30,6 +30,7 @@ excluded_packages=( 'cluster-autoscaler/cloudprovider/exoscale/internal' 'cluster-autoscaler/cloudprovider/huaweicloud/huaweicloud-sdk-go-v3' 'cluster-autoscaler/cloudprovider/ionoscloud/ionos-cloud-sdk-go' + 'cluster-autoscaler/expander/grpcplugin/protos' ) FIND_PACKAGES='go list ./... '