diff --git a/cluster-autoscaler/cloudprovider/aws/aws_manager.go b/cluster-autoscaler/cloudprovider/aws/aws_manager.go index c8af57f30a55..b35bd2d698df 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_manager.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_manager.go @@ -410,6 +410,7 @@ func (m *AwsManager) buildNodeFromTemplate(asg *asg, template *asgTemplate) (*ap } resourcesFromTags := extractAllocatableResourcesFromAsg(template.Tags) + klog.V(5).Infof("Extracted resources from ASG tags %v", resourcesFromTags) for resourceName, val := range resourcesFromTags { node.Status.Capacity[apiv1.ResourceName(resourceName)] = *val } diff --git a/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go b/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go index a5b41780d9f3..a7ae8aac708c 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go @@ -113,6 +113,10 @@ func TestExtractAllocatableResourcesFromAsg(t *testing.T) { Key: aws.String("k8s.io/cluster-autoscaler/node-template/resources/ephemeral-storage"), Value: aws.String("20G"), }, + { + Key: aws.String("k8s.io/cluster-autoscaler/node-template/resources/custom-resource"), + Value: aws.String("5"), + }, } labels := extractAllocatableResourcesFromAsg(tags) @@ -122,6 +126,7 @@ func TestExtractAllocatableResourcesFromAsg(t *testing.T) { assert.Equal(t, (&expectedMemory).String(), labels["memory"].String()) expectedEphemeralStorage := resource.MustParse("20G") assert.Equal(t, (&expectedEphemeralStorage).String(), labels["ephemeral-storage"].String()) + assert.Equal(t, resource.NewQuantity(5, resource.DecimalSI).String(), labels["custom-resource"].String()) } func TestGetAsgOptions(t *testing.T) { @@ -390,6 +395,8 @@ func TestBuildNodeFromTemplate(t *testing.T) { // Node with custom resource ephemeralStorageKey := "ephemeral-storage" ephemeralStorageValue := int64(20) + customResourceKey := "custom-resource" + customResourceValue := int64(5) vpcIPKey := "vpc.amazonaws.com/PrivateIPv4Address" observedNode, observedErr := awsManager.buildNodeFromTemplate(asg, &asgTemplate{ InstanceType: c5Instance, @@ -398,12 +405,19 @@ func TestBuildNodeFromTemplate(t *testing.T) { Key: aws.String(fmt.Sprintf("k8s.io/cluster-autoscaler/node-template/resources/%s", ephemeralStorageKey)), Value: aws.String(strconv.FormatInt(ephemeralStorageValue, 10)), }, + { + Key: aws.String(fmt.Sprintf("k8s.io/cluster-autoscaler/node-template/resources/%s", customResourceKey)), + Value: aws.String(strconv.FormatInt(customResourceValue, 10)), + }, }, }) assert.NoError(t, observedErr) esValue, esExist := observedNode.Status.Capacity[apiv1.ResourceName(ephemeralStorageKey)] assert.True(t, esExist) assert.Equal(t, int64(20), esValue.Value()) + crValue, crExist := observedNode.Status.Capacity[apiv1.ResourceName(customResourceKey)] + assert.True(t, crExist) + assert.Equal(t, int64(5), crValue.Value()) _, ipExist := observedNode.Status.Capacity[apiv1.ResourceName(vpcIPKey)] assert.False(t, ipExist) diff --git a/cluster-autoscaler/main.go b/cluster-autoscaler/main.go index 7d0e7c3d56c4..9ae3c3ee6d3d 100644 --- a/cluster-autoscaler/main.go +++ b/cluster-autoscaler/main.go @@ -367,6 +367,7 @@ func buildAutoscaler(debuggingSnapshotter debuggingsnapshot.DebuggingSnapshotter nodeInfoComparatorBuilder = nodegroupset.CreateAzureNodeInfoComparator } else if autoscalingOptions.CloudProviderName == cloudprovider.AwsProviderName { nodeInfoComparatorBuilder = nodegroupset.CreateAwsNodeInfoComparator + opts.Processors.TemplateNodeInfoProvider = nodeinfosprovider.NewAsgTagResourceNodeInfoProvider(nodeInfoCacheExpireTime) } else if autoscalingOptions.CloudProviderName == cloudprovider.GceProviderName { nodeInfoComparatorBuilder = nodegroupset.CreateGceNodeInfoComparator opts.Processors.TemplateNodeInfoProvider = nodeinfosprovider.NewAnnotationNodeInfoProvider(nodeInfoCacheExpireTime) diff --git a/cluster-autoscaler/processors/nodeinfosprovider/asg_tag_resource_node_info_provider.go b/cluster-autoscaler/processors/nodeinfosprovider/asg_tag_resource_node_info_provider.go new file mode 100644 index 000000000000..e4d746a47dc2 --- /dev/null +++ b/cluster-autoscaler/processors/nodeinfosprovider/asg_tag_resource_node_info_provider.go @@ -0,0 +1,73 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodeinfosprovider + +import ( + "time" + + appsv1 "k8s.io/api/apps/v1" + apiv1 "k8s.io/api/core/v1" + "k8s.io/autoscaler/cluster-autoscaler/context" + "k8s.io/autoscaler/cluster-autoscaler/utils/errors" + "k8s.io/autoscaler/cluster-autoscaler/utils/taints" + schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework" +) + +// AsgTagResourceNodeInfoProvider is a wrapper for MixedTemplateNodeInfoProvider. +type AsgTagResourceNodeInfoProvider struct { + mixedTemplateNodeInfoProvider *MixedTemplateNodeInfoProvider +} + +// NewAsgTagResourceNodeInfoProvider returns AsgTagResourceNodeInfoProvider. +func NewAsgTagResourceNodeInfoProvider(t *time.Duration) *AsgTagResourceNodeInfoProvider { + return &AsgTagResourceNodeInfoProvider{ + mixedTemplateNodeInfoProvider: NewMixedTemplateNodeInfoProvider(t), + } +} + +// Process returns the nodeInfos set for this cluster. +func (p *AsgTagResourceNodeInfoProvider) Process(ctx *context.AutoscalingContext, nodes []*apiv1.Node, daemonsets []*appsv1.DaemonSet, ignoredTaints taints.TaintKeySet, currentTime time.Time) (map[string]*schedulerframework.NodeInfo, errors.AutoscalerError) { + nodeInfos, err := p.mixedTemplateNodeInfoProvider.Process(ctx, nodes, daemonsets, ignoredTaints, currentTime) + if err != nil { + return nil, err + } + // Add annotatios to the NodeInfo to use later in expander. + nodeGroups := ctx.CloudProvider.NodeGroups() + for _, ng := range nodeGroups { + if nodeInfo, ok := nodeInfos[ng.Id()]; ok { + template, err := ng.TemplateNodeInfo() + if err != nil { + continue + } + for resourceName, val := range template.Node().Status.Capacity { + if _, ok := nodeInfo.Node().Status.Capacity[resourceName]; !ok { + nodeInfo.Node().Status.Capacity[resourceName] = val + } + } + for resourceName, val := range template.Node().Status.Allocatable { + if _, ok := nodeInfo.Node().Status.Allocatable[resourceName]; !ok { + nodeInfo.Node().Status.Allocatable[resourceName] = val + } + } + } + } + return nodeInfos, nil +} + +// CleanUp cleans up processor's internal structures. +func (p *AsgTagResourceNodeInfoProvider) CleanUp() { +}