From 42c079781a36a14a46f065e84853c9dee5eb1ea0 Mon Sep 17 00:00:00 2001 From: Jiaxin Shan Date: Mon, 12 Aug 2019 00:50:19 -0700 Subject: [PATCH] Load AWS EC2 Instance Types dynamically --- .../cloudprovider/aws/aws_cloud_provider.go | 15 +- .../aws/aws_cloud_provider_test.go | 4 +- .../cloudprovider/aws/aws_manager.go | 2 +- .../cloudprovider/aws/aws_manager_test.go | 2 +- .../cloudprovider/aws/aws_util.go | 131 ++++++++++++++++++ .../cloudprovider/aws/aws_util_test.go | 53 +++++++ .../cloudprovider/aws/ec2_instance_types.go | 5 +- .../aws/ec2_instance_types/gen.go | 115 ++------------- 8 files changed, 213 insertions(+), 114 deletions(-) create mode 100644 cluster-autoscaler/cloudprovider/aws/aws_util.go create mode 100644 cluster-autoscaler/cloudprovider/aws/aws_util_test.go diff --git a/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go b/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go index f524c537b438..36118f8839de 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider.go @@ -49,13 +49,16 @@ var ( type awsCloudProvider struct { awsManager *AwsManager resourceLimiter *cloudprovider.ResourceLimiter + // InstanceTypes is a map of ec2 resources + instanceTypes map[string]*InstanceType } // BuildAwsCloudProvider builds CloudProvider implementation for AWS. -func BuildAwsCloudProvider(awsManager *AwsManager, resourceLimiter *cloudprovider.ResourceLimiter) (cloudprovider.CloudProvider, error) { +func BuildAwsCloudProvider(awsManager *AwsManager, instanceTypes map[string]*InstanceType, resourceLimiter *cloudprovider.ResourceLimiter) (cloudprovider.CloudProvider, error) { aws := &awsCloudProvider{ awsManager: awsManager, resourceLimiter: resourceLimiter, + instanceTypes: instanceTypes, } return aws, nil } @@ -348,7 +351,15 @@ func BuildAWS(opts config.AutoscalingOptions, do cloudprovider.NodeGroupDiscover klog.Fatalf("Failed to create AWS Manager: %v", err) } - provider, err := BuildAwsCloudProvider(manager, rl) + // Generate EC2 list + instanceTypes, err := GenerateEC2InstanceTypes() + if err != nil { + klog.Warningf("Failed to generate AWS EC2 Instance Types: %v", err) + klog.Warning("Use static EC2 Instance Types, list could be outdated") + instanceTypes = GetStaticEC2InstanceTypes() + } + + provider, err := BuildAwsCloudProvider(manager, instanceTypes, rl) if err != nil { klog.Fatalf("Failed to create AWS cloud provider: %v", err) } diff --git a/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go b/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go index 2f880c9a05c1..b99cd3b173da 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_cloud_provider_test.go @@ -133,7 +133,7 @@ func testProvider(t *testing.T, m *AwsManager) *awsCloudProvider { map[string]int64{cloudprovider.ResourceNameCores: 1, cloudprovider.ResourceNameMemory: 10000000}, map[string]int64{cloudprovider.ResourceNameCores: 10, cloudprovider.ResourceNameMemory: 100000000}) - provider, err := BuildAwsCloudProvider(m, resourceLimiter) + provider, err := BuildAwsCloudProvider(m, GetStaticEC2InstanceTypes(), resourceLimiter) assert.NoError(t, err) return provider.(*awsCloudProvider) } @@ -143,7 +143,7 @@ func TestBuildAwsCloudProvider(t *testing.T) { map[string]int64{cloudprovider.ResourceNameCores: 1, cloudprovider.ResourceNameMemory: 10000000}, map[string]int64{cloudprovider.ResourceNameCores: 10, cloudprovider.ResourceNameMemory: 100000000}) - _, err := BuildAwsCloudProvider(testAwsManager, resourceLimiter) + _, err := BuildAwsCloudProvider(testAwsManager, GetStaticEC2InstanceTypes(), resourceLimiter) assert.NoError(t, err) } diff --git a/cluster-autoscaler/cloudprovider/aws/aws_manager.go b/cluster-autoscaler/cloudprovider/aws/aws_manager.go index 911ad03bc3ee..afbbcc78f3d7 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_manager.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_manager.go @@ -62,7 +62,7 @@ type AwsManager struct { } type asgTemplate struct { - InstanceType *instanceType + InstanceType *InstanceType Region string Zone string Tags []*autoscaling.TagDescription diff --git a/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go b/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go index 37bcb4132cdb..b6cf0a5b8a03 100644 --- a/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go +++ b/cluster-autoscaler/cloudprovider/aws/aws_manager_test.go @@ -70,7 +70,7 @@ func TestGetRegion(t *testing.T) { func TestBuildGenericLabels(t *testing.T) { labels := buildGenericLabels(&asgTemplate{ - InstanceType: &instanceType{ + InstanceType: &InstanceType{ InstanceType: "c4.large", VCPU: 2, MemoryMb: 3840, diff --git a/cluster-autoscaler/cloudprovider/aws/aws_util.go b/cluster-autoscaler/cloudprovider/aws/aws_util.go new file mode 100644 index 000000000000..5cc27b4814b7 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/aws/aws_util.go @@ -0,0 +1,131 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "encoding/json" + "errors" + "github.com/aws/aws-sdk-go/aws/endpoints" + "io/ioutil" + "k8s.io/klog" + "net/http" + "regexp" + "strconv" + "strings" +) + +type response struct { + Products map[string]product `json:"products"` +} + +type product struct { + Attributes productAttributes `json:"attributes"` +} + +type productAttributes struct { + InstanceType string `json:"instanceType"` + VCPU string `json:"vcpu"` + Memory string `json:"memory"` + GPU string `json:"gpu"` +} + +// GenerateEC2InstanceTypes returns a map of ec2 resources +func GenerateEC2InstanceTypes() (map[string]*InstanceType, error) { + instanceTypes := make(map[string]*InstanceType) + + resolver := endpoints.DefaultResolver() + partitions := resolver.(endpoints.EnumPartitions).Partitions() + + for _, p := range partitions { + for _, r := range p.Regions() { + url := "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/" + r.ID() + "/index.json" + klog.V(1).Infof("fetching %s\n", url) + res, err := http.Get(url) + if err != nil { + klog.Warningf("Error fetching %s skipping...\n", url) + continue + } + + defer res.Body.Close() + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + klog.Warningf("Error parsing %s skipping...\n", url) + continue + } + + var unmarshalled = response{} + err = json.Unmarshal(body, &unmarshalled) + if err != nil { + klog.Warningf("Error unmarshalling %s, skip...\n", url) + continue + } + + for _, product := range unmarshalled.Products { + attr := product.Attributes + if attr.InstanceType != "" { + instanceTypes[attr.InstanceType] = &InstanceType{ + InstanceType: attr.InstanceType, + } + if attr.Memory != "" && attr.Memory != "NA" { + instanceTypes[attr.InstanceType].MemoryMb = parseMemory(attr.Memory) + } + if attr.VCPU != "" { + instanceTypes[attr.InstanceType].VCPU = parseCPU(attr.VCPU) + } + if attr.GPU != "" { + instanceTypes[attr.InstanceType].GPU = parseCPU(attr.GPU) + } + } + } + } + } + + if len(instanceTypes) == 0 { + return nil, errors.New("unable to load EC2 Instance Type list") + } + + return instanceTypes, nil +} + +// GetStaticEC2InstanceTypes return pregenerated ec2 instance type list +func GetStaticEC2InstanceTypes() map[string]*InstanceType { + return InstanceTypes +} + +func parseMemory(memory string) int64 { + reg, err := regexp.Compile("[^0-9\\.]+") + if err != nil { + klog.Fatal(err) + } + + parsed := strings.TrimSpace(reg.ReplaceAllString(memory, "")) + mem, err := strconv.ParseFloat(parsed, 64) + if err != nil { + klog.Fatal(err) + } + + return int64(mem * float64(1024)) +} + +func parseCPU(cpu string) int64 { + i, err := strconv.ParseInt(cpu, 10, 64) + if err != nil { + klog.Fatal(err) + } + return i +} diff --git a/cluster-autoscaler/cloudprovider/aws/aws_util_test.go b/cluster-autoscaler/cloudprovider/aws/aws_util_test.go new file mode 100644 index 000000000000..80f086213de7 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/aws/aws_util_test.go @@ -0,0 +1,53 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package aws + +import ( + "github.com/stretchr/testify/assert" + "strconv" + "testing" +) + +func TestGetStaticEC2InstanceTypes(t *testing.T) { + result := GetStaticEC2InstanceTypes() + assert.True(t, len(result) != 0) +} + +func TestParseMemory(t *testing.T) { + memoryInGiB := 3.75 + expectedResultInMiB := int64(memoryInGiB * 1024) + result := parseMemory("3.75 GiB") + assert.Equal(t, expectedResultInMiB, result) + + result = parseMemory("3.75 Gib") + assert.Equal(t, expectedResultInMiB, result) + + result = parseMemory("3.75 Gib") + assert.Equal(t, expectedResultInMiB, result) + + result = parseMemory("3.75GiB") + assert.Equal(t, expectedResultInMiB, result) + + result = parseMemory("3.75") + assert.Equal(t, expectedResultInMiB, result) +} + +func TestParseCPU(t *testing.T) { + cpu := int64(8) + result := parseCPU(strconv.FormatInt(cpu, 10)) + assert.Equal(t, cpu, result) +} diff --git a/cluster-autoscaler/cloudprovider/aws/ec2_instance_types.go b/cluster-autoscaler/cloudprovider/aws/ec2_instance_types.go index 2f317322481a..bd07c0a1a55b 100644 --- a/cluster-autoscaler/cloudprovider/aws/ec2_instance_types.go +++ b/cluster-autoscaler/cloudprovider/aws/ec2_instance_types.go @@ -18,7 +18,8 @@ limitations under the License. package aws -type instanceType struct { +// InstanceType represents ec2 resource +type InstanceType struct { InstanceType string VCPU int64 MemoryMb int64 @@ -26,7 +27,7 @@ type instanceType struct { } // InstanceTypes is a map of ec2 resources -var InstanceTypes = map[string]*instanceType{ +var InstanceTypes = map[string]*InstanceType{ "a1": { InstanceType: "a1", VCPU: 16, diff --git a/cluster-autoscaler/cloudprovider/aws/ec2_instance_types/gen.go b/cluster-autoscaler/cloudprovider/aws/ec2_instance_types/gen.go index e6861a2f5ad2..93e2c2001f74 100644 --- a/cluster-autoscaler/cloudprovider/aws/ec2_instance_types/gen.go +++ b/cluster-autoscaler/cloudprovider/aws/ec2_instance_types/gen.go @@ -19,42 +19,13 @@ limitations under the License. package main import ( - "encoding/json" "flag" "html/template" - "io/ioutil" - "net/http" - "os" - "regexp" - "strconv" - "strings" - - "github.com/aws/aws-sdk-go/aws/endpoints" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider/aws" "k8s.io/klog" + "os" ) -type response struct { - Products map[string]product `json:"products"` -} - -type product struct { - Attributes productAttributes `json:"attributes"` -} - -type productAttributes struct { - InstanceType string `json:"instanceType"` - VCPU string `json:"vcpu"` - Memory string `json:"memory"` - GPU string `json:"gpu"` -} - -type instanceType struct { - InstanceType string - VCPU int64 - Memory int64 - GPU int64 -} - var packageTemplate = template.Must(template.New("").Parse(`/* Copyright The Kubernetes Authors. @@ -75,7 +46,7 @@ limitations under the License. package aws -type instanceType struct { +type InstanceType struct { InstanceType string VCPU int64 MemoryMb int64 @@ -83,12 +54,12 @@ type instanceType struct { } // InstanceTypes is a map of ec2 resources -var InstanceTypes = map[string]*instanceType{ +var InstanceTypes = map[string]*InstanceType{ {{- range .InstanceTypes }} "{{ .InstanceType }}": { InstanceType: "{{ .InstanceType }}", VCPU: {{ .VCPU }}, - MemoryMb: {{ .Memory }}, + MemoryMb: {{ .MemoryMb }}, GPU: {{ .GPU }}, }, {{- end }} @@ -99,54 +70,9 @@ func main() { flag.Parse() defer klog.Flush() - instanceTypes := make(map[string]*instanceType) - - resolver := endpoints.DefaultResolver() - partitions := resolver.(endpoints.EnumPartitions).Partitions() - - for _, p := range partitions { - for _, r := range p.Regions() { - url := "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/" + r.ID() + "/index.json" - klog.V(1).Infof("fetching %s\n", url) - res, err := http.Get(url) - if err != nil { - klog.Warningf("Error fetching %s skipping...\n", url) - continue - } - - defer res.Body.Close() - - body, err := ioutil.ReadAll(res.Body) - if err != nil { - klog.Warningf("Error parsing %s skipping...\n", url) - continue - } - - var unmarshalled = response{} - err = json.Unmarshal(body, &unmarshalled) - if err != nil { - klog.Warningf("Error unmarshalling %s skipping...\n", url) - continue - } - - for _, product := range unmarshalled.Products { - attr := product.Attributes - if attr.InstanceType != "" { - instanceTypes[attr.InstanceType] = &instanceType{ - InstanceType: attr.InstanceType, - } - if attr.Memory != "" && attr.Memory != "NA" { - instanceTypes[attr.InstanceType].Memory = parseMemory(attr.Memory) - } - if attr.VCPU != "" { - instanceTypes[attr.InstanceType].VCPU = parseCPU(attr.VCPU) - } - if attr.GPU != "" { - instanceTypes[attr.InstanceType].GPU = parseCPU(attr.GPU) - } - } - } - } + instanceTypes, err := aws.GenerateEC2InstanceTypes() + if err != nil { + klog.Fatal(err) } f, err := os.Create("ec2_instance_types.go") @@ -157,7 +83,7 @@ func main() { defer f.Close() err = packageTemplate.Execute(f, struct { - InstanceTypes map[string]*instanceType + InstanceTypes map[string]*aws.InstanceType }{ InstanceTypes: instanceTypes, }) @@ -166,26 +92,3 @@ func main() { klog.Fatal(err) } } - -func parseMemory(memory string) int64 { - reg, err := regexp.Compile("[^0-9\\.]+") - if err != nil { - klog.Fatal(err) - } - - parsed := strings.TrimSpace(reg.ReplaceAllString(memory, "")) - mem, err := strconv.ParseFloat(parsed, 64) - if err != nil { - klog.Fatal(err) - } - - return int64(mem * float64(1024)) -} - -func parseCPU(cpu string) int64 { - i, err := strconv.ParseInt(cpu, 10, 64) - if err != nil { - klog.Fatal(err) - } - return i -}