diff --git a/cluster-autoscaler/cloudprovider/azure/azure_scale_set.go b/cluster-autoscaler/cloudprovider/azure/azure_scale_set.go index 95166c1c6518..39e77e407e37 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_scale_set.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_scale_set.go @@ -20,20 +20,14 @@ import ( "fmt" "math/rand" "net/http" - "regexp" "strings" "sync" "time" apiv1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/autoscaler/cluster-autoscaler/config/dynamic" - "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" - cloudvolume "k8s.io/cloud-provider/volume" klog "k8s.io/klog/v2" - kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" schedulerframework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" "k8s.io/legacy-cloud-providers/azure/retry" @@ -517,152 +511,6 @@ func (scaleSet *ScaleSet) Debug() string { return fmt.Sprintf("%s (%d:%d)", scaleSet.Id(), scaleSet.MinSize(), scaleSet.MaxSize()) } -func buildInstanceOS(template compute.VirtualMachineScaleSet) string { - instanceOS := cloudprovider.DefaultOS - if template.VirtualMachineProfile != nil && template.VirtualMachineProfile.OsProfile != nil && template.VirtualMachineProfile.OsProfile.WindowsConfiguration != nil { - instanceOS = "windows" - } - - return instanceOS -} - -func buildGenericLabels(template compute.VirtualMachineScaleSet, nodeName string) map[string]string { - result := make(map[string]string) - - result[kubeletapis.LabelArch] = cloudprovider.DefaultArch - result[kubeletapis.LabelOS] = buildInstanceOS(template) - result[apiv1.LabelInstanceType] = *template.Sku.Name - result[apiv1.LabelZoneRegion] = strings.ToLower(*template.Location) - - if template.Zones != nil && len(*template.Zones) > 0 { - failureDomains := make([]string, len(*template.Zones)) - for k, v := range *template.Zones { - failureDomains[k] = strings.ToLower(*template.Location) + "-" + v - } - - result[apiv1.LabelZoneFailureDomain] = strings.Join(failureDomains[:], cloudvolume.LabelMultiZoneDelimiter) - } else { - result[apiv1.LabelZoneFailureDomain] = "0" - } - - result[apiv1.LabelHostname] = nodeName - return result -} - -func (scaleSet *ScaleSet) buildNodeFromTemplate(template compute.VirtualMachineScaleSet) (*apiv1.Node, error) { - node := apiv1.Node{} - nodeName := fmt.Sprintf("%s-asg-%d", scaleSet.Name, rand.Int63()) - - node.ObjectMeta = metav1.ObjectMeta{ - Name: nodeName, - SelfLink: fmt.Sprintf("/api/v1/nodes/%s", nodeName), - Labels: map[string]string{}, - } - - node.Status = apiv1.NodeStatus{ - Capacity: apiv1.ResourceList{}, - } - - var vmssType *InstanceType - for k := range InstanceTypes { - if strings.EqualFold(k, *template.Sku.Name) { - vmssType = InstanceTypes[k] - break - } - } - - promoRe := regexp.MustCompile(`(?i)_promo`) - if promoRe.MatchString(*template.Sku.Name) { - if vmssType == nil { - // We didn't find an exact match but this is a promo type, check for matching standard - klog.V(1).Infof("No exact match found for %s, checking standard types", *template.Sku.Name) - skuName := promoRe.ReplaceAllString(*template.Sku.Name, "") - for k := range InstanceTypes { - if strings.EqualFold(k, skuName) { - vmssType = InstanceTypes[k] - break - } - } - } - } - - if vmssType == nil { - return nil, fmt.Errorf("instance type %q not supported", *template.Sku.Name) - } - node.Status.Capacity[apiv1.ResourcePods] = *resource.NewQuantity(110, resource.DecimalSI) - node.Status.Capacity[apiv1.ResourceCPU] = *resource.NewQuantity(vmssType.VCPU, resource.DecimalSI) - node.Status.Capacity[gpu.ResourceNvidiaGPU] = *resource.NewQuantity(vmssType.GPU, resource.DecimalSI) - node.Status.Capacity[apiv1.ResourceMemory] = *resource.NewQuantity(vmssType.MemoryMb*1024*1024, resource.DecimalSI) - - // TODO: set real allocatable. - node.Status.Allocatable = node.Status.Capacity - - // NodeLabels - if template.Tags != nil { - for k, v := range template.Tags { - if v != nil { - node.Labels[k] = *v - } else { - node.Labels[k] = "" - } - - } - } - - // GenericLabels - node.Labels = cloudprovider.JoinStringMaps(node.Labels, buildGenericLabels(template, nodeName)) - // Labels from the Scale Set's Tags - node.Labels = cloudprovider.JoinStringMaps(node.Labels, extractLabelsFromScaleSet(template.Tags)) - - // Taints from the Scale Set's Tags - node.Spec.Taints = extractTaintsFromScaleSet(template.Tags) - - node.Status.Conditions = cloudprovider.BuildReadyConditions() - return &node, nil -} - -func extractLabelsFromScaleSet(tags map[string]*string) map[string]string { - result := make(map[string]string) - - for tagName, tagValue := range tags { - splits := strings.Split(tagName, nodeLabelTagName) - if len(splits) > 1 { - label := strings.Replace(splits[1], "_", "/", -1) - if label != "" { - result[label] = *tagValue - } - } - } - - return result -} - -func extractTaintsFromScaleSet(tags map[string]*string) []apiv1.Taint { - taints := make([]apiv1.Taint, 0) - - for tagName, tagValue := range tags { - // The tag value must be in the format :NoSchedule - r, _ := regexp.Compile("(.*):(?:NoSchedule|NoExecute|PreferNoSchedule)") - - if r.MatchString(*tagValue) { - splits := strings.Split(tagName, nodeTaintTagName) - if len(splits) > 1 { - values := strings.SplitN(*tagValue, ":", 2) - if len(values) > 1 { - taintKey := strings.Replace(splits[1], "_", "/", -1) - taints = append(taints, apiv1.Taint{ - Key: taintKey, - Value: values[0], - Effect: apiv1.TaintEffect(values[1]), - }) - } - } - } - } - - return taints -} - // TemplateNodeInfo returns a node template for this scale set. func (scaleSet *ScaleSet) TemplateNodeInfo() (*schedulerframework.NodeInfo, error) { template, rerr := scaleSet.getVMSSInfo() @@ -670,7 +518,7 @@ func (scaleSet *ScaleSet) TemplateNodeInfo() (*schedulerframework.NodeInfo, erro return nil, rerr.Error() } - node, err := scaleSet.buildNodeFromTemplate(template) + node, err := buildNodeFromTemplate(scaleSet.Name, template) if err != nil { return nil, err } diff --git a/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go b/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go index 4d0f99048878..f1e1cbdbaba0 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go @@ -470,73 +470,3 @@ func TestTemplateNodeInfo(t *testing.T) { assert.NotNil(t, nodeInfo) assert.NotEmpty(t, nodeInfo.Pods) } -func TestExtractLabelsFromScaleSet(t *testing.T) { - expectedNodeLabelKey := "zip" - expectedNodeLabelValue := "zap" - extraNodeLabelValue := "buzz" - blankString := "" - - tags := map[string]*string{ - fmt.Sprintf("%s%s", nodeLabelTagName, expectedNodeLabelKey): &expectedNodeLabelValue, - "fizz": &extraNodeLabelValue, - "bip": &blankString, - } - - labels := extractLabelsFromScaleSet(tags) - assert.Len(t, labels, 1) - assert.Equal(t, expectedNodeLabelValue, labels[expectedNodeLabelKey]) -} - -func TestExtractTaintsFromScaleSet(t *testing.T) { - noScheduleTaintValue := "foo:NoSchedule" - noExecuteTaintValue := "bar:NoExecute" - preferNoScheduleTaintValue := "fizz:PreferNoSchedule" - noSplitTaintValue := "some_value" - blankTaintValue := "" - regularTagValue := "baz" - - tags := map[string]*string{ - fmt.Sprintf("%s%s", nodeTaintTagName, "dedicated"): &noScheduleTaintValue, - fmt.Sprintf("%s%s", nodeTaintTagName, "group"): &noExecuteTaintValue, - fmt.Sprintf("%s%s", nodeTaintTagName, "app"): &preferNoScheduleTaintValue, - fmt.Sprintf("%s%s", nodeTaintTagName, "k8s.io_testing_underscore_to_slash"): &preferNoScheduleTaintValue, - "bar": ®ularTagValue, - fmt.Sprintf("%s%s", nodeTaintTagName, "blank"): &blankTaintValue, - fmt.Sprintf("%s%s", nodeTaintTagName, "nosplit"): &noSplitTaintValue, - } - - expectedTaints := []apiv1.Taint{ - { - Key: "dedicated", - Value: "foo", - Effect: apiv1.TaintEffectNoSchedule, - }, - { - Key: "group", - Value: "bar", - Effect: apiv1.TaintEffectNoExecute, - }, - { - Key: "app", - Value: "fizz", - Effect: apiv1.TaintEffectPreferNoSchedule, - }, - { - Key: "k8s.io/testing/underscore/to/slash", - Value: "fizz", - Effect: apiv1.TaintEffectPreferNoSchedule, - }, - } - - taints := extractTaintsFromScaleSet(tags) - assert.Len(t, taints, 4) - assert.Equal(t, makeTaintSet(expectedTaints), makeTaintSet(taints)) -} - -func makeTaintSet(taints []apiv1.Taint) map[apiv1.Taint]bool { - set := make(map[apiv1.Taint]bool) - for _, taint := range taints { - set[taint] = true - } - return set -} diff --git a/cluster-autoscaler/cloudprovider/azure/azure_template.go b/cluster-autoscaler/cloudprovider/azure/azure_template.go new file mode 100644 index 000000000000..1c67dc8da4f5 --- /dev/null +++ b/cluster-autoscaler/cloudprovider/azure/azure_template.go @@ -0,0 +1,183 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "fmt" + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-12-01/compute" + apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" + "k8s.io/autoscaler/cluster-autoscaler/utils/gpu" + cloudvolume "k8s.io/cloud-provider/volume" + "k8s.io/klog/v2" + kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis" + "math/rand" + "regexp" + "strings" +) + +func buildInstanceOS(template compute.VirtualMachineScaleSet) string { + instanceOS := cloudprovider.DefaultOS + if template.VirtualMachineProfile != nil && template.VirtualMachineProfile.OsProfile != nil && template.VirtualMachineProfile.OsProfile.WindowsConfiguration != nil { + instanceOS = "windows" + } + + return instanceOS +} + +func buildGenericLabels(template compute.VirtualMachineScaleSet, nodeName string) map[string]string { + result := make(map[string]string) + + result[kubeletapis.LabelArch] = cloudprovider.DefaultArch + result[apiv1.LabelArchStable] = cloudprovider.DefaultArch + + result[kubeletapis.LabelOS] = buildInstanceOS(template) + result[apiv1.LabelOSStable] = buildInstanceOS(template) + + result[apiv1.LabelInstanceType] = *template.Sku.Name + result[apiv1.LabelZoneRegion] = strings.ToLower(*template.Location) + + if template.Zones != nil && len(*template.Zones) > 0 { + failureDomains := make([]string, len(*template.Zones)) + for k, v := range *template.Zones { + failureDomains[k] = strings.ToLower(*template.Location) + "-" + v + } + + result[apiv1.LabelZoneFailureDomain] = strings.Join(failureDomains[:], cloudvolume.LabelMultiZoneDelimiter) + } else { + result[apiv1.LabelZoneFailureDomain] = "0" + } + + result[apiv1.LabelHostname] = nodeName + return result +} + +func buildNodeFromTemplate(scaleSetName string, template compute.VirtualMachineScaleSet) (*apiv1.Node, error) { + node := apiv1.Node{} + nodeName := fmt.Sprintf("%s-asg-%d", scaleSetName, rand.Int63()) + + node.ObjectMeta = metav1.ObjectMeta{ + Name: nodeName, + SelfLink: fmt.Sprintf("/api/v1/nodes/%s", nodeName), + Labels: map[string]string{}, + } + + node.Status = apiv1.NodeStatus{ + Capacity: apiv1.ResourceList{}, + } + + var vmssType *InstanceType + for k := range InstanceTypes { + if strings.EqualFold(k, *template.Sku.Name) { + vmssType = InstanceTypes[k] + break + } + } + + promoRe := regexp.MustCompile(`(?i)_promo`) + if promoRe.MatchString(*template.Sku.Name) { + if vmssType == nil { + // We didn't find an exact match but this is a promo type, check for matching standard + klog.V(1).Infof("No exact match found for %s, checking standard types", *template.Sku.Name) + skuName := promoRe.ReplaceAllString(*template.Sku.Name, "") + for k := range InstanceTypes { + if strings.EqualFold(k, skuName) { + vmssType = InstanceTypes[k] + break + } + } + } + } + + if vmssType == nil { + return nil, fmt.Errorf("instance type %q not supported", *template.Sku.Name) + } + node.Status.Capacity[apiv1.ResourcePods] = *resource.NewQuantity(110, resource.DecimalSI) + node.Status.Capacity[apiv1.ResourceCPU] = *resource.NewQuantity(vmssType.VCPU, resource.DecimalSI) + node.Status.Capacity[gpu.ResourceNvidiaGPU] = *resource.NewQuantity(vmssType.GPU, resource.DecimalSI) + node.Status.Capacity[apiv1.ResourceMemory] = *resource.NewQuantity(vmssType.MemoryMb*1024*1024, resource.DecimalSI) + + // TODO: set real allocatable. + node.Status.Allocatable = node.Status.Capacity + + // NodeLabels + if template.Tags != nil { + for k, v := range template.Tags { + if v != nil { + node.Labels[k] = *v + } else { + node.Labels[k] = "" + } + + } + } + + // GenericLabels + node.Labels = cloudprovider.JoinStringMaps(node.Labels, buildGenericLabels(template, nodeName)) + // Labels from the Scale Set's Tags + node.Labels = cloudprovider.JoinStringMaps(node.Labels, extractLabelsFromScaleSet(template.Tags)) + + // Taints from the Scale Set's Tags + node.Spec.Taints = extractTaintsFromScaleSet(template.Tags) + + node.Status.Conditions = cloudprovider.BuildReadyConditions() + return &node, nil +} + +func extractLabelsFromScaleSet(tags map[string]*string) map[string]string { + result := make(map[string]string) + + for tagName, tagValue := range tags { + splits := strings.Split(tagName, nodeLabelTagName) + if len(splits) > 1 { + label := strings.Replace(splits[1], "_", "/", -1) + if label != "" { + result[label] = *tagValue + } + } + } + + return result +} + +func extractTaintsFromScaleSet(tags map[string]*string) []apiv1.Taint { + taints := make([]apiv1.Taint, 0) + + for tagName, tagValue := range tags { + // The tag value must be in the format :NoSchedule + r, _ := regexp.Compile("(.*):(?:NoSchedule|NoExecute|PreferNoSchedule)") + + if r.MatchString(*tagValue) { + splits := strings.Split(tagName, nodeTaintTagName) + if len(splits) > 1 { + values := strings.SplitN(*tagValue, ":", 2) + if len(values) > 1 { + taintKey := strings.Replace(splits[1], "_", "/", -1) + taints = append(taints, apiv1.Taint{ + Key: taintKey, + Value: values[0], + Effect: apiv1.TaintEffect(values[1]), + }) + } + } + } + } + + return taints +} diff --git a/cluster-autoscaler/cloudprovider/azure/azure_template_test.go b/cluster-autoscaler/cloudprovider/azure/azure_template_test.go new file mode 100644 index 000000000000..b0179288cb0e --- /dev/null +++ b/cluster-autoscaler/cloudprovider/azure/azure_template_test.go @@ -0,0 +1,95 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package azure + +import ( + "fmt" + "github.com/stretchr/testify/assert" + apiv1 "k8s.io/api/core/v1" + "testing" +) + +func TestExtractLabelsFromScaleSet(t *testing.T) { + expectedNodeLabelKey := "zip" + expectedNodeLabelValue := "zap" + extraNodeLabelValue := "buzz" + blankString := "" + + tags := map[string]*string{ + fmt.Sprintf("%s%s", nodeLabelTagName, expectedNodeLabelKey): &expectedNodeLabelValue, + "fizz": &extraNodeLabelValue, + "bip": &blankString, + } + + labels := extractLabelsFromScaleSet(tags) + assert.Len(t, labels, 1) + assert.Equal(t, expectedNodeLabelValue, labels[expectedNodeLabelKey]) +} + +func TestExtractTaintsFromScaleSet(t *testing.T) { + noScheduleTaintValue := "foo:NoSchedule" + noExecuteTaintValue := "bar:NoExecute" + preferNoScheduleTaintValue := "fizz:PreferNoSchedule" + noSplitTaintValue := "some_value" + blankTaintValue := "" + regularTagValue := "baz" + + tags := map[string]*string{ + fmt.Sprintf("%s%s", nodeTaintTagName, "dedicated"): &noScheduleTaintValue, + fmt.Sprintf("%s%s", nodeTaintTagName, "group"): &noExecuteTaintValue, + fmt.Sprintf("%s%s", nodeTaintTagName, "app"): &preferNoScheduleTaintValue, + fmt.Sprintf("%s%s", nodeTaintTagName, "k8s.io_testing_underscore_to_slash"): &preferNoScheduleTaintValue, + "bar": ®ularTagValue, + fmt.Sprintf("%s%s", nodeTaintTagName, "blank"): &blankTaintValue, + fmt.Sprintf("%s%s", nodeTaintTagName, "nosplit"): &noSplitTaintValue, + } + + expectedTaints := []apiv1.Taint{ + { + Key: "dedicated", + Value: "foo", + Effect: apiv1.TaintEffectNoSchedule, + }, + { + Key: "group", + Value: "bar", + Effect: apiv1.TaintEffectNoExecute, + }, + { + Key: "app", + Value: "fizz", + Effect: apiv1.TaintEffectPreferNoSchedule, + }, + { + Key: "k8s.io/testing/underscore/to/slash", + Value: "fizz", + Effect: apiv1.TaintEffectPreferNoSchedule, + }, + } + + taints := extractTaintsFromScaleSet(tags) + assert.Len(t, taints, 4) + assert.Equal(t, makeTaintSet(expectedTaints), makeTaintSet(taints)) +} + +func makeTaintSet(taints []apiv1.Taint) map[apiv1.Taint]bool { + set := make(map[apiv1.Taint]bool) + for _, taint := range taints { + set[taint] = true + } + return set +}