Skip to content

Commit

Permalink
Merge pull request #3562 from marwanad/cherry-pick-3558-1.16
Browse files Browse the repository at this point in the history 
Cherry pick #3558 onto 1.16 - Add missing stable labels in the azure template
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot authored Sep 30, 2020
2 parents ab02814 + b7ad334 commit 49f25f2
Show file tree
Hide file tree
Showing 4 changed files with 280 additions and 225 deletions.
155 changes: 1 addition & 154 deletions cluster-autoscaler/cloudprovider/azure/azure_scale_set.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,15 @@ package azure

import (
"fmt"
"math/rand"
"net/http"
"regexp"
"strings"
"sync"
"time"

apiv1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
"k8s.io/autoscaler/cluster-autoscaler/config/dynamic"
"k8s.io/autoscaler/cluster-autoscaler/utils/gpu"
cloudvolume "k8s.io/cloud-provider/volume"
"k8s.io/klog"
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
schedulernodeinfo "k8s.io/kubernetes/pkg/scheduler/nodeinfo"

"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
Expand Down Expand Up @@ -506,160 +499,14 @@ func (scaleSet *ScaleSet) Debug() string {
return fmt.Sprintf("%s (%d:%d)", scaleSet.Id(), scaleSet.MinSize(), scaleSet.MaxSize())
}

func buildInstanceOS(template compute.VirtualMachineScaleSet) string {
instanceOS := cloudprovider.DefaultOS
if template.VirtualMachineProfile != nil && template.VirtualMachineProfile.OsProfile != nil && template.VirtualMachineProfile.OsProfile.WindowsConfiguration != nil {
instanceOS = "windows"
}

return instanceOS
}

func buildGenericLabels(template compute.VirtualMachineScaleSet, nodeName string) map[string]string {
result := make(map[string]string)

result[kubeletapis.LabelArch] = cloudprovider.DefaultArch
result[kubeletapis.LabelOS] = buildInstanceOS(template)
result[apiv1.LabelInstanceType] = *template.Sku.Name
result[apiv1.LabelZoneRegion] = strings.ToLower(*template.Location)

if template.Zones != nil && len(*template.Zones) > 0 {
failureDomains := make([]string, len(*template.Zones))
for k, v := range *template.Zones {
failureDomains[k] = strings.ToLower(*template.Location) + "-" + v
}

result[apiv1.LabelZoneFailureDomain] = strings.Join(failureDomains[:], cloudvolume.LabelMultiZoneDelimiter)
} else {
result[apiv1.LabelZoneFailureDomain] = "0"
}

result[apiv1.LabelHostname] = nodeName
return result
}

func (scaleSet *ScaleSet) buildNodeFromTemplate(template compute.VirtualMachineScaleSet) (*apiv1.Node, error) {
node := apiv1.Node{}
nodeName := fmt.Sprintf("%s-asg-%d", scaleSet.Name, rand.Int63())

node.ObjectMeta = metav1.ObjectMeta{
Name: nodeName,
SelfLink: fmt.Sprintf("/api/v1/nodes/%s", nodeName),
Labels: map[string]string{},
}

node.Status = apiv1.NodeStatus{
Capacity: apiv1.ResourceList{},
}

var vmssType *InstanceType
for k := range InstanceTypes {
if strings.EqualFold(k, *template.Sku.Name) {
vmssType = InstanceTypes[k]
break
}
}

promoRe := regexp.MustCompile(`(?i)_promo`)
if promoRe.MatchString(*template.Sku.Name) {
if vmssType == nil {
// We didn't find an exact match but this is a promo type, check for matching standard
klog.V(1).Infof("No exact match found for %s, checking standard types", *template.Sku.Name)
skuName := promoRe.ReplaceAllString(*template.Sku.Name, "")
for k := range InstanceTypes {
if strings.EqualFold(k, skuName) {
vmssType = InstanceTypes[k]
break
}
}
}
}

if vmssType == nil {
return nil, fmt.Errorf("instance type %q not supported", *template.Sku.Name)
}
node.Status.Capacity[apiv1.ResourcePods] = *resource.NewQuantity(110, resource.DecimalSI)
node.Status.Capacity[apiv1.ResourceCPU] = *resource.NewQuantity(vmssType.VCPU, resource.DecimalSI)
node.Status.Capacity[gpu.ResourceNvidiaGPU] = *resource.NewQuantity(vmssType.GPU, resource.DecimalSI)
node.Status.Capacity[apiv1.ResourceMemory] = *resource.NewQuantity(vmssType.MemoryMb*1024*1024, resource.DecimalSI)

// TODO: set real allocatable.
node.Status.Allocatable = node.Status.Capacity

// NodeLabels
if template.Tags != nil {
for k, v := range template.Tags {
if v != nil {
node.Labels[k] = *v
} else {
node.Labels[k] = ""
}

}
}

// GenericLabels
node.Labels = cloudprovider.JoinStringMaps(node.Labels, buildGenericLabels(template, nodeName))
// Labels from the Scale Set's Tags
node.Labels = cloudprovider.JoinStringMaps(node.Labels, extractLabelsFromScaleSet(template.Tags))

// Taints from the Scale Set's Tags
node.Spec.Taints = extractTaintsFromScaleSet(template.Tags)

node.Status.Conditions = cloudprovider.BuildReadyConditions()
return &node, nil
}

func extractLabelsFromScaleSet(tags map[string]*string) map[string]string {
result := make(map[string]string)

for tagName, tagValue := range tags {
splits := strings.Split(tagName, nodeLabelTagName)
if len(splits) > 1 {
label := strings.Replace(splits[1], "_", "/", -1)
if label != "" {
result[label] = *tagValue
}
}
}

return result
}

func extractTaintsFromScaleSet(tags map[string]*string) []apiv1.Taint {
taints := make([]apiv1.Taint, 0)

for tagName, tagValue := range tags {
// The tag value must be in the format <tag>:NoSchedule
r, _ := regexp.Compile("(.*):(?:NoSchedule|NoExecute|PreferNoSchedule)")

if r.MatchString(*tagValue) {
splits := strings.Split(tagName, nodeTaintTagName)
if len(splits) > 1 {
values := strings.SplitN(*tagValue, ":", 2)
if len(values) > 1 {
taintKey := strings.Replace(splits[1], "_", "/", -1)
taints = append(taints, apiv1.Taint{
Key: taintKey,
Value: values[0],
Effect: apiv1.TaintEffect(values[1]),
})
}
}
}
}

return taints
}

// TemplateNodeInfo returns a node template for this scale set.
func (scaleSet *ScaleSet) TemplateNodeInfo() (*schedulernodeinfo.NodeInfo, error) {
template, err := scaleSet.getVMSSInfo()
if err != nil {
return nil, err
}

node, err := scaleSet.buildNodeFromTemplate(template)
node, err := buildNodeFromTemplate(scaleSet.Name, template)
if err != nil {
return nil, err
}
Expand Down
71 changes: 0 additions & 71 deletions cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package azure

import (
"errors"
"fmt"
"net/http"
"testing"

Expand Down Expand Up @@ -358,73 +357,3 @@ func TestTemplateNodeInfo(t *testing.T) {
assert.NotNil(t, nodeInfo)
assert.NotEmpty(t, nodeInfo.Pods())
}
func TestExtractLabelsFromScaleSet(t *testing.T) {
expectedNodeLabelKey := "zip"
expectedNodeLabelValue := "zap"
extraNodeLabelValue := "buzz"
blankString := ""

tags := map[string]*string{
fmt.Sprintf("%s%s", nodeLabelTagName, expectedNodeLabelKey): &expectedNodeLabelValue,
"fizz": &extraNodeLabelValue,
"bip": &blankString,
}

labels := extractLabelsFromScaleSet(tags)
assert.Len(t, labels, 1)
assert.Equal(t, expectedNodeLabelValue, labels[expectedNodeLabelKey])
}

func TestExtractTaintsFromScaleSet(t *testing.T) {
noScheduleTaintValue := "foo:NoSchedule"
noExecuteTaintValue := "bar:NoExecute"
preferNoScheduleTaintValue := "fizz:PreferNoSchedule"
noSplitTaintValue := "some_value"
blankTaintValue := ""
regularTagValue := "baz"

tags := map[string]*string{
fmt.Sprintf("%s%s", nodeTaintTagName, "dedicated"): &noScheduleTaintValue,
fmt.Sprintf("%s%s", nodeTaintTagName, "group"): &noExecuteTaintValue,
fmt.Sprintf("%s%s", nodeTaintTagName, "app"): &preferNoScheduleTaintValue,
fmt.Sprintf("%s%s", nodeTaintTagName, "k8s.io_testing_underscore_to_slash"): &preferNoScheduleTaintValue,
"bar": &regularTagValue,
fmt.Sprintf("%s%s", nodeTaintTagName, "blank"): &blankTaintValue,
fmt.Sprintf("%s%s", nodeTaintTagName, "nosplit"): &noSplitTaintValue,
}

expectedTaints := []apiv1.Taint{
{
Key: "dedicated",
Value: "foo",
Effect: apiv1.TaintEffectNoSchedule,
},
{
Key: "group",
Value: "bar",
Effect: apiv1.TaintEffectNoExecute,
},
{
Key: "app",
Value: "fizz",
Effect: apiv1.TaintEffectPreferNoSchedule,
},
{
Key: "k8s.io/testing/underscore/to/slash",
Value: "fizz",
Effect: apiv1.TaintEffectPreferNoSchedule,
},
}

taints := extractTaintsFromScaleSet(tags)
assert.Len(t, taints, 4)
assert.Equal(t, makeTaintSet(expectedTaints), makeTaintSet(taints))
}

func makeTaintSet(taints []apiv1.Taint) map[apiv1.Taint]bool {
set := make(map[apiv1.Taint]bool)
for _, taint := range taints {
set[taint] = true
}
return set
}
Loading

0 comments on commit 49f25f2

Please sign in to comment.