From 816a74250debf74575e21587404126bbd830a8ca Mon Sep 17 00:00:00 2001 From: qini Date: Mon, 19 Oct 2020 11:02:40 +0800 Subject: [PATCH] Azure: support allocatable resources overrides via VMSS tags --- .../cloudprovider/azure/README.md | 9 +++++++ .../azure/azure_scale_set_test.go | 17 +++++++++++++ .../cloudprovider/azure/azure_template.go | 24 +++++++++++++++++++ .../cloudprovider/azure/azure_util.go | 5 ++-- 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/cluster-autoscaler/cloudprovider/azure/README.md b/cluster-autoscaler/cloudprovider/azure/README.md index 9854802a43a3..4ede19bb21eb 100644 --- a/cluster-autoscaler/cloudprovider/azure/README.md +++ b/cluster-autoscaler/cloudprovider/azure/README.md @@ -45,6 +45,15 @@ To add the taint of `foo=bar:NoSchedule` to a node from a VMSS pool, you would a You can also use forward slashes in taints by setting them as an underscore in the tag name. For example to add the taint of `k8s.io/foo=bar:NoSchedule` to a node from a VMSS pool, you would add the following tag to the VMSS `k8s.io_cluster-autoscaler_node-template_taint_k8s.io_foo: bar:NoSchedule` +#### Resources + +When scaling from an empty VM Scale Set (0 instances), Cluster Autoscaler will evaluate the provided presources (cpu, memory, ephemeral-storage) based on that VM Scale Set's backing instance type. +This can be overridden (for instance, to account for system reserved resources) by specifying capacities with VMSS tags, formated as: `k8s.io_cluster-autoscaler_node-template_resources_: `. For instance: +``` +k8s.io_cluster-autoscaler_node-template_resources_cpu: 3800m +k8s.io_cluster-autoscaler_node-template_resources_memory: 11Gi +``` + ## Deployment manifests Cluster autoscaler supports four Kubernetes cluster options on Azure: diff --git a/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go b/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go index f1e1cbdbaba0..51efd81ca0d0 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go @@ -23,6 +23,7 @@ import ( "time" apiv1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/autoscaler/cluster-autoscaler/cloudprovider" "k8s.io/legacy-cloud-providers/azure/clients/vmssclient/mockvmssclient" "k8s.io/legacy-cloud-providers/azure/clients/vmssvmclient/mockvmssvmclient" @@ -470,3 +471,19 @@ func TestTemplateNodeInfo(t *testing.T) { assert.NotNil(t, nodeInfo) assert.NotEmpty(t, nodeInfo.Pods) } + +func TestExtractAllocatableResourcesFromScaleSet(t *testing.T) { + tags := map[string]*string{ + fmt.Sprintf("%s%s", nodeResourcesTagName, "cpu"): to.StringPtr("100m"), + fmt.Sprintf("%s%s", nodeResourcesTagName, "memory"): to.StringPtr("100M"), + fmt.Sprintf("%s%s", nodeResourcesTagName, "ephemeral-storage"): to.StringPtr("20G"), + } + + labels := extractAllocatableResourcesFromScaleSet(tags) + + assert.Equal(t, resource.NewMilliQuantity(100, resource.DecimalSI).String(), labels["cpu"].String()) + expectedMemory := resource.MustParse("100M") + assert.Equal(t, (&expectedMemory).String(), labels["memory"].String()) + expectedEphemeralStorage := resource.MustParse("20G") + assert.Equal(t, (&expectedEphemeralStorage).String(), labels["ephemeral-storage"].String()) +} diff --git a/cluster-autoscaler/cloudprovider/azure/azure_template.go b/cluster-autoscaler/cloudprovider/azure/azure_template.go index 1c67dc8da4f5..8e45ce82f16f 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_template.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_template.go @@ -113,6 +113,11 @@ func buildNodeFromTemplate(scaleSetName string, template compute.VirtualMachineS node.Status.Capacity[gpu.ResourceNvidiaGPU] = *resource.NewQuantity(vmssType.GPU, resource.DecimalSI) node.Status.Capacity[apiv1.ResourceMemory] = *resource.NewQuantity(vmssType.MemoryMb*1024*1024, resource.DecimalSI) + resourcesFromTags := extractAllocatableResourcesFromScaleSet(template.Tags) + for resourceName, val := range resourcesFromTags { + node.Status.Capacity[apiv1.ResourceName(resourceName)] = *val + } + // TODO: set real allocatable. node.Status.Allocatable = node.Status.Capacity @@ -140,6 +145,25 @@ func buildNodeFromTemplate(scaleSetName string, template compute.VirtualMachineS return &node, nil } +func extractAllocatableResourcesFromScaleSet(tags map[string]*string) map[string]*resource.Quantity { + resources := make(map[string]*resource.Quantity) + + for tagName, tagValue := range tags { + resourceName := strings.Split(tagName, nodeResourcesTagName) + if len(resourceName) < 2 || resourceName[1] == "" { + continue + } + + quantity, err := resource.ParseQuantity(*tagValue) + if err != nil { + continue + } + resources[resourceName[1]] = &quantity + } + + return resources +} + func extractLabelsFromScaleSet(tags map[string]*string) map[string]string { result := make(map[string]string) diff --git a/cluster-autoscaler/cloudprovider/azure/azure_util.go b/cluster-autoscaler/cloudprovider/azure/azure_util.go index bfc19d5f1462..fb8bbb80a05b 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_util.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_util.go @@ -78,8 +78,9 @@ const ( k8sWindowsVMAgentOrchestratorNameIndex = 2 k8sWindowsVMAgentPoolInfoIndex = 3 - nodeLabelTagName = "k8s.io_cluster-autoscaler_node-template_label_" - nodeTaintTagName = "k8s.io_cluster-autoscaler_node-template_taint_" + nodeLabelTagName = "k8s.io_cluster-autoscaler_node-template_label_" + nodeTaintTagName = "k8s.io_cluster-autoscaler_node-template_taint_" + nodeResourcesTagName = "k8s.io_cluster-autoscaler_node-template_resources_" ) var (