Merge pull request #2474 from DavidLangworthy/azure_api_batch_status

Improve performance of the Azure cloud provider

cluster-autoscaler/cloudprovider/azure/azure_cloud_provider_test.go

@@ -29,6 +29,10 @@ import (
 )
 
 func newTestAzureManager(t *testing.T) *AzureManager {
+	vmssName := "test-asg"
+	skuName := "Standard_D4_v2"
+	location := "eastus"
+	var vmssCapacity int64 = 3
 	manager := &AzureManager{
 		env:                  azure.PublicCloud,
 		explicitlyConfigured: make(map[string]bool),
@@ -48,11 +52,24 @@ func newTestAzureManager(t *testing.T) *AzureManager {
 				FakeStore: make(map[string]map[string]compute.VirtualMachine),
 			},
 			virtualMachineScaleSetsClient: &VirtualMachineScaleSetsClientMock{
-				FakeStore: make(map[string]map[string]compute.VirtualMachineScaleSet),
+				FakeStore: map[string]map[string]compute.VirtualMachineScaleSet{
+					"test": {
+						"test-asg": {
+							Name: &vmssName,
+							Sku: &compute.Sku{
+								Capacity: &vmssCapacity,
+								Name:     &skuName,
+							},
+							VirtualMachineScaleSetProperties: &compute.VirtualMachineScaleSetProperties{},
+							Location:                         &location,
+						},
+					},
+				},
 			},
 			virtualMachineScaleSetVMsClient: &VirtualMachineScaleSetVMsClientMock{},
 		},
 	}
+
 	cache, error := newAsgCache()
 	assert.NoError(t, error)
 

cluster-autoscaler/cloudprovider/azure/azure_scale_set.go

@@ -43,8 +43,15 @@ import (
 var (
 	vmssSizeRefreshPeriod      = 15 * time.Second
 	vmssInstancesRefreshPeriod = 5 * time.Minute
+	vmssSizeMutex              sync.Mutex
 )
 
+var scaleSetStatusCache struct {
+	lastRefresh time.Time
+	mutex       sync.Mutex
+	scaleSets   map[string]compute.VirtualMachineScaleSet
+}
+
 func init() {
 	// In go-autorest SDK https://github.com/Azure/go-autorest/blob/master/autorest/sender.go#L242,
 	// if ARM returns http.StatusTooManyRequests, the sender doesn't increase the retry attempt count,
@@ -125,13 +132,42 @@ func (scaleSet *ScaleSet) MaxSize() int {
 }
 
 func (scaleSet *ScaleSet) getVMSSInfo() (compute.VirtualMachineScaleSet, error) {
+	scaleSetStatusCache.mutex.Lock()
+	defer scaleSetStatusCache.mutex.Unlock()
+
+	if scaleSetStatusCache.lastRefresh.Add(vmssSizeRefreshPeriod).After(time.Now()) {
+		if status, exists := scaleSetStatusCache.scaleSets[scaleSet.Name]; exists {
+			return status, nil
+		}
+	}
+
+	var allVMSS []compute.VirtualMachineScaleSet
+	var err error
+
+	allVMSS, err = scaleSet.getAllVMSSInfo()
+	if err != nil {
+		return compute.VirtualMachineScaleSet{}, err
+	}
+
+	var newStatus = make(map[string]compute.VirtualMachineScaleSet)
+	for _, vmss := range allVMSS {
+		newStatus[*vmss.Name] = vmss
+	}
+
+	scaleSetStatusCache.lastRefresh = time.Now()
+	scaleSetStatusCache.scaleSets = newStatus
+
+	return scaleSetStatusCache.scaleSets[scaleSet.Name], nil
+}
+
+func (scaleSet *ScaleSet) getAllVMSSInfo() ([]compute.VirtualMachineScaleSet, error) {
 	ctx, cancel := getContextWithCancel()
 	defer cancel()
 
 	resourceGroup := scaleSet.manager.config.ResourceGroup
-	setInfo, err := scaleSet.manager.azClient.virtualMachineScaleSetsClient.Get(ctx, resourceGroup, scaleSet.Name)
+	setInfo, err := scaleSet.manager.azClient.virtualMachineScaleSetsClient.List(ctx, resourceGroup)
 	if err != nil {
-		return compute.VirtualMachineScaleSet{}, err
+		return []compute.VirtualMachineScaleSet{}, err
 	}
 
 	return setInfo, nil
@@ -156,14 +192,19 @@ func (scaleSet *ScaleSet) getCurSize() (int64, error) {
 		}
 		return -1, err
 	}
-	klog.V(5).Infof("Getting scale set (%q) capacity: %d\n", scaleSet.Name, *set.Sku.Capacity)
 
-	if scaleSet.curSize != *set.Sku.Capacity {
+	vmssSizeMutex.Lock()
+	curSize := *set.Sku.Capacity
+	vmssSizeMutex.Unlock()
+
+	klog.V(5).Infof("Getting scale set (%q) capacity: %d\n", scaleSet.Name, curSize)
+
+	if scaleSet.curSize != curSize {
 		// Invalidate the instance cache if the capacity has changed.
 		scaleSet.invalidateInstanceCache()
 	}
 
-	scaleSet.curSize = *set.Sku.Capacity
+	scaleSet.curSize = curSize
 	scaleSet.lastSizeRefresh = time.Now()
 	return scaleSet.curSize, nil
 }
@@ -197,7 +238,9 @@ func (scaleSet *ScaleSet) updateVMSSCapacity(size int64) {
 		return
 	}
 
+	vmssSizeMutex.Lock()
 	op.Sku.Capacity = &size
+	vmssSizeMutex.Unlock()
 	op.Identity = nil
 	op.VirtualMachineScaleSetProperties.ProvisioningState = nil
 	ctx, cancel := getContextWithCancel()

cluster-autoscaler/cloudprovider/azure/azure_scale_set_test.go

@@ -27,6 +27,8 @@ import (
 
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/autoscaler/cluster-autoscaler/cloudprovider"
+
+	"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2018-10-01/compute"
 )
 
 func newTestScaleSet(manager *AzureManager, name string) *ScaleSet {
@@ -65,9 +67,12 @@ func TestTargetSize(t *testing.T) {
 	assert.True(t, registered)
 	assert.Equal(t, len(provider.NodeGroups()), 1)
 
+	ng := provider.NodeGroups()[0]
+	size, err := ng.TargetSize()
+	println(size)
 	targetSize, err := provider.NodeGroups()[0].TargetSize()
 	assert.NoError(t, err)
-	assert.Equal(t, targetSize, 2)
+	assert.Equal(t, 3, targetSize)
 }
 
 func TestIncreaseSize(t *testing.T) {
@@ -80,10 +85,10 @@ func TestIncreaseSize(t *testing.T) {
 	// current target size is 2.
 	targetSize, err := provider.NodeGroups()[0].TargetSize()
 	assert.NoError(t, err)
-	assert.Equal(t, targetSize, 2)
+	assert.Equal(t, 3, targetSize)
 
 	// increase 3 nodes.
-	err = provider.NodeGroups()[0].IncreaseSize(3)
+	err = provider.NodeGroups()[0].IncreaseSize(2)
 	assert.NoError(t, err)
 
 	// new target size should be 5.
@@ -121,7 +126,20 @@ func TestBelongs(t *testing.T) {
 
 func TestDeleteNodes(t *testing.T) {
 	manager := newTestAzureManager(t)
-	scaleSetClient := &VirtualMachineScaleSetsClientMock{}
+	vmssName := "test-asg"
+	var vmssCapacity int64 = 3
+	scaleSetClient := &VirtualMachineScaleSetsClientMock{
+		FakeStore: map[string]map[string]compute.VirtualMachineScaleSet{
+			"test": {
+				"test-asg": {
+					Name: &vmssName,
+					Sku: &compute.Sku{
+						Capacity: &vmssCapacity,
+					},
+				},
+			},
+		},
+	}
 	response := autorest.Response{
 		Response: &http.Response{
 			Status: "OK",
@@ -212,7 +230,7 @@ func TestTemplateNodeInfo(t *testing.T) {
 		minSize: 1,
 		maxSize: 5,
 	}
-	asg.Name = "test-scale-set"
+	asg.Name = "test-asg"
 
 	nodeInfo, err := asg.TemplateNodeInfo()
 	assert.NoError(t, err)