diff --git a/cluster-autoscaler/cloudprovider/azure/README.md b/cluster-autoscaler/cloudprovider/azure/README.md index 9cab61ee767a..a4019467ec2f 100644 --- a/cluster-autoscaler/cloudprovider/azure/README.md +++ b/cluster-autoscaler/cloudprovider/azure/README.md @@ -260,12 +260,12 @@ The new version of [Azure client][] supports rate limit and back-off retries whe | CloudProviderBackoffDuration | 5 | BACKOFF_DURATION | cloudProviderBackoffDuration | | CloudProviderBackoffJitter | 1.0 | BACKOFF_JITTER | cloudProviderBackoffJitter | | CloudProviderRateLimit * | false | CLOUD_PROVIDER_RATE_LIMIT | cloudProviderRateLimit | -| CloudProviderRateLimitQPS * | 1 | | cloudProviderRateLimitQPS | -| CloudProviderRateLimitBucket * | 5 | | cloudProviderRateLimitBucket | -| CloudProviderRateLimitQPSWrite * | 1 | | cloudProviderRateLimitQPSWrite | -| CloudProviderRateLimitBucketWrite * | 5 | | cloudProviderRateLimitBucketWrite | +| CloudProviderRateLimitQPS * | 1 | RATE_LIMIT_READ_QPS | cloudProviderRateLimitQPS | +| CloudProviderRateLimitBucket * | 5 | RATE_LIMIT_READ_BUCKETS | cloudProviderRateLimitBucket | +| CloudProviderRateLimitQPSWrite * | 1 | RATE_LIMIT_WRITE_QPS | cloudProviderRateLimitQPSWrite | +| CloudProviderRateLimitBucketWrite * | 5 | RATE_LIMIT_WRITE_BUCKETS | cloudProviderRateLimitBucketWrite | -> **_NOTE_**: * These rate limit configs can be set per-client. Customizing `QPS` and `Bucket` through environment variables is not supported. +> **_NOTE_**: * These rate limit configs can be set per-client. Customizing `QPS` and `Bucket` through environment variables per client is not supported. [AKS]: https://docs.microsoft.com/azure/aks/ [AKS autoscaler documentation]: https://docs.microsoft.com/azure/aks/autoscaler diff --git a/cluster-autoscaler/cloudprovider/azure/azure_manager.go b/cluster-autoscaler/cloudprovider/azure/azure_manager.go index 7427addc5058..a6b2ce0849df 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_manager.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_manager.go @@ -66,8 +66,12 @@ const ( backoffJitterDefault = 1.0 // rate limit - rateLimitQPSDefault = 1.0 + rateLimitQPSDefault float32 = 1.0 rateLimitBucketDefault = 5 + rateLimitReadQPSEnvVar = "RATE_LIMIT_READ_QPS" + rateLimitReadBucketsEnvVar = "RATE_LIMIT_READ_BUCKETS" + rateLimitWriteQPSEnvVar = "RATE_LIMIT_WRITE_QPS" + rateLimitWriteBucketsEnvVar = "RATE_LIMIT_WRITE_BUCKETS" ) var validLabelAutoDiscovererKeys = strings.Join([]string{ @@ -148,24 +152,59 @@ type Config struct { } // InitializeCloudProviderRateLimitConfig initializes rate limit configs. -func InitializeCloudProviderRateLimitConfig(config *CloudProviderRateLimitConfig) { +func InitializeCloudProviderRateLimitConfig(config *CloudProviderRateLimitConfig) error { if config == nil { - return + return nil } // Assign read rate limit defaults if no configuration was passed in. if config.CloudProviderRateLimitQPS == 0 { - config.CloudProviderRateLimitQPS = rateLimitQPSDefault + if rateLimitQPSFromEnv := os.Getenv(rateLimitReadQPSEnvVar); rateLimitQPSFromEnv != "" { + rateLimitQPS, err := strconv.ParseFloat(rateLimitQPSFromEnv, 0) + if err != nil { + return fmt.Errorf("failed to parse %s: %q, %v", rateLimitReadQPSEnvVar, rateLimitQPSFromEnv, err) + } + config.CloudProviderRateLimitQPS = float32(rateLimitQPS) + } else { + config.CloudProviderRateLimitQPS = rateLimitQPSDefault + } } + if config.CloudProviderRateLimitBucket == 0 { - config.CloudProviderRateLimitBucket = rateLimitBucketDefault + if rateLimitBucketFromEnv := os.Getenv(rateLimitReadBucketsEnvVar); rateLimitBucketFromEnv != "" { + rateLimitBucket, err := strconv.ParseInt(rateLimitBucketFromEnv, 10, 0) + if err != nil { + return fmt.Errorf("failed to parse %s: %q, %v", rateLimitReadBucketsEnvVar, rateLimitBucketFromEnv, err) + } + config.CloudProviderRateLimitBucket = int(rateLimitBucket) + } else { + config.CloudProviderRateLimitBucket = rateLimitBucketDefault + } } - // Assing write rate limit defaults if no configuration was passed in. + + // Assign write rate limit defaults if no configuration was passed in. if config.CloudProviderRateLimitQPSWrite == 0 { - config.CloudProviderRateLimitQPSWrite = config.CloudProviderRateLimitQPS + if rateLimitQPSWriteFromEnv := os.Getenv(rateLimitWriteQPSEnvVar); rateLimitQPSWriteFromEnv != "" { + rateLimitQPSWrite, err := strconv.ParseFloat(rateLimitQPSWriteFromEnv, 0) + if err != nil { + return fmt.Errorf("failed to parse %s: %q, %v", rateLimitWriteQPSEnvVar, rateLimitQPSWriteFromEnv, err) + } + config.CloudProviderRateLimitQPSWrite = float32(rateLimitQPSWrite) + } else { + config.CloudProviderRateLimitQPSWrite = config.CloudProviderRateLimitQPS + } } + if config.CloudProviderRateLimitBucketWrite == 0 { - config.CloudProviderRateLimitBucketWrite = config.CloudProviderRateLimitBucket + if rateLimitBucketWriteFromEnv := os.Getenv(rateLimitWriteBucketsEnvVar); rateLimitBucketWriteFromEnv != "" { + rateLimitBucketWrite, err := strconv.ParseInt(rateLimitBucketWriteFromEnv, 10, 0) + if err != nil { + return fmt.Errorf("failed to parse %s: %q, %v", rateLimitWriteBucketsEnvVar, rateLimitBucketWriteFromEnv, err) + } + config.CloudProviderRateLimitBucketWrite = int(rateLimitBucketWrite) + } else { + config.CloudProviderRateLimitBucketWrite = config.CloudProviderRateLimitBucket + } } config.InterfaceRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.InterfaceRateLimit) @@ -173,6 +212,8 @@ func InitializeCloudProviderRateLimitConfig(config *CloudProviderRateLimitConfig config.StorageAccountRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.StorageAccountRateLimit) config.DiskRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.DiskRateLimit) config.VirtualMachineScaleSetRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.VirtualMachineScaleSetRateLimit) + + return nil } // overrideDefaultRateLimitConfig overrides the default CloudProviderRateLimitConfig. @@ -358,7 +399,11 @@ func CreateAzureManager(configReader io.Reader, discoveryOpts cloudprovider.Node return nil, fmt.Errorf("failed to parse CLOUD_PROVIDER_RATE_LIMIT: %q, %v", cloudProviderRateLimit, err) } } - InitializeCloudProviderRateLimitConfig(&cfg.CloudProviderRateLimitConfig) + + err = InitializeCloudProviderRateLimitConfig(&cfg.CloudProviderRateLimitConfig) + if err != nil { + return nil, err + } // Defaulting vmType to vmss. if cfg.VMType == "" { diff --git a/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go b/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go index 69787faa29eb..646715d9097d 100644 --- a/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go +++ b/cluster-autoscaler/cloudprovider/azure/azure_manager_test.go @@ -18,6 +18,7 @@ package azure import ( "fmt" + "os" "reflect" "strings" "testing" @@ -366,3 +367,97 @@ func TestFetchAutoAsgsVmss(t *testing.T) { assert.Equal(t, minVal, asgs[0].MinSize()) assert.Equal(t, maxVal, asgs[0].MaxSize()) } + +func TestInitializeCloudProviderRateLimitConfigWithNoConfigReturnsNoError(t *testing.T) { + err := InitializeCloudProviderRateLimitConfig(nil) + assert.Nil(t, err, "err should be nil") +} + +func TestInitializeCloudProviderRateLimitConfigWithNoRateLimitSettingsReturnsDefaults(t *testing.T) { + emptyConfig := &CloudProviderRateLimitConfig{} + err := InitializeCloudProviderRateLimitConfig(emptyConfig) + + assert.NoError(t, err) + assert.Equal(t, emptyConfig.CloudProviderRateLimitQPS, rateLimitQPSDefault) + assert.Equal(t, emptyConfig.CloudProviderRateLimitBucket, rateLimitBucketDefault) + assert.Equal(t, emptyConfig.CloudProviderRateLimitQPSWrite, rateLimitQPSDefault) + assert.Equal(t, emptyConfig.CloudProviderRateLimitBucketWrite, rateLimitBucketDefault) +} + +func TestInitializeCloudProviderRateLimitConfigWithReadRateLimitSettingsFromEnv(t *testing.T) { + emptyConfig := &CloudProviderRateLimitConfig{} + var rateLimitReadQPS float32 = 3.0 + rateLimitReadBuckets := 10 + os.Setenv(rateLimitReadQPSEnvVar, fmt.Sprintf("%.1f", rateLimitReadQPS)) + os.Setenv(rateLimitReadBucketsEnvVar, fmt.Sprintf("%d", rateLimitReadBuckets)) + + err := InitializeCloudProviderRateLimitConfig(emptyConfig) + assert.NoError(t, err) + assert.Equal(t, emptyConfig.CloudProviderRateLimitQPS, rateLimitReadQPS) + assert.Equal(t, emptyConfig.CloudProviderRateLimitBucket, rateLimitReadBuckets) + assert.Equal(t, emptyConfig.CloudProviderRateLimitQPSWrite, rateLimitReadQPS) + assert.Equal(t, emptyConfig.CloudProviderRateLimitBucketWrite, rateLimitReadBuckets) + + os.Unsetenv(rateLimitReadBucketsEnvVar) + os.Unsetenv(rateLimitReadQPSEnvVar) +} + +func TestInitializeCloudProviderRateLimitConfigWithReadAndWriteRateLimitSettingsFromEnv(t *testing.T) { + emptyConfig := &CloudProviderRateLimitConfig{} + var rateLimitReadQPS float32 = 3.0 + rateLimitReadBuckets := 10 + var rateLimitWriteQPS float32 = 6.0 + rateLimitWriteBuckets := 20 + + os.Setenv(rateLimitReadQPSEnvVar, fmt.Sprintf("%.1f", rateLimitReadQPS)) + os.Setenv(rateLimitReadBucketsEnvVar, fmt.Sprintf("%d", rateLimitReadBuckets)) + os.Setenv(rateLimitWriteQPSEnvVar, fmt.Sprintf("%.1f", rateLimitWriteQPS)) + os.Setenv(rateLimitWriteBucketsEnvVar, fmt.Sprintf("%d", rateLimitWriteBuckets)) + + err := InitializeCloudProviderRateLimitConfig(emptyConfig) + + assert.NoError(t, err) + assert.Equal(t, emptyConfig.CloudProviderRateLimitQPS, rateLimitReadQPS) + assert.Equal(t, emptyConfig.CloudProviderRateLimitBucket, rateLimitReadBuckets) + assert.Equal(t, emptyConfig.CloudProviderRateLimitQPSWrite, rateLimitWriteQPS) + assert.Equal(t, emptyConfig.CloudProviderRateLimitBucketWrite, rateLimitWriteBuckets) + + os.Unsetenv(rateLimitReadQPSEnvVar) + os.Unsetenv(rateLimitReadBucketsEnvVar) + os.Unsetenv(rateLimitWriteQPSEnvVar) + os.Unsetenv(rateLimitWriteBucketsEnvVar) +} + +func TestInitializeCloudProviderRateLimitConfigWithReadAndWriteRateLimitAlreadySetInConfig(t *testing.T) { + var rateLimitReadQPS float32 = 3.0 + rateLimitReadBuckets := 10 + var rateLimitWriteQPS float32 = 6.0 + rateLimitWriteBuckets := 20 + + configWithRateLimits := &CloudProviderRateLimitConfig{ + RateLimitConfig: azclients.RateLimitConfig{ + CloudProviderRateLimitBucket: rateLimitReadBuckets, + CloudProviderRateLimitBucketWrite: rateLimitWriteBuckets, + CloudProviderRateLimitQPS: rateLimitReadQPS, + CloudProviderRateLimitQPSWrite: rateLimitWriteQPS, + }, + } + + os.Setenv(rateLimitReadQPSEnvVar, "99") + os.Setenv(rateLimitReadBucketsEnvVar, "99") + os.Setenv(rateLimitWriteQPSEnvVar, "99") + os.Setenv(rateLimitWriteBucketsEnvVar, "99") + + err := InitializeCloudProviderRateLimitConfig(configWithRateLimits) + + assert.NoError(t, err) + assert.Equal(t, configWithRateLimits.CloudProviderRateLimitQPS, rateLimitReadQPS) + assert.Equal(t, configWithRateLimits.CloudProviderRateLimitBucket, rateLimitReadBuckets) + assert.Equal(t, configWithRateLimits.CloudProviderRateLimitQPSWrite, rateLimitWriteQPS) + assert.Equal(t, configWithRateLimits.CloudProviderRateLimitBucketWrite, rateLimitWriteBuckets) + + os.Unsetenv(rateLimitReadQPSEnvVar) + os.Unsetenv(rateLimitReadBucketsEnvVar) + os.Unsetenv(rateLimitWriteQPSEnvVar) + os.Unsetenv(rateLimitWriteBucketsEnvVar) +}