Skip to content

Commit

Permalink
Merge pull request #3114 from marc-sensenich/feature-rate-limit-from-env
Browse files Browse the repository at this point in the history
cluster-autoscaler: Allow Azure Rate Limit Defaults to be set from the environment
k8s-ci-robot authored May 10, 2020
2 parents 362aec3 + 1d6f18f commit 952085e
Showing 3 changed files with 154 additions and 14 deletions.
10 changes: 5 additions & 5 deletions cluster-autoscaler/cloudprovider/azure/README.md
Original file line number Diff line number Diff line change
@@ -260,12 +260,12 @@ The new version of [Azure client][] supports rate limit and back-off retries whe
| CloudProviderBackoffDuration | 5 | BACKOFF_DURATION | cloudProviderBackoffDuration |
| CloudProviderBackoffJitter | 1.0 | BACKOFF_JITTER | cloudProviderBackoffJitter |
| CloudProviderRateLimit * | false | CLOUD_PROVIDER_RATE_LIMIT | cloudProviderRateLimit |
| CloudProviderRateLimitQPS * | 1 | | cloudProviderRateLimitQPS |
| CloudProviderRateLimitBucket * | 5 | | cloudProviderRateLimitBucket |
| CloudProviderRateLimitQPSWrite * | 1 | | cloudProviderRateLimitQPSWrite |
| CloudProviderRateLimitBucketWrite * | 5 | | cloudProviderRateLimitBucketWrite |
| CloudProviderRateLimitQPS * | 1 | RATE_LIMIT_READ_QPS | cloudProviderRateLimitQPS |
| CloudProviderRateLimitBucket * | 5 | RATE_LIMIT_READ_BUCKETS | cloudProviderRateLimitBucket |
| CloudProviderRateLimitQPSWrite * | 1 | RATE_LIMIT_WRITE_QPS | cloudProviderRateLimitQPSWrite |
| CloudProviderRateLimitBucketWrite * | 5 | RATE_LIMIT_WRITE_BUCKETS | cloudProviderRateLimitBucketWrite |

> **_NOTE_**: * These rate limit configs can be set per-client. Customizing `QPS` and `Bucket` through environment variables is not supported.
> **_NOTE_**: * These rate limit configs can be set per-client. Customizing `QPS` and `Bucket` through environment variables per client is not supported.

[AKS]: https://docs.microsoft.com/azure/aks/
[AKS autoscaler documentation]: https://docs.microsoft.com/azure/aks/autoscaler
63 changes: 54 additions & 9 deletions cluster-autoscaler/cloudprovider/azure/azure_manager.go
Original file line number Diff line number Diff line change
@@ -66,8 +66,12 @@ const (
backoffJitterDefault = 1.0

// rate limit
rateLimitQPSDefault = 1.0
rateLimitQPSDefault float32 = 1.0
rateLimitBucketDefault = 5
rateLimitReadQPSEnvVar = "RATE_LIMIT_READ_QPS"
rateLimitReadBucketsEnvVar = "RATE_LIMIT_READ_BUCKETS"
rateLimitWriteQPSEnvVar = "RATE_LIMIT_WRITE_QPS"
rateLimitWriteBucketsEnvVar = "RATE_LIMIT_WRITE_BUCKETS"
)

var validLabelAutoDiscovererKeys = strings.Join([]string{
@@ -148,31 +152,68 @@ type Config struct {
}

// InitializeCloudProviderRateLimitConfig initializes rate limit configs.
func InitializeCloudProviderRateLimitConfig(config *CloudProviderRateLimitConfig) {
func InitializeCloudProviderRateLimitConfig(config *CloudProviderRateLimitConfig) error {
if config == nil {
return
return nil
}

// Assign read rate limit defaults if no configuration was passed in.
if config.CloudProviderRateLimitQPS == 0 {
config.CloudProviderRateLimitQPS = rateLimitQPSDefault
if rateLimitQPSFromEnv := os.Getenv(rateLimitReadQPSEnvVar); rateLimitQPSFromEnv != "" {
rateLimitQPS, err := strconv.ParseFloat(rateLimitQPSFromEnv, 0)
if err != nil {
return fmt.Errorf("failed to parse %s: %q, %v", rateLimitReadQPSEnvVar, rateLimitQPSFromEnv, err)
}
config.CloudProviderRateLimitQPS = float32(rateLimitQPS)
} else {
config.CloudProviderRateLimitQPS = rateLimitQPSDefault
}
}

if config.CloudProviderRateLimitBucket == 0 {
config.CloudProviderRateLimitBucket = rateLimitBucketDefault
if rateLimitBucketFromEnv := os.Getenv(rateLimitReadBucketsEnvVar); rateLimitBucketFromEnv != "" {
rateLimitBucket, err := strconv.ParseInt(rateLimitBucketFromEnv, 10, 0)
if err != nil {
return fmt.Errorf("failed to parse %s: %q, %v", rateLimitReadBucketsEnvVar, rateLimitBucketFromEnv, err)
}
config.CloudProviderRateLimitBucket = int(rateLimitBucket)
} else {
config.CloudProviderRateLimitBucket = rateLimitBucketDefault
}
}
// Assing write rate limit defaults if no configuration was passed in.

// Assign write rate limit defaults if no configuration was passed in.
if config.CloudProviderRateLimitQPSWrite == 0 {
config.CloudProviderRateLimitQPSWrite = config.CloudProviderRateLimitQPS
if rateLimitQPSWriteFromEnv := os.Getenv(rateLimitWriteQPSEnvVar); rateLimitQPSWriteFromEnv != "" {
rateLimitQPSWrite, err := strconv.ParseFloat(rateLimitQPSWriteFromEnv, 0)
if err != nil {
return fmt.Errorf("failed to parse %s: %q, %v", rateLimitWriteQPSEnvVar, rateLimitQPSWriteFromEnv, err)
}
config.CloudProviderRateLimitQPSWrite = float32(rateLimitQPSWrite)
} else {
config.CloudProviderRateLimitQPSWrite = config.CloudProviderRateLimitQPS
}
}

if config.CloudProviderRateLimitBucketWrite == 0 {
config.CloudProviderRateLimitBucketWrite = config.CloudProviderRateLimitBucket
if rateLimitBucketWriteFromEnv := os.Getenv(rateLimitWriteBucketsEnvVar); rateLimitBucketWriteFromEnv != "" {
rateLimitBucketWrite, err := strconv.ParseInt(rateLimitBucketWriteFromEnv, 10, 0)
if err != nil {
return fmt.Errorf("failed to parse %s: %q, %v", rateLimitWriteBucketsEnvVar, rateLimitBucketWriteFromEnv, err)
}
config.CloudProviderRateLimitBucketWrite = int(rateLimitBucketWrite)
} else {
config.CloudProviderRateLimitBucketWrite = config.CloudProviderRateLimitBucket
}
}

config.InterfaceRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.InterfaceRateLimit)
config.VirtualMachineRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.VirtualMachineRateLimit)
config.StorageAccountRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.StorageAccountRateLimit)
config.DiskRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.DiskRateLimit)
config.VirtualMachineScaleSetRateLimit = overrideDefaultRateLimitConfig(&config.RateLimitConfig, config.VirtualMachineScaleSetRateLimit)

return nil
}

// overrideDefaultRateLimitConfig overrides the default CloudProviderRateLimitConfig.
@@ -358,7 +399,11 @@ func CreateAzureManager(configReader io.Reader, discoveryOpts cloudprovider.Node
return nil, fmt.Errorf("failed to parse CLOUD_PROVIDER_RATE_LIMIT: %q, %v", cloudProviderRateLimit, err)
}
}
InitializeCloudProviderRateLimitConfig(&cfg.CloudProviderRateLimitConfig)

err = InitializeCloudProviderRateLimitConfig(&cfg.CloudProviderRateLimitConfig)
if err != nil {
return nil, err
}

// Defaulting vmType to vmss.
if cfg.VMType == "" {
95 changes: 95 additions & 0 deletions cluster-autoscaler/cloudprovider/azure/azure_manager_test.go
Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@ package azure

import (
"fmt"
"os"
"reflect"
"strings"
"testing"
@@ -366,3 +367,97 @@ func TestFetchAutoAsgsVmss(t *testing.T) {
assert.Equal(t, minVal, asgs[0].MinSize())
assert.Equal(t, maxVal, asgs[0].MaxSize())
}

func TestInitializeCloudProviderRateLimitConfigWithNoConfigReturnsNoError(t *testing.T) {
err := InitializeCloudProviderRateLimitConfig(nil)
assert.Nil(t, err, "err should be nil")
}

func TestInitializeCloudProviderRateLimitConfigWithNoRateLimitSettingsReturnsDefaults(t *testing.T) {
emptyConfig := &CloudProviderRateLimitConfig{}
err := InitializeCloudProviderRateLimitConfig(emptyConfig)

assert.NoError(t, err)
assert.Equal(t, emptyConfig.CloudProviderRateLimitQPS, rateLimitQPSDefault)
assert.Equal(t, emptyConfig.CloudProviderRateLimitBucket, rateLimitBucketDefault)
assert.Equal(t, emptyConfig.CloudProviderRateLimitQPSWrite, rateLimitQPSDefault)
assert.Equal(t, emptyConfig.CloudProviderRateLimitBucketWrite, rateLimitBucketDefault)
}

func TestInitializeCloudProviderRateLimitConfigWithReadRateLimitSettingsFromEnv(t *testing.T) {
emptyConfig := &CloudProviderRateLimitConfig{}
var rateLimitReadQPS float32 = 3.0
rateLimitReadBuckets := 10
os.Setenv(rateLimitReadQPSEnvVar, fmt.Sprintf("%.1f", rateLimitReadQPS))
os.Setenv(rateLimitReadBucketsEnvVar, fmt.Sprintf("%d", rateLimitReadBuckets))

err := InitializeCloudProviderRateLimitConfig(emptyConfig)
assert.NoError(t, err)
assert.Equal(t, emptyConfig.CloudProviderRateLimitQPS, rateLimitReadQPS)
assert.Equal(t, emptyConfig.CloudProviderRateLimitBucket, rateLimitReadBuckets)
assert.Equal(t, emptyConfig.CloudProviderRateLimitQPSWrite, rateLimitReadQPS)
assert.Equal(t, emptyConfig.CloudProviderRateLimitBucketWrite, rateLimitReadBuckets)

os.Unsetenv(rateLimitReadBucketsEnvVar)
os.Unsetenv(rateLimitReadQPSEnvVar)
}

func TestInitializeCloudProviderRateLimitConfigWithReadAndWriteRateLimitSettingsFromEnv(t *testing.T) {
emptyConfig := &CloudProviderRateLimitConfig{}
var rateLimitReadQPS float32 = 3.0
rateLimitReadBuckets := 10
var rateLimitWriteQPS float32 = 6.0
rateLimitWriteBuckets := 20

os.Setenv(rateLimitReadQPSEnvVar, fmt.Sprintf("%.1f", rateLimitReadQPS))
os.Setenv(rateLimitReadBucketsEnvVar, fmt.Sprintf("%d", rateLimitReadBuckets))
os.Setenv(rateLimitWriteQPSEnvVar, fmt.Sprintf("%.1f", rateLimitWriteQPS))
os.Setenv(rateLimitWriteBucketsEnvVar, fmt.Sprintf("%d", rateLimitWriteBuckets))

err := InitializeCloudProviderRateLimitConfig(emptyConfig)

assert.NoError(t, err)
assert.Equal(t, emptyConfig.CloudProviderRateLimitQPS, rateLimitReadQPS)
assert.Equal(t, emptyConfig.CloudProviderRateLimitBucket, rateLimitReadBuckets)
assert.Equal(t, emptyConfig.CloudProviderRateLimitQPSWrite, rateLimitWriteQPS)
assert.Equal(t, emptyConfig.CloudProviderRateLimitBucketWrite, rateLimitWriteBuckets)

os.Unsetenv(rateLimitReadQPSEnvVar)
os.Unsetenv(rateLimitReadBucketsEnvVar)
os.Unsetenv(rateLimitWriteQPSEnvVar)
os.Unsetenv(rateLimitWriteBucketsEnvVar)
}

func TestInitializeCloudProviderRateLimitConfigWithReadAndWriteRateLimitAlreadySetInConfig(t *testing.T) {
var rateLimitReadQPS float32 = 3.0
rateLimitReadBuckets := 10
var rateLimitWriteQPS float32 = 6.0
rateLimitWriteBuckets := 20

configWithRateLimits := &CloudProviderRateLimitConfig{
RateLimitConfig: azclients.RateLimitConfig{
CloudProviderRateLimitBucket: rateLimitReadBuckets,
CloudProviderRateLimitBucketWrite: rateLimitWriteBuckets,
CloudProviderRateLimitQPS: rateLimitReadQPS,
CloudProviderRateLimitQPSWrite: rateLimitWriteQPS,
},
}

os.Setenv(rateLimitReadQPSEnvVar, "99")
os.Setenv(rateLimitReadBucketsEnvVar, "99")
os.Setenv(rateLimitWriteQPSEnvVar, "99")
os.Setenv(rateLimitWriteBucketsEnvVar, "99")

err := InitializeCloudProviderRateLimitConfig(configWithRateLimits)

assert.NoError(t, err)
assert.Equal(t, configWithRateLimits.CloudProviderRateLimitQPS, rateLimitReadQPS)
assert.Equal(t, configWithRateLimits.CloudProviderRateLimitBucket, rateLimitReadBuckets)
assert.Equal(t, configWithRateLimits.CloudProviderRateLimitQPSWrite, rateLimitWriteQPS)
assert.Equal(t, configWithRateLimits.CloudProviderRateLimitBucketWrite, rateLimitWriteBuckets)

os.Unsetenv(rateLimitReadQPSEnvVar)
os.Unsetenv(rateLimitReadBucketsEnvVar)
os.Unsetenv(rateLimitWriteQPSEnvVar)
os.Unsetenv(rateLimitWriteBucketsEnvVar)
}

0 comments on commit 952085e

Please sign in to comment.