feat: retryable http client for datadog metric provider

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/golang/protobuf v1.5.2
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
 	github.com/hashicorp/go-plugin v1.4.8
+	github.com/hashicorp/go-retryablehttp v0.7.1
 	github.com/influxdata/influxdb-client-go/v2 v2.12.2
 	github.com/juju/ansiterm v1.0.0
 	github.com/mitchellh/mapstructure v1.5.0
@@ -122,7 +123,6 @@ require (
 	github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-hclog v0.14.1 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.1 // indirect
 	github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect

metricproviders/datadog/datadog.go

@@ -18,6 +18,7 @@ import (
 	"github.com/argoproj/argo-rollouts/utils/evaluate"
 	metricutil "github.com/argoproj/argo-rollouts/utils/metric"
 	timeutil "github.com/argoproj/argo-rollouts/utils/time"
+	retryablehttp "github.com/hashicorp/go-retryablehttp"
 
 	log "github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -27,12 +28,19 @@ import (
 var unixNow = func() int64 { return timeutil.Now().Unix() }
 
 const (
-	// ProviderType indicates the provider is datadog
-	ProviderType            = "Datadog"
-	DatadogTokensSecretName = "datadog"
-	DatadogApiKey           = "api-key"
-	DatadogAppKey           = "app-key"
-	DatadogAddress          = "address"
+	//ProviderType indicates the provider is datadog
+	ProviderType                             = "Datadog"
+	DatadogTokensSecretName                  = "datadog"
+	DatadogApiKey                            = "api-key"
+	DatadogAppKey                            = "app-key"
+	DatadogAddress                           = "address"
+	DefaultRetryMaxWaitSeconds time.Duration = 120
+	DefaultRetryMax            int           = 10
+)
+
+var (
+	retryMaxWaitSeconds = DefaultRetryMaxWaitSeconds
+	retryMax            = DefaultRetryMax
 )
 
 // Provider contains all the required components to run a Datadog query
@@ -54,7 +62,7 @@ type datadogConfig struct {
 	AppKey  string `yaml:"app-key,omitempty"`
 }
 
-// Type incidates provider is a Datadog provider
+// Type indicates provider is a Datadog provider
 func (p *Provider) Type() string {
 	return ProviderType
 }
@@ -106,10 +114,15 @@ func (p *Provider) Run(run *v1alpha1.AnalysisRun, metric v1alpha1.Metric) v1alph
 	request.Header.Set("DD-API-KEY", p.config.ApiKey)
 	request.Header.Set("DD-APPLICATION-KEY", p.config.AppKey)
 
-	// Send Request
-	httpClient := &http.Client{
-		Timeout: time.Duration(10) * time.Second,
-	}
+	// Configure retryable HTTP Client that automatically retries on 429 status codes
+	// with exponential backoff.
+	retryClient := retryablehttp.NewClient()
+	retryClient.RetryMax = retryMax
+	retryClient.RetryWaitMax = time.Second * retryMaxWaitSeconds
+	retryClient.HTTPClient.Timeout = time.Duration(10) * time.Second
+	httpClient := retryClient.StandardClient()
+
+	// Send request
 	response, err := httpClient.Do(request)
 
 	if err != nil {
@@ -202,6 +215,14 @@ func lookupKeysInEnv(keys []string) map[string]string {
 	return valuesByKey
 }
 
+func SetRetryMax(max int) {
+	retryMax = max
+}
+
+func SetRetryMaxWait(seconds time.Duration) {
+	retryMaxWaitSeconds = seconds
+}
+
 func NewDatadogProvider(logCtx log.Entry, kubeclientset kubernetes.Interface) (*Provider, error) {
 	ns := defaults.Namespace()
 

metricproviders/datadog/datadog_test.go

@@ -126,6 +126,21 @@ func TestRunSuite(t *testing.T) {
 			useEnvVarForKeys:        false,
 		},
 
+		// Error if ratelimit response
+		{
+			webServerStatus:   429,
+			webServerResponse: `{"errors":["Bad Request"]}`,
+			metric: v1alpha1.Metric{
+				Name:             "foo",
+				SuccessCondition: "default(result, 1) < 0.05",
+				Provider:         ddProviderIntervalDefault,
+			},
+			expectedIntervalSeconds: 300,
+			expectedPhase:           v1alpha1.AnalysisPhaseError,
+			expectedErrorMessage:    "giving up after 3 attempt(s)",
+			useEnvVarForKeys:        false,
+		},
+
 		// Expect success with default() and data
 		{
 			webServerStatus:   200,
@@ -231,6 +246,8 @@ func TestRunSuite(t *testing.T) {
 
 	for _, test := range tests {
 		serverURL := test.serverURL
+		SetRetryMaxWait(5)
+		SetRetryMax(2)
 
 		if serverURL == "" {
 			// Server setup with response