feat: use retryable http client for datadog metric provider

metricproviders/datadog/datadog.go

@@ -18,13 +18,16 @@ import (
 	"github.com/argoproj/argo-rollouts/utils/evaluate"
 	metricutil "github.com/argoproj/argo-rollouts/utils/metric"
 	timeutil "github.com/argoproj/argo-rollouts/utils/time"
+	retryablehttp "github.com/hashicorp/go-retryablehttp"
 
 	log "github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 )
 
 var unixNow = func() int64 { return timeutil.Now().Unix() }
+var RetryMaxWait = time.Minute * 2
+var RetryMax = 10
 
 const (
 	//ProviderType indicates the provider is datadog
@@ -54,7 +57,7 @@ type datadogConfig struct {
 	AppKey  string `yaml:"app-key,omitempty"`
 }
 
-// Type incidates provider is a Datadog provider
+// Type indicates provider is a Datadog provider
 func (p *Provider) Type() string {
 	return ProviderType
 }
@@ -106,10 +109,15 @@ func (p *Provider) Run(run *v1alpha1.AnalysisRun, metric v1alpha1.Metric) v1alph
 	request.Header.Set("DD-API-KEY", p.config.ApiKey)
 	request.Header.Set("DD-APPLICATION-KEY", p.config.AppKey)
 
-	// Send Request
-	httpClient := &http.Client{
-		Timeout: time.Duration(10) * time.Second,
-	}
+	// Configure retryable HTTP Client that automatically retries on 429 status codes
+	// with exponential backoff.
+	retryClient := retryablehttp.NewClient()
+	retryClient.RetryMax = RetryMax
+	retryClient.RetryWaitMax = RetryMaxWait
+	retryClient.HTTPClient.Timeout = time.Duration(10) * time.Second
+	httpClient := retryClient.StandardClient()
+
+	// Send request
 	response, err := httpClient.Do(request)
 
 	if err != nil {

metricproviders/datadog/datadog_test.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"strconv"
 	"testing"
+	"time"
 
 	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
 	log "github.com/sirupsen/logrus"
@@ -126,6 +127,21 @@ func TestRunSuite(t *testing.T) {
 			useEnvVarForKeys:        false,
 		},
 
+		// Error if ratelimit response
+		{
+			webServerStatus:   429,
+			webServerResponse: `{"errors":["Bad Request"]}`,
+			metric: v1alpha1.Metric{
+				Name:             "foo",
+				SuccessCondition: "default(result, 1) < 0.05",
+				Provider:         ddProviderIntervalDefault,
+			},
+			expectedIntervalSeconds: 300,
+			expectedPhase:           v1alpha1.AnalysisPhaseError,
+			expectedErrorMessage:    "giving up after 3 attempt(s)",
+			useEnvVarForKeys:        false,
+		},
+
 		// Expect success with default() and data
 		{
 			webServerStatus:   200,
@@ -231,6 +247,8 @@ func TestRunSuite(t *testing.T) {
 
 	for _, test := range tests {
 		serverURL := test.serverURL
+		RetryMaxWait = time.Second * 5
+		RetryMax = 2
 
 		if serverURL == "" {
 			// Server setup with response