feat: use retryable http client for datadog metric provider

Signed-off-by: Nate Kaldor [[email protected]]

metricproviders/datadog/datadog.go

@@ -18,6 +18,7 @@ import (
 	"github.com/argoproj/argo-rollouts/utils/evaluate"
 	metricutil "github.com/argoproj/argo-rollouts/utils/metric"
 	timeutil "github.com/argoproj/argo-rollouts/utils/time"
+	retryablehttp "github.com/hashicorp/go-retryablehttp"
 
 	log "github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -54,7 +55,7 @@ type datadogConfig struct {
 	AppKey  string `yaml:"app-key,omitempty"`
 }
 
-// Type incidates provider is a Datadog provider
+// Type indicates provider is a Datadog provider
 func (p *Provider) Type() string {
 	return ProviderType
 }
@@ -106,10 +107,15 @@ func (p *Provider) Run(run *v1alpha1.AnalysisRun, metric v1alpha1.Metric) v1alph
 	request.Header.Set("DD-API-KEY", p.config.ApiKey)
 	request.Header.Set("DD-APPLICATION-KEY", p.config.AppKey)
 
-	// Send Request
-	httpClient := &http.Client{
-		Timeout: time.Duration(10) * time.Second,
-	}
+	// Configure retryable HTTP Client that automatically retries on 429 status codes
+	// with exponential backoff.
+	retryClient := retryablehttp.NewClient()
+	retryClient.RetryMax = 10
+	retryClient.RetryWaitMax = time.Minute * 2
+	retryClient.HTTPClient.Timeout = time.Duration(10) * time.Second
+	httpClient := retryClient.StandardClient()
+
+	// Send request
 	response, err := httpClient.Do(request)
 
 	if err != nil {