From 98a73bec4d37dea3461d8753187b2ea3b0dc2697 Mon Sep 17 00:00:00 2001
From: kaldorn <nate.kaldor@outreach.io>
Date: Thu, 9 Feb 2023 10:53:34 -0800
Subject: [PATCH] feat: use retryable http client for datadog metric provider

---
 metricproviders/datadog/datadog.go | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/metricproviders/datadog/datadog.go b/metricproviders/datadog/datadog.go
index 9461a61ed6..9edf496881 100644
--- a/metricproviders/datadog/datadog.go
+++ b/metricproviders/datadog/datadog.go
@@ -18,6 +18,7 @@ import (
 	"github.com/argoproj/argo-rollouts/utils/evaluate"
 	metricutil "github.com/argoproj/argo-rollouts/utils/metric"
 	timeutil "github.com/argoproj/argo-rollouts/utils/time"
+	retryablehttp "github.com/hashicorp/go-retryablehttp"
 
 	log "github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -54,7 +55,7 @@ type datadogConfig struct {
 	AppKey  string `yaml:"app-key,omitempty"`
 }
 
-// Type incidates provider is a Datadog provider
+// Type indicates provider is a Datadog provider
 func (p *Provider) Type() string {
 	return ProviderType
 }
@@ -106,10 +107,15 @@ func (p *Provider) Run(run *v1alpha1.AnalysisRun, metric v1alpha1.Metric) v1alph
 	request.Header.Set("DD-API-KEY", p.config.ApiKey)
 	request.Header.Set("DD-APPLICATION-KEY", p.config.AppKey)
 
-	// Send Request
-	httpClient := &http.Client{
-		Timeout: time.Duration(10) * time.Second,
-	}
+	// Configure retryable HTTP Client that automatically retries on 429 status codes
+	// with exponential backoff.
+	retryClient := retryablehttp.NewClient()
+	retryClient.RetryMax = 10
+	retryClient.RetryWaitMax = time.Minute * 2
+	retryClient.HTTPClient.Timeout = time.Duration(10) * time.Second
+	httpClient := retryClient.StandardClient()
+
+	// Send request
 	response, err := httpClient.Do(request)
 
 	if err != nil {