Add max_retries for client HTTP calls (#463)

huaweicloud/config.go

@@ -47,6 +47,7 @@ type Config struct {
 	AgencyName       string
 	AgencyDomainName string
 	DelegatedProject string
+	MaxRetries       int
 	terraformVersion string
 
 	HwClient *golangsdk.ProviderClient
@@ -56,6 +57,10 @@ type Config struct {
 }
 
 func (c *Config) LoadAndValidate() error {
+	if c.MaxRetries < 0 {
+		return fmt.Errorf("max_retries should be a positive value")
+	}
+
 	err := fmt.Errorf("Must config token or aksk or username password to be authorized")
 
 	if c.Token != "" {
@@ -135,8 +140,9 @@ func genClient(c *Config, ao golangsdk.AuthOptionsProvider) (*golangsdk.Provider
 
 	client.HTTPClient = http.Client{
 		Transport: &LogRoundTripper{
-			Rt:      transport,
-			OsDebug: logging.IsDebugOrHigher(),
+			Rt:         transport,
+			OsDebug:    logging.IsDebugOrHigher(),
+			MaxRetries: c.MaxRetries,
 		},
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
 			if client.AKSKAuthOptions.AccessKey != "" {

huaweicloud/config_test.go

@@ -0,0 +1,57 @@
+package huaweicloud
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"sync"
+	"testing"
+
+	"github.com/huaweicloud/golangsdk"
+	th "github.com/huaweicloud/golangsdk/testhelper"
+)
+
+func testRequestRetry(t *testing.T, count int) {
+	th.SetupHTTP()
+	defer th.TeardownHTTP()
+
+	retryCount := count
+
+	var info = struct {
+		retries int
+		mut     *sync.RWMutex
+	}{
+		0,
+		new(sync.RWMutex),
+	}
+
+	th.Mux.HandleFunc("/route/", func(w http.ResponseWriter, r *http.Request) {
+		defer r.Body.Close()
+		_, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			t.Errorf("Error hadling test request")
+		}
+		if info.retries < retryCount {
+			info.mut.RLock()
+			info.retries += 1
+			info.mut.RUnlock()
+			panic(err) // simulate EOF
+		}
+		w.WriteHeader(500)
+		_, _ = fmt.Fprintf(w, `%v`, info.retries)
+	})
+
+	cfg := &Config{MaxRetries: retryCount}
+	_, err := genClient(cfg, golangsdk.AuthOptions{
+		IdentityEndpoint: fmt.Sprintf("%s/route", th.Endpoint()),
+	})
+	_, ok := err.(golangsdk.ErrDefault500)
+	th.AssertEquals(t, true, ok)
+	th.AssertEquals(t, retryCount, info.retries)
+}
+
+func TestRequestRetry(t *testing.T) {
+	t.Run("TestRequestMultipleRetries", func(t *testing.T) { testRequestRetry(t, 2) })
+	t.Run("TestRequestSingleRetry", func(t *testing.T) { testRequestRetry(t, 1) })
+	t.Run("TestRequestZeroRetry", func(t *testing.T) { testRequestRetry(t, 0) })
+}

huaweicloud/provider.go

@@ -166,6 +166,12 @@ func Provider() terraform.ResourceProvider {
 				DefaultFunc: schema.EnvDefaultFunc("OS_DELEGATED_PROJECT", ""),
 				Description: descriptions["delegated_project"],
 			},
+			"max_retries": {
+				Type:        schema.TypeInt,
+				Optional:    true,
+				Default:     25,
+				Description: descriptions["max_retries"],
+			},
 		},
 
 		DataSourcesMap: map[string]*schema.Resource{
@@ -394,6 +400,7 @@ func init() {
 		"agency_domain_name": "The name of domain who created the agency (Identity v3).",
 
 		"delegated_project": "The name of delegated project (Identity v3).",
+		"max_retries":       "How many times HTTP connection should be retried until giving up.",
 	}
 }
 
@@ -434,6 +441,7 @@ func configureProvider(d *schema.ResourceData, terraformVersion string) (interfa
 		AgencyName:       d.Get("agency_name").(string),
 		AgencyDomainName: d.Get("agency_domain_name").(string),
 		DelegatedProject: delegated_project,
+		MaxRetries:       d.Get("max_retries").(int),
 		terraformVersion: terraformVersion,
 	}
 

huaweicloud/types.go

@@ -7,8 +7,10 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math"
 	"net/http"
 	"strings"
+	"time"
 
 	"github.com/huaweicloud/golangsdk/openstack/compute/v2/extensions/keypairs"
 	"github.com/huaweicloud/golangsdk/openstack/compute/v2/extensions/servergroups"
@@ -31,11 +33,23 @@ import (
 	"github.com/huaweicloud/golangsdk/openstack/networking/v2/subnets"
 )
 
+var maxTimeout = 10 * time.Minute
+
 // LogRoundTripper satisfies the http.RoundTripper interface and is used to
 // customize the default http client RoundTripper to allow for logging.
 type LogRoundTripper struct {
-	Rt      http.RoundTripper
-	OsDebug bool
+	Rt         http.RoundTripper
+	OsDebug    bool
+	MaxRetries int
+}
+
+func retryTimeout(count int) time.Duration {
+	seconds := math.Pow(2, float64(count))
+	timeout := time.Duration(seconds) * time.Second
+	if timeout > maxTimeout { // won't wait more than maxTimeout
+		timeout = maxTimeout
+	}
+	return timeout
 }
 
 // RoundTrip performs a round-trip HTTP request and logs relevant information about it.
@@ -64,8 +78,24 @@ func (lrt *LogRoundTripper) RoundTrip(request *http.Request) (*http.Response, er
 	}
 
 	response, err := lrt.Rt.RoundTrip(request)
-	if response == nil {
-		return nil, err
+	// Retrying connection
+	retry := 1
+	for response == nil {
+
+		if retry > lrt.MaxRetries {
+			if lrt.OsDebug {
+				log.Printf("[DEBUG] HuaweiCloud connection error, retries exhausted. Aborting")
+			}
+			err = fmt.Errorf("HuaweiCloud connection error, retries exhausted. Aborting. Last error was: %s", err)
+			return nil, err
+		}
+
+		if lrt.OsDebug {
+			log.Printf("[DEBUG] HuaweiCloud connection error, retry number %d: %s", retry, err)
+		}
+		time.Sleep(retryTimeout(retry))
+		response, err = lrt.Rt.RoundTrip(request)
+		retry += 1
 	}
 
 	if lrt.OsDebug {