aws/client: Adds configurations to the default retryer

CHANGELOG_PENDING.md

@@ -1,7 +1,11 @@
 ### SDK Features
 
 ### SDK Enhancements
+* `aws/client`: Adds configurations to the default retryer ([#2830](https://github.com/aws/aws-sdk-go/pull/2830))
+  * Exposes members of the default retryer. Adds NoOpRetryer to support no retry behavior. 
+  * Updates the underlying logic used by the default retryer to calculate jittered delay for retry. 
+  * Fixes [#2829](https://github.com/aws/aws-sdk-go/issues/2829)
 * `aws`: Add value/pointer conversion functions for all basic number types ([#2740](https://github.com/aws/aws-sdk-go/pull/2740))
   * Adds value and pointer conversion utilities for the remaining set of integer and float number types.
-
+  
 ### SDK Bugs

aws/client/client.go

@@ -64,7 +64,7 @@ func New(cfg aws.Config, info metadata.ClientInfo, handlers request.Handlers, op
 	default:
 		maxRetries := aws.IntValue(cfg.MaxRetries)
 		if cfg.MaxRetries == nil || maxRetries == aws.UseServiceDefaultRetries {
-			maxRetries = 3
+			maxRetries = DefaultRetryerMaxNumRetries
 		}
 		svc.Retryer = DefaultRetryer{NumMaxRetries: maxRetries}
 	}

aws/client/default_retryer.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"math"
 	"strconv"
 	"time"
 
@@ -9,56 +10,131 @@ import (
 )
 
 // DefaultRetryer implements basic retry logic using exponential backoff for
-// most services. If you want to implement custom retry logic, implement the
-// request.Retryer interface or create a structure type that composes this
-// struct and override the specific methods. For example, to override only
-// the MaxRetries method:
+// most services. If you want to implement custom retry logic, you can implement the
+// request.Retryer interface.
 //
-//   type retryer struct {
-//       client.DefaultRetryer
-//   }
-//
-//   // This implementation always has 100 max retries
-//   func (d retryer) MaxRetries() int { return 100 }
 type DefaultRetryer struct {
-	NumMaxRetries int
+	// Num max Retries is the number of max retries that will be performed.
+	// By default, this is zero.
+	NumMaxRetries    int
+
+	// MinRetryDelay is the minimum retry delay after which retry will be performed.
+	// If not set, the value is 0ns.
+	MinRetryDelay    time.Duration
+
+	// MinThrottleRetryDelay is the minimum retry delay when throttled.
+	// If not set, the value is 0ns.
+	MinThrottleDelay time.Duration
+
+	// MaxRetryDelay is the maximum retry delay before which retry must be performed.
+	// If not set, the value is 0ns.
+	MaxRetryDelay    time.Duration
+
+	// MaxThrottleDelay is the maximum retry delay when throttled.
+	// If not set, the value is 0ns.
+	MaxThrottleDelay time.Duration
 }
 
+const (
+	// DefaultRetryerMaxNumRetries sets maximum number of retries
+	DefaultRetryerMaxNumRetries = 3
+
+	// DefaultRetryerMinRetryDelay sets minimum retry delay
+	DefaultRetryerMinRetryDelay = 30 * time.Millisecond
+
+	// DefaultRetryerMinThrottleDelay sets minimum delay when throttled
+	DefaultRetryerMinThrottleDelay = 500 * time.Millisecond
+
+	// DefaultRetryerMaxRetryDelay sets maximum retry delay
+	DefaultRetryerMaxRetryDelay = 300 * time.Second
+
+	// DefaultRetryerMaxThrottleDelay sets maximum delay when throttled
+	DefaultRetryerMaxThrottleDelay = 300 * time.Second
+)
+
 // MaxRetries returns the number of maximum returns the service will use to make
 // an individual API request.
 func (d DefaultRetryer) MaxRetries() int {
 	return d.NumMaxRetries
 }
 
+// setRetryerDefaults sets the default values of the retryer if not set
+func (d *DefaultRetryer) setRetryerDefaults() {
+	if d.MinRetryDelay == 0 {
+		d.MinRetryDelay = DefaultRetryerMinRetryDelay
+	}
+	if d.MaxRetryDelay == 0 {
+		d.MaxRetryDelay = DefaultRetryerMaxRetryDelay
+	}
+	if d.MinThrottleDelay == 0 {
+		d.MinThrottleDelay = DefaultRetryerMinThrottleDelay
+	}
+	if d.MaxThrottleDelay == 0 {
+		d.MaxThrottleDelay = DefaultRetryerMaxThrottleDelay
+	}
+}
+
 // RetryRules returns the delay duration before retrying this request again
 func (d DefaultRetryer) RetryRules(r *request.Request) time.Duration {
-	// Set the upper limit of delay in retrying at ~five minutes
-	var minTime int64 = 30
+
+	// if number of max retries is zero, no retries will be performed.
+	if d.NumMaxRetries == 0 {
+		return 0
+	}
+
+	// Sets default value for retryer members
+	d.setRetryerDefaults()
+
+	// minDelay is the minimum retryer delay
+	minDelay := d.MinRetryDelay
+
 	var initialDelay time.Duration
 
 	isThrottle := r.IsErrorThrottle()
 	if isThrottle {
 		if delay, ok := getRetryAfterDelay(r); ok {
 			initialDelay = delay
 		}
-
-		minTime = 500
+		minDelay = d.MinThrottleDelay
 	}
 
 	retryCount := r.RetryCount
-	if isThrottle && retryCount > 8 {
-		retryCount = 8
-	} else if retryCount > 12 {
-		retryCount = 12
+
+	// maxDelay the maximum retryer delay
+	maxDelay := d.MaxRetryDelay
+
+	if isThrottle {
+		maxDelay = d.MaxThrottleDelay
 	}
 
-	delay := (1 << uint(retryCount)) * (sdkrand.SeededRand.Int63n(minTime) + minTime)
-	return (time.Duration(delay) * time.Millisecond) + initialDelay
+	var delay time.Duration
 
+	// Logic to cap the retry count based on the minDelay provided
+	actualRetryCount := int(math.Log2(float64(minDelay))) + 1
+	if actualRetryCount < 63-retryCount {
+		delay = time.Duration(1<<uint64(retryCount)) * getJitterDelay(minDelay)
+		if delay > maxDelay {
+			delay = getJitterDelay(maxDelay / 2)
+		}
+	} else {
+		delay = getJitterDelay(maxDelay / 2)
+	}
+	return delay + initialDelay
+}
+
+// getJitterDelay returns a jittered delay for retry
+func getJitterDelay(duration time.Duration) time.Duration {
+	return time.Duration(sdkrand.SeededRand.Int63n(int64(duration)) + int64(duration))
 }
 
 // ShouldRetry returns true if the request should be retried.
 func (d DefaultRetryer) ShouldRetry(r *request.Request) bool {
+
+	// ShouldRetry returns false if number of max retries is 0.
+	if d.NumMaxRetries == 0 {
+		return false
+	}
+
 	// If one of the other handlers already set the retry state
 	// we don't want to override it based on the service's state
 	if r.Retryable != nil {

aws/client/default_retryer_test.go

@@ -166,7 +166,7 @@ func TestGetRetryDelay(t *testing.T) {
 }
 
 func TestRetryDelay(t *testing.T) {
-	d := DefaultRetryer{100}
+	d := DefaultRetryer{NumMaxRetries:100}
 	r := request.Request{}
 	for i := 0; i < 100; i++ {
 		rTemp := r

aws/client/no_op_retryer.go

@@ -0,0 +1,28 @@
+package client
+
+import (
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+// NoOpRetryer provides a retryer that performs no retries.
+// It should be used when we do not want retries to be performed.
+type NoOpRetryer struct{}
+
+// MaxRetries returns the number of maximum returns the service will use to make
+// an individual API; For NoOpRetryer the MaxRetries will always be zero.
+func (d NoOpRetryer) MaxRetries() int {
+	return 0
+}
+
+// ShouldRetry will always return false for NoOpRetryer, as it should never retry.
+func (d NoOpRetryer) ShouldRetry(_ *request.Request) bool {
+	return false
+}
+
+// RetryRules returns the delay duration before retrying this request again;
+// since NoOpRetryer does not retry, RetryRules always returns 0.
+func (d NoOpRetryer) RetryRules(_ *request.Request) time.Duration {
+	return 0
+}

aws/client/no_op_retryer_test.go

@@ -0,0 +1,46 @@
+package client
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+func TestNoOpRetryer(t *testing.T) {
+	cases := []struct {
+		r                request.Request
+		expectMaxRetries int
+		expectRetryDelay time.Duration
+		expectRetry      bool
+	}{
+		{
+			r: request.Request{
+				HTTPResponse: &http.Response{StatusCode: 200},
+			},
+			expectMaxRetries: 0,
+			expectRetryDelay: 0,
+			expectRetry:      false,
+		},
+	}
+
+	d := NoOpRetryer{}
+	for i, c := range cases {
+		maxRetries := d.MaxRetries()
+		retry := d.ShouldRetry(&c.r)
+		retryDelay := d.RetryRules(&c.r)
+
+		if e, a := c.expectMaxRetries, maxRetries; e != a {
+			t.Errorf("%d: expected %v, but received %v for number of max retries", i, e, a)
+		}
+
+		if e, a := c.expectRetry, retry; e != a {
+			t.Errorf("%d: expected %v, but received %v for should retry", i, e, a)
+		}
+
+		if e, a := c.expectRetryDelay, retryDelay; e != a {
+			t.Errorf("%d: expected %v, but received %v as retry delay", i, e, a)
+		}
+	}
+}

aws/request/request_test.go

@@ -257,7 +257,7 @@ func TestRequestExhaustRetries(t *testing.T) {
 		t.Errorf("expect %d retry count, got %d", e, a)
 	}
 
-	expectDelays := []struct{ min, max time.Duration }{{30, 59}, {60, 118}, {120, 236}}
+	expectDelays := []struct{ min, max time.Duration }{{30, 60}, {60, 120}, {120, 240}}
 	for i, v := range delays {
 		min := expectDelays[i].min * time.Millisecond
 		max := expectDelays[i].max * time.Millisecond
@@ -361,7 +361,7 @@ func TestRequestUserAgent(t *testing.T) {
 }
 
 func TestRequestThrottleRetries(t *testing.T) {
-	delays := []time.Duration{}
+	var delays []time.Duration
 	sleepDelay := func(delay time.Duration) {
 		delays = append(delays, delay)
 	}
@@ -402,7 +402,7 @@ func TestRequestThrottleRetries(t *testing.T) {
 		t.Errorf("expect %d retry count, got %d", e, a)
 	}
 
-	expectDelays := []struct{ min, max time.Duration }{{500, 999}, {1000, 1998}, {2000, 3996}}
+	expectDelays := []struct{ min, max time.Duration }{{500, 1000}, {1000, 2000}, {2000, 4000}}
 	for i, v := range delays {
 		min := expectDelays[i].min * time.Millisecond
 		max := expectDelays[i].max * time.Millisecond

aws/request/retryer.go

@@ -94,10 +94,6 @@ var validParentCodes = map[string]struct{}{
 	ErrCodeRead:          {},
 }
 
-type temporaryError interface {
-	Temporary() bool
-}
-
 func isNestedErrorRetryable(parentErr awserr.Error) bool {
 	if parentErr == nil {
 		return false
@@ -116,7 +112,7 @@ func isNestedErrorRetryable(parentErr awserr.Error) bool {
 		return isCodeRetryable(aerr.Code())
 	}
 
-	if t, ok := err.(temporaryError); ok {
+	if t, ok := err.(temporary); ok {
 		return t.Temporary() || isErrConnectionReset(err)
 	}
 

example/aws/request/customRetryer/custom_retryer.go

@@ -20,7 +20,10 @@ func main() {
 	sess := session.Must(
 		session.NewSession(&aws.Config{
 			// Use a custom retryer to provide custom retry rules.
-			Retryer: CustomRetryer{DefaultRetryer: client.DefaultRetryer{NumMaxRetries: 3}},
+			Retryer: CustomRetryer{
+				DefaultRetryer: client.DefaultRetryer{
+					NumMaxRetries: client.DefaultRetryerMaxNumRetries,
+				}},
 
 			// Use the SDK's SharedCredentialsProvider directly instead of the
 			// SDK's default credential chain. This ensures that the

service/dynamodb/customizations.go

@@ -5,7 +5,6 @@ import (
 	"hash/crc32"
 	"io"
 	"io/ioutil"
-	"math"
 	"strconv"
 	"time"
 
@@ -15,15 +14,6 @@ import (
 	"github.com/aws/aws-sdk-go/aws/request"
 )
 
-type retryer struct {
-	client.DefaultRetryer
-}
-
-func (d retryer) RetryRules(r *request.Request) time.Duration {
-	delay := time.Duration(math.Pow(2, float64(r.RetryCount))) * 50
-	return delay * time.Millisecond
-}
-
 func init() {
 	initClient = func(c *client.Client) {
 		if c.Config.Retryer == nil {
@@ -43,10 +33,9 @@ func setCustomRetryer(c *client.Client) {
 		maxRetries = 10
 	}
 
-	c.Retryer = retryer{
-		DefaultRetryer: client.DefaultRetryer{
-			NumMaxRetries: maxRetries,
-		},
+	c.Retryer = client.DefaultRetryer{
+		NumMaxRetries: maxRetries,
+		MinRetryDelay: 50 * time.Millisecond,
 	}
 }