Add retry to S3 connection reset errors

plugins/indexing/pluginaws/aws_session.go

@@ -4,6 +4,8 @@ package pluginaws
 
 import (
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 )
 
@@ -14,5 +16,13 @@ func NewSession() (*session.Session, error) {
 }
 
 func NewS3Config() (*aws.Config, error) {
-	return aws.NewConfig(), nil
+	cfg := aws.NewConfig()
+	request.WithRetryer(cfg, CustomRetryer{DefaultRetryer: client.DefaultRetryer{
+		NumMaxRetries:    client.DefaultRetryerMaxNumRetries,
+		MinRetryDelay:    client.DefaultRetryerMinRetryDelay,
+		MaxRetryDelay:    client.DefaultRetryerMaxRetryDelay,
+		MinThrottleDelay: client.DefaultRetryerMinThrottleDelay,
+		MaxThrottleDelay: client.DefaultRetryerMaxThrottleDelay,
+	}})
+	return cfg, nil
 }

plugins/indexing/pluginaws/aws_session_dev.go

@@ -7,7 +7,9 @@ import (
 	"os"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 )
 
@@ -35,5 +37,13 @@ func NewS3Config() (*aws.Config, error) {
 		return nil, fmt.Errorf("missing environment variable '%s'", s3EndpointEnvName)
 	}
 	// The local emulator requires path style access
-	return aws.NewConfig().WithEndpoint(endpoint).WithS3ForcePathStyle(true), nil
+	cfg := aws.NewConfig().WithEndpoint(endpoint).WithS3ForcePathStyle(true)
+	request.WithRetryer(cfg, CustomRetryer{DefaultRetryer: client.DefaultRetryer{
+		NumMaxRetries:    client.DefaultRetryerMaxNumRetries,
+		MinRetryDelay:    client.DefaultRetryerMinRetryDelay,
+		MaxRetryDelay:    client.DefaultRetryerMaxRetryDelay,
+		MinThrottleDelay: client.DefaultRetryerMinThrottleDelay,
+		MaxThrottleDelay: client.DefaultRetryerMaxThrottleDelay,
+	}})
+	return cfg, nil
 }

plugins/indexing/pluginaws/custom_retryer.go

@@ -0,0 +1,30 @@
+package pluginaws
+
+import (
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/client"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+type CustomRetryer struct {
+	client.DefaultRetryer
+}
+
+func (r CustomRetryer) MaxRetries() int {
+	return r.DefaultRetryer.MaxRetries()
+}
+
+func (r CustomRetryer) RetryRules(req *request.Request) time.Duration {
+	return r.DefaultRetryer.RetryRules(req)
+}
+
+func (r CustomRetryer) ShouldRetry(req *request.Request) bool {
+	if strings.Contains(req.Error.Error(), "read: connection reset") {
+		return true
+	}
+
+	// Fallback to SDK's built in retry rules
+	return r.DefaultRetryer.ShouldRetry(req)
+}