Retry Write on kv-v2 config

testutil/testutil.go

@@ -907,3 +907,28 @@ func GetTestCertPool(t *testing.T, cert []byte) *x509.CertPool {
 	}
 	return pool
 }
+
+type TestRetryHandler struct {
+	Requests    int
+	Retries     int
+	OKAtCount   int
+	RespData    []byte
+	RetryStatus int
+}
+
+func (r *TestRetryHandler) Handler() http.HandlerFunc {
+	return func(w http.ResponseWriter, req *http.Request) {
+		if r.Requests > 0 {
+			r.Retries++
+		}
+
+		r.Requests++
+		if r.OKAtCount > 0 && (r.Requests == r.OKAtCount) {
+			w.WriteHeader(http.StatusOK)
+			_, _ = w.Write(r.RespData)
+			return
+		} else {
+			w.WriteHeader(r.RetryStatus)
+		}
+	}
+}

util/util.go

@@ -13,6 +13,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cenkalti/backoff/v4"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/vault/api"
 
@@ -405,3 +406,63 @@ func Remount(d *schema.ResourceData, client *api.Client, mountField string, isAu
 
 	return ret, nil
 }
+
+type RetryRequestOpts struct {
+	MaxTries    uint64
+	Delay       time.Duration
+	StatusCodes []int
+}
+
+func (r *RetryRequestOpts) IsRetryableStatus(statusCode int) bool {
+	for _, s := range r.StatusCodes {
+		if s == statusCode {
+			return true
+		}
+	}
+
+	return false
+}
+
+func DefaultRequestOpts() *RetryRequestOpts {
+	return &RetryRequestOpts{
+		MaxTries:    60,
+		Delay:       time.Millisecond * 500,
+		StatusCodes: []int{http.StatusBadRequest},
+	}
+}
+
+// RetryWrite attempts to retry a Logical.Write() to Vault for the
+// RetryRequestOpts. Primary useful for handling some of Vault's eventually
+// consistent APIs.
+func RetryWrite(client *api.Client, path string, data map[string]interface{}, req *RetryRequestOpts) (*api.Secret, error) {
+	if req == nil {
+		req = DefaultRequestOpts()
+	}
+
+	if path == "" {
+		return nil, fmt.Errorf("path is empty")
+	}
+
+	bo := backoff.NewConstantBackOff(req.Delay)
+
+	var resp *api.Secret
+	return resp, backoff.RetryNotify(
+		func() error {
+			r, err := client.Logical().Write(path, data)
+			if err != nil {
+				e := fmt.Errorf("error writing to path %q, err=%w", path, err)
+				if respErr, ok := err.(*api.ResponseError); ok {
+					if req.IsRetryableStatus(respErr.StatusCode) {
+						return e
+					}
+				}
+
+				return backoff.Permanent(e)
+			}
+			resp = r
+			return nil
+		}, backoff.WithMaxRetries(bo, req.MaxTries),
+		func(err error, duration time.Duration) {
+			log.Printf("[WARN] Writing to path %q failed, retrying in %s", path, duration)
+		})
+}

util/util_test.go

@@ -4,11 +4,17 @@
 package util
 
 import (
+	"encoding/json"
 	"fmt"
+	"net/http"
 	"reflect"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	"github.com/hashicorp/vault/api"
+
+	"github.com/hashicorp/terraform-provider-vault/testutil"
 )
 
 type testingStruct struct {
@@ -694,3 +700,174 @@ func TestCalculateConflictsWith(t *testing.T) {
 		})
 	}
 }
+
+func TestRetryWrite(t *testing.T) {
+	tests := []struct {
+		name         string
+		path         string
+		reqData      map[string]interface{}
+		req          *RetryRequestOpts
+		retryHandler *testutil.TestRetryHandler
+		want         *api.Secret
+		wantErr      bool
+	}{
+		{
+			name: "ok-without-retries",
+			path: "foo/baz",
+			reqData: map[string]interface{}{
+				"qux": "baz",
+			},
+			req: &RetryRequestOpts{
+				MaxTries:    3,
+				Delay:       time.Millisecond * 100,
+				StatusCodes: []int{http.StatusBadRequest},
+			},
+			retryHandler: &testutil.TestRetryHandler{
+				OKAtCount: 1,
+			},
+			want: &api.Secret{
+				Data: map[string]interface{}{
+					"qux": "baz",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "ok-with-retries",
+			path: "foo/baz",
+			reqData: map[string]interface{}{
+				"baz": "biff",
+			},
+			req: &RetryRequestOpts{
+				MaxTries:    3,
+				Delay:       time.Millisecond * 100,
+				StatusCodes: []int{http.StatusBadRequest},
+			},
+			retryHandler: &testutil.TestRetryHandler{
+				OKAtCount:   2,
+				RetryStatus: http.StatusBadRequest,
+			},
+			want: &api.Secret{
+				Data: map[string]interface{}{
+					"baz": "biff",
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "non-retryable-no-status",
+			path: "foo/baz",
+			reqData: map[string]interface{}{
+				"baz": "biff",
+			},
+			req: &RetryRequestOpts{
+				MaxTries:    3,
+				Delay:       time.Millisecond * 100,
+				StatusCodes: []int{},
+			},
+			retryHandler: &testutil.TestRetryHandler{
+				RetryStatus: http.StatusConflict,
+			},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "max-retries-exceeded-single",
+			path: "foo/baz",
+			reqData: map[string]interface{}{
+				"baz": "biff",
+			},
+			req: &RetryRequestOpts{
+				MaxTries:    3,
+				Delay:       time.Millisecond * 100,
+				StatusCodes: []int{http.StatusBadRequest},
+			},
+			retryHandler: &testutil.TestRetryHandler{
+				RetryStatus: http.StatusBadRequest,
+			},
+			want:    nil,
+			wantErr: true,
+		},
+		{
+			name: "max-retries-exceeded-choices",
+			path: "foo/baz",
+			reqData: map[string]interface{}{
+				"baz": "biff",
+			},
+			req: &RetryRequestOpts{
+				MaxTries:    3,
+				Delay:       time.Millisecond * 100,
+				StatusCodes: []int{http.StatusBadRequest, http.StatusConflict},
+			},
+			retryHandler: &testutil.TestRetryHandler{
+				RetryStatus: http.StatusConflict,
+			},
+			want:    nil,
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			config, ln := testutil.TestHTTPServer(t, tt.retryHandler.Handler())
+			defer ln.Close()
+
+			config.Address = fmt.Sprintf("http://%s", ln.Addr())
+			client, err := api.NewClient(config)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if !tt.wantErr && tt.retryHandler.RespData == nil {
+				b, err := json.Marshal(tt.reqData)
+				if err != nil {
+					t.Fatal(err)
+				}
+				tt.retryHandler.RespData = b
+			}
+
+			got, err := RetryWrite(client, tt.path, tt.reqData, tt.req)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("RetryWrite() error = %v, wantErr %v", err,
+					tt.wantErr)
+				return
+			}
+
+			if tt.wantErr {
+				if len(tt.req.StatusCodes) == 0 {
+					if tt.retryHandler.Retries != 0 {
+						t.Fatalf("expected 0 retries, actual %d",
+							tt.retryHandler.Retries)
+					}
+					if tt.retryHandler.Requests != 1 {
+						t.Fatalf("expected 1 requests, actual %d",
+							tt.retryHandler.Requests)
+					}
+				} else {
+					if int(tt.req.MaxTries) != tt.retryHandler.Retries {
+						t.Fatalf("expected %d retries, actual %d",
+							tt.req.MaxTries, tt.retryHandler.Requests)
+					}
+				}
+			} else {
+				if tt.retryHandler.OKAtCount != tt.retryHandler.Requests {
+					t.Fatalf("expected %d retries, actual %d",
+						tt.retryHandler.OKAtCount, tt.retryHandler.Requests)
+				}
+
+				var expectedRetries int
+				if tt.retryHandler.OKAtCount > 1 {
+					expectedRetries = tt.retryHandler.Requests - 1
+				}
+
+				if expectedRetries != tt.retryHandler.Retries {
+					t.Fatalf("expected %d retries, actual %d",
+						expectedRetries, tt.retryHandler.Requests)
+				}
+			}
+
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("RetryWrite() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

vault/resource_generic_secret.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/hashicorp/terraform-provider-vault/internal/consts"
 	"github.com/hashicorp/terraform-provider-vault/internal/provider"
+	"github.com/hashicorp/terraform-provider-vault/util"
 )
 
 const latestSecretVersion = -1
@@ -148,7 +149,7 @@ func genericSecretResourceWrite(d *schema.ResourceData, meta interface{}) error
 
 	}
 
-	if err := writeSecretDataWithRetry(client, path, data); err != nil {
+	if _, err := util.RetryWrite(client, path, data, util.DefaultRequestOpts()); err != nil {
 		return err
 	}