Add support for retrying identity entity reads

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-hclog v1.0.0
 	github.com/hashicorp/go-multierror v1.1.1
+	github.com/hashicorp/go-retryablehttp v0.7.0
 	github.com/hashicorp/go-secure-stdlib/awsutil v0.1.5
 	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.2
 	github.com/hashicorp/terraform-plugin-sdk/v2 v2.10.0

util/util.go

@@ -1,19 +1,23 @@
 package util
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"log"
+	"net/http"
 	"os"
 	"reflect"
 	"regexp"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/hashicorp/go-retryablehttp"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 	"github.com/hashicorp/terraform-plugin-sdk/v2/terraform"
+	"github.com/hashicorp/vault/api"
 )
 
 func JsonDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
@@ -328,3 +332,53 @@ func PathParameters(endpoint, vaultPath string) (map[string]string, error) {
 	}
 	return result, nil
 }
+
+// StatusCheckRetry for any response having a status code in statusCode.
+func StatusCheckRetry(statusCodes ...int) retryablehttp.CheckRetry {
+	return func(ctx context.Context, resp *http.Response, err error) (bool, error) {
+		// ensure that the client controlled consistency policy is honoured.
+		if retry, err := api.DefaultRetryPolicy(ctx, resp, err); err != nil || retry {
+			return retry, err
+		}
+
+		if resp != nil {
+			for _, code := range statusCodes {
+				if code == resp.StatusCode {
+					return true, nil
+				}
+			}
+		}
+		return false, nil
+	}
+}
+
+// SetupCCCRetryClient for handling Client Controlled Consistency related
+// requests.
+func SetupCCCRetryClient(client *api.Client, maxRetry int) {
+	if !client.ReadYourWrites() {
+		client.SetReadYourWrites(true)
+	}
+
+	client.SetMaxRetries(maxRetry)
+	client.SetCheckRetry(StatusCheckRetry(http.StatusNotFound))
+
+	// ensure that the clone has the reasonable backoff min/max durations set.
+	if client.MinRetryWait() == 0 {
+		client.SetMinRetryWait(time.Millisecond * 1000)
+	}
+	if client.MaxRetryWait() == 0 {
+		client.SetMaxRetryWait(time.Millisecond * 1500)
+	}
+	if client.MaxRetryWait() < client.MinRetryWait() {
+		client.SetMaxRetryWait(client.MinRetryWait())
+	}
+
+	bo := retryablehttp.LinearJitterBackoff
+	client.SetBackoff(bo)
+
+	to := time.Duration(0)
+	for i := 0; i < client.MaxRetries(); i++ {
+		to += bo(client.MaxRetryWait(), client.MaxRetryWait(), i, nil)
+	}
+	client.SetClientTimeout(to + time.Second*30)
+}

vault/provider.go

@@ -29,8 +29,17 @@ const (
 	// versions of Vault.
 	// We aim to deprecate items in this category.
 	UnknownPath = "unknown"
+
+	// DefaultMaxHTTPRetries is used for configuring the api.Client's MaxRetries.
+	DefaultMaxHTTPRetries = 2
+
+	// DefaultMaxHTTPRetriesCCC is used for configuring the api.Client's MaxRetries
+	// for Client Controlled Consistency related operations.
+	DefaultMaxHTTPRetriesCCC = 10
 )
 
+var maxHTTPRetriesCCC int
+
 // This is a global MutexKV for use within this provider.
 // Use this when you need to have multiple resources or even multiple instances
 // of the same resource write to the same path in Vault.
@@ -157,16 +166,20 @@ func Provider() *schema.Provider {
 				// significantly longer, so that any leases are revoked shortly
 				// after Terraform has finished running.
 				DefaultFunc: schema.EnvDefaultFunc("TERRAFORM_VAULT_MAX_TTL", 1200),
-
 				Description: "Maximum TTL for secret leases requested by this provider.",
 			},
 			"max_retries": {
-				Type:     schema.TypeInt,
-				Optional: true,
-
-				DefaultFunc: schema.EnvDefaultFunc("VAULT_MAX_RETRIES", 2),
+				Type:        schema.TypeInt,
+				Optional:    true,
+				DefaultFunc: schema.EnvDefaultFunc("VAULT_MAX_RETRIES", DefaultMaxHTTPRetries),
 				Description: "Maximum number of retries when a 5xx error code is encountered.",
 			},
+			"max_retries_ccc": {
+				Type:        schema.TypeInt,
+				Optional:    true,
+				DefaultFunc: schema.EnvDefaultFunc("VAULT_MAX_RETRIES_CCC", DefaultMaxHTTPRetriesCCC),
+				Description: "Maximum number of retries for Client Controlled Consistency related operations",
+			},
 			"namespace": {
 				Type:        schema.TypeString,
 				Optional:    true,
@@ -757,6 +770,9 @@ func providerConfigure(d *schema.ResourceData) (interface{}, error) {
 	// enable ReadYourWrites to support read-after-write on Vault Enterprise
 	clientConfig.ReadYourWrites = true
 
+	// set default MaxRetries
+	clientConfig.MaxRetries = DefaultMaxHTTPRetries
+
 	client, err := api.NewClient(clientConfig)
 	if err != nil {
 		return nil, fmt.Errorf("failed to configure Vault API: %s", err)
@@ -782,6 +798,8 @@ func providerConfigure(d *schema.ResourceData) (interface{}, error) {
 
 	client.SetMaxRetries(d.Get("max_retries").(int))
 
+	maxHTTPRetriesCCC = d.Get("max_retries_ccc").(int)
+
 	// Try an get the token from the config or token helper
 	token, err := providerToken(d)
 	if err != nil {

vault/resource_identity_entity.go

@@ -1,16 +1,20 @@
 package vault
 
 import (
+	"errors"
 	"fmt"
 	"log"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
-	"github.com/hashicorp/terraform-provider-vault/util"
 	"github.com/hashicorp/vault/api"
+
+	"github.com/hashicorp/terraform-provider-vault/util"
 )
 
 const identityEntityPath = "/identity/entity"
 
+var errEntityNotFound = errors.New("entity not found")
+
 func identityEntityResource() *schema.Resource {
 	return &schema.Resource{
 		Create: identityEntityCreate,
@@ -117,7 +121,6 @@ func identityEntityCreate(d *schema.ResourceData, meta interface{}) error {
 	identityEntityUpdateFields(d, data, true)
 
 	resp, err := client.Logical().Write(path, data)
-
 	if err != nil {
 		return fmt.Errorf("error writing IdentityEntity to %q: %s", name, err)
 	}
@@ -155,7 +158,6 @@ func identityEntityUpdate(d *schema.ResourceData, meta interface{}) error {
 	identityEntityUpdateFields(d, data, false)
 
 	_, err := client.Logical().Write(path, data)
-
 	if err != nil {
 		return fmt.Errorf("error updating IdentityEntity %q: %s", id, err)
 	}
@@ -168,14 +170,15 @@ func identityEntityRead(d *schema.ResourceData, meta interface{}) error {
 	client := meta.(*api.Client)
 	id := d.Id()
 
-	resp, err := readIdentityEntity(client, id)
+	resp, err := readIdentityEntity(client, id, d.IsNewResource())
 	if err != nil {
 		// We need to check if the secret_id has expired
-		if util.IsExpiredTokenErr(err) {
+		if resp == nil && util.IsExpiredTokenErr(err) {
 			return nil
 		}
 		return fmt.Errorf("error reading IdentityEntity %q: %s", id, err)
 	}
+
 	log.Printf("[DEBUG] Read IdentityEntity %s", id)
 	if resp == nil {
 		log.Printf("[WARN] IdentityEntity %q not found, removing from state", id)
@@ -242,28 +245,48 @@ func identityEntityIDPath(id string) string {
 }
 
 func readIdentityEntityPolicies(client *api.Client, entityID string) ([]interface{}, error) {
-	resp, err := readIdentityEntity(client, entityID)
+	resp, err := readIdentityEntity(client, entityID, false)
 	if err != nil {
 		return nil, err
 	}
-	if resp == nil {
-		return nil, fmt.Errorf("error IdentityEntity %s does not exist", entityID)
-	}
 
 	if v, ok := resp.Data["policies"]; ok && v != nil {
 		return v.([]interface{}), nil
 	}
 	return make([]interface{}, 0), nil
 }
 
-// May return nil if entity does not exist
-func readIdentityEntity(client *api.Client, entityID string) (*api.Secret, error) {
+func readIdentityEntity(client *api.Client, entityID string, retry bool) (*api.Secret, error) {
 	path := identityEntityIDPath(entityID)
-	log.Printf("[DEBUG] Reading Entity %s from %q", entityID, path)
+	log.Printf("[DEBUG] Reading Entity %q from %q", entityID, path)
+
+	return readEntity(client, path, retry)
+}
+
+func readEntity(client *api.Client, path string, retry bool) (*api.Secret, error) {
+	log.Printf("[DEBUG] Reading Entity from %q", path)
+
+	var err error
+	if retry {
+		client, err = client.Clone()
+		if err != nil {
+			return nil, err
+		}
+		util.SetupCCCRetryClient(client, maxHTTPRetriesCCC)
+	}
 
 	resp, err := client.Logical().Read(path)
 	if err != nil {
-		return resp, fmt.Errorf("failed reading IdentityEntity %s from %s", entityID, path)
+		return resp, fmt.Errorf("failed reading %q", path)
+	}
+
+	if resp == nil {
+		return nil, fmt.Errorf("%w: %q", errEntityNotFound, path)
 	}
+
 	return resp, nil
 }
+
+func isIdentityNotFoundError(err error) bool {
+	return err != nil && errors.Is(err, errEntityNotFound)
+}

vault/resource_identity_entity_policies.go

@@ -5,8 +5,9 @@ import (
 	"log"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
-	"github.com/hashicorp/terraform-provider-vault/util"
 	"github.com/hashicorp/vault/api"
+
+	"github.com/hashicorp/terraform-provider-vault/util"
 )
 
 func identityEntityPoliciesResource() *schema.Resource {
@@ -96,7 +97,7 @@ func identityEntityPoliciesRead(d *schema.ResourceData, meta interface{}) error
 	client := meta.(*api.Client)
 	id := d.Id()
 
-	resp, err := readIdentityEntity(client, id)
+	resp, err := readIdentityEntity(client, id, d.IsNewResource())
 	if err != nil {
 		return err
 	}

vault/resource_identity_entity_policies_test.go

@@ -77,13 +77,13 @@ func testAccCheckidentityEntityPoliciesDestroy(s *terraform.State) error {
 			continue
 		}
 
-		entity, err := readIdentityEntity(client, rs.Primary.ID)
-		if err != nil {
+		if _, err := readIdentityEntity(client, rs.Primary.ID, false); err != nil {
+			if isIdentityNotFoundError(err) {
+				continue
+			}
 			return err
 		}
-		if entity == nil {
-			continue
-		}
+
 		apiPolicies, err := readIdentityEntityPolicies(client, rs.Primary.ID)
 		if err != nil {
 			return err