feat(spanner): Add RESOURCE_EXHAUSTED to retryable transaction codes (#…

…10412) * feat: Add "RESOURCE_EXHAUSTED" to list of retryable codes for streaming operations. * feat: Add RESOURCE_EXHAUSTED as a retryable code for transactions. * feat: Add RESOURCE_EXHAUSTED to retryable transaction codes Update formatting * feat: Add RESOURCE_EXHAUSTED to retryable transaction codes Update formatting * feat(spanner): Add RESOURCE_EXHAUSTED to retryable transaction codes update formatting * Update retry_test.go --------- Co-authored-by: rahul2393 <[email protected]>
googleapis · Jul 18, 2024 · 29b52dc · 29b52dc
1 parent a880fb8
commit 29b52dc
Show file tree

Hide file tree

Showing 2 changed files with 52 additions and 1 deletion.
diff --git a/spanner/retry.go b/spanner/retry.go
@@ -85,7 +85,7 @@ func (r *spannerRetryer) Retry(err error) (time.Duration, bool) {
 // a minimum of 10ms and maximum of 32s. There is no delay before the retry if
 // the error was Session not found or failed inline begin transaction.
 func runWithRetryOnAbortedOrFailedInlineBeginOrSessionNotFound(ctx context.Context, f func(context.Context) error) error {
-	retryer := onCodes(DefaultRetryBackoff, codes.Aborted, codes.Internal)
+	retryer := onCodes(DefaultRetryBackoff, codes.Aborted, codes.ResourceExhausted, codes.Internal)
 	funcWithRetry := func(ctx context.Context) error {
 		for {
 			err := f(ctx)

diff --git a/spanner/retry_test.go b/spanner/retry_test.go
@@ -42,6 +42,20 @@ func TestRetryInfo(t *testing.T) {
 	}
 }
 
+func TestRetryInfoResourceExhausted(t *testing.T) {
+	s := status.New(codes.ResourceExhausted, "")
+	s, err := s.WithDetails(&edpb.RetryInfo{
+		RetryDelay: durationpb.New(time.Second),
+	})
+	if err != nil {
+		t.Fatalf("Error setting retry details: %v", err)
+	}
+	gotDelay, ok := ExtractRetryDelay(toSpannerErrorWithCommitInfo(s.Err(), true))
+	if !ok || !testEqual(time.Second, gotDelay) {
+		t.Errorf("<ok, retryDelay> = <%t, %v>, want <true, %v>", ok, gotDelay, time.Second)
+	}
+}
+
 func TestRetryInfoInWrappedError(t *testing.T) {
 	s := status.New(codes.Aborted, "")
 	s, err := s.WithDetails(&edpb.RetryInfo{
@@ -58,6 +72,22 @@ func TestRetryInfoInWrappedError(t *testing.T) {
 	}
 }
 
+func TestRetryInfoInWrappedErrorResourceExhausted(t *testing.T) {
+	s := status.New(codes.ResourceExhausted, "")
+	s, err := s.WithDetails(&edpb.RetryInfo{
+		RetryDelay: durationpb.New(time.Second),
+	})
+	if err != nil {
+		t.Fatalf("Error setting retry details: %v", err)
+	}
+	gotDelay, ok := ExtractRetryDelay(
+		&wrappedTestError{wrapped: toSpannerErrorWithCommitInfo(s.Err(), true), msg: "Error that is wrapping a Spanner error"},
+	)
+	if !ok || !testEqual(time.Second, gotDelay) {
+		t.Errorf("<ok, retryDelay> = <%t, %v>, want <true, %v>", ok, gotDelay, time.Second)
+	}
+}
+
 func TestRetryInfoTransactionOutcomeUnknownError(t *testing.T) {
 	err := toSpannerErrorWithCommitInfo(context.DeadlineExceeded, true)
 	if gotDelay, ok := ExtractRetryDelay(err); ok {
@@ -89,3 +119,24 @@ func TestRetryerRespectsServerDelay(t *testing.T) {
 		t.Fatalf("Retry delay mismatch:\ngot: %v\nwant: %v", maxSeenDelay, serverDelay)
 	}
 }
+
+func TestRetryerRespectsServerDelayResourceExhausted(t *testing.T) {
+	t.Parallel()
+	serverDelay := 50 * time.Millisecond
+	s := status.New(codes.ResourceExhausted, "transaction was aborted")
+	s, err := s.WithDetails(&edpb.RetryInfo{
+		RetryDelay: durationpb.New(serverDelay),
+	})
+	if err != nil {
+		t.Fatalf("Error setting retry details: %v", err)
+	}
+	retryer := onCodes(gax.Backoff{}, codes.ResourceExhausted)
+	err = toSpannerErrorWithCommitInfo(s.Err(), true)
+	maxSeenDelay, shouldRetry := retryer.Retry(err)
+	if !shouldRetry {
+		t.Fatalf("expected shouldRetry to be true")
+	}
+	if maxSeenDelay != serverDelay {
+		t.Fatalf("Retry delay mismatch:\ngot: %v\nwant: %v", maxSeenDelay, serverDelay)
+	}
+}