diff --git a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/GoneAndRetryWithRetryPolicy.java b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/GoneAndRetryWithRetryPolicy.java
index e9343dbaa1e61..78080a1487840 100644
--- a/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/GoneAndRetryWithRetryPolicy.java
+++ b/sdk/cosmos/azure-cosmos/src/main/java/com/azure/cosmos/implementation/directconnectivity/GoneAndRetryWithRetryPolicy.java
@@ -54,11 +54,11 @@ public Mono<ShouldRetryResult> shouldRetry(Exception exception) {
 
         return this.retryWithRetryPolicy.shouldRetry(exception)
                                         .flatMap((retryWithResult) -> {
-
-            if (retryWithResult.shouldRetry) {
+            if (!retryWithResult.nonRelatedException) {
                 return Mono.just(retryWithResult);
             }
 
+            // only pass request to gone retry policy if retryWithRetryPolicy can not handle the exception.
             return this.goneRetryPolicy.shouldRetry(exception)
                 .flatMap((goneRetryResult) -> {
                     if (!goneRetryResult.shouldRetry) {
@@ -109,7 +109,6 @@ public GoneRetryPolicy(
 
         private boolean isNonRetryableException(Exception exception) {
             if (exception instanceof GoneException ||
-                exception instanceof RetryWithException ||
                 exception instanceof PartitionIsMigratingException ||
                 exception instanceof PartitionKeyRangeIsSplittingException) {
 
diff --git a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/implementation/directconnectivity/GoneAndRetryWithRetryPolicyTest.java b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/implementation/directconnectivity/GoneAndRetryWithRetryPolicyTest.java
index 6019bbda468c3..387d6951cbaa6 100644
--- a/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/implementation/directconnectivity/GoneAndRetryWithRetryPolicyTest.java
+++ b/sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/implementation/directconnectivity/GoneAndRetryWithRetryPolicyTest.java
@@ -14,12 +14,16 @@
 import com.azure.cosmos.implementation.PartitionKeyRangeIsSplittingException;
 import com.azure.cosmos.implementation.RequestTimeoutException;
 import com.azure.cosmos.implementation.ResourceType;
+import com.azure.cosmos.implementation.RetryWithException;
 import com.azure.cosmos.implementation.RxDocumentServiceRequest;
 import com.azure.cosmos.implementation.ShouldRetryResult;
 import com.azure.cosmos.implementation.guava25.base.Supplier;
+import org.mockito.Mockito;
 import org.testng.annotations.Test;
 import reactor.core.publisher.Mono;
 
+import java.time.Duration;
+
 import static com.azure.cosmos.implementation.TestUtils.mockDiagnosticsClientContext;
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -321,4 +325,22 @@ public void shouldRetryWithGenericException() {
         assertThat(shouldRetryResult.shouldRetry).isFalse();
     }
 
+    /**
+     * After waitTimeInSeconds exhausted, retryWithException will not be retried.
+     */
+    @Test(groups = { "unit" }, timeOut = TIMEOUT)
+    public void shouldRetryWithRetryWithException() {
+        RxDocumentServiceRequest request = RxDocumentServiceRequest.create(
+            mockDiagnosticsClientContext(),
+            OperationType.Read,
+            ResourceType.Document);
+        GoneAndRetryWithRetryPolicy goneAndRetryWithRetryPolicy = new GoneAndRetryWithRetryPolicy(request, 1);
+
+        ShouldRetryResult shouldRetryResult = Mono.delay(Duration.ofSeconds(1))
+            .flatMap(t -> goneAndRetryWithRetryPolicy.shouldRetry(new RetryWithException("Test", null, null)))
+            .block();
+
+        assertThat(shouldRetryResult.shouldRetry).isFalse();
+        assertThat(shouldRetryResult.nonRelatedException).isFalse();
+    }
 }