Retry Legacy Sweep Iterations (#5655)

palantir · Sep 23, 2021 · a5f1f68 · a5f1f68
1 parent 396611a
commit a5f1f68
Showing 1 changed file with 17 additions and 8 deletions.
diff --git a/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/sweep/SpecificTableSweeper.java b/atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/sweep/SpecificTableSweeper.java
@@ -16,8 +16,10 @@
 package com.palantir.atlasdb.sweep;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
 import com.palantir.atlasdb.encoding.PtBytes;
 import com.palantir.atlasdb.keyvalue.api.KeyValueService;
+import com.palantir.atlasdb.keyvalue.api.RetryLimitReachedException;
 import com.palantir.atlasdb.keyvalue.api.SweepResults;
 import com.palantir.atlasdb.keyvalue.api.TableReference;
 import com.palantir.atlasdb.logging.LoggingArgs;
@@ -126,16 +128,23 @@ void runOnceAndSaveResults(TableToSweep tableToSweep, SweepBatchConfig batchConf
 
     SweepResults runOneIteration(
             TableReference tableRef, byte[] startRow, SweepBatchConfig batchConfig, SweepTaskRunner.RunType runType) {
-        try {
-            SweepResults results = sweepRunner.run(tableRef, batchConfig, startRow, runType);
-            logSweepPerformance(tableRef, startRow, results);
+        for (int attempts = 0; attempts < 100; attempts++) {
+            try {
+                SweepResults results = sweepRunner.run(tableRef, batchConfig, startRow, runType);
+                logSweepPerformance(tableRef, startRow, results);
 
-            return results;
-        } catch (RuntimeException e) {
-            // This error may be logged on some paths above, but I prefer to log defensively.
-            logSweepError(tableRef, startRow, batchConfig, e);
-            throw e;
+                return results;
+            } catch (RuntimeException e) {
+                logSweepError(tableRef, startRow, batchConfig, e);
+                try {
+                    Thread.sleep(5000);
+                } catch (InterruptedException interrupted) {
+                    Thread.currentThread().interrupt();
+                    throw e;
+                }
+            }
         }
+        throw new RetryLimitReachedException(ImmutableList.of());
     }
 
     private void logSweepPerformance(TableReference tableRef, byte[] startRow, SweepResults results) {