Make S3 anti-contention delay configurable (#101245) (#101251)

The anti-contention delay in the S3 repository's compare-and-exchange
operation is hard-coded at 1 second today, but sometimes we encounter a
repository that needs much longer to perform a compare-and-exchange
operation when under contention. With this commit we make the
anti-contention delay configurable.

docs/changelog/101245.yaml

@@ -0,0 +1,5 @@
+pr: 101245
+summary: Make S3 anti-contention delay configurable
+area: Snapshot/Restore
+type: bug
+issues: []

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java

@@ -751,7 +751,9 @@ void run(BytesReference expected, BytesReference updated, ActionListener<Optiona
 
                     if (uploadIndex > 0) {
                         threadPool.scheduleUnlessShuttingDown(
-                            TimeValue.timeValueMillis(TimeValue.timeValueSeconds(uploadIndex).millis() + Randomness.get().nextInt(50)),
+                            TimeValue.timeValueMillis(
+                                uploadIndex * blobStore.getCompareAndExchangeAntiContentionDelay().millis() + Randomness.get().nextInt(50)
+                            ),
                             blobStore.getSnapshotExecutor(),
                             cancelConcurrentUpdates
                         );

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java

@@ -157,6 +157,10 @@ public TimeValue getCompareAndExchangeTimeToLive() {
         return service.compareAndExchangeTimeToLive;
     }
 
+    public TimeValue getCompareAndExchangeAntiContentionDelay() {
+        return service.compareAndExchangeAntiContentionDelay;
+    }
+
     // metrics collector that ignores null responses that we interpret as the request not reaching the S3 endpoint due to a network
     // issue
     private abstract static class IgnoreNoResponseMetricsCollector extends RequestMetricCollector {

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -60,6 +60,14 @@ class S3Service implements Closeable {
         Setting.Property.NodeScope
     );
 
+    private static final Setting<TimeValue> REPOSITORY_S3_CAS_ANTI_CONTENTION_DELAY_SETTING = Setting.timeSetting(
+        "repository_s3.compare_and_exchange.anti_contention_delay",
+        TimeValue.timeValueSeconds(1),
+        TimeValue.timeValueMillis(1),
+        TimeValue.timeValueHours(24),
+        Setting.Property.NodeScope
+    );
+
     private volatile Map<S3ClientSettings, AmazonS3Reference> clientsCache = emptyMap();
 
     /**
@@ -79,6 +87,7 @@ class S3Service implements Closeable {
     final CustomWebIdentityTokenCredentialsProvider webIdentityTokenCredentialsProvider;
 
     final TimeValue compareAndExchangeTimeToLive;
+    final TimeValue compareAndExchangeAntiContentionDelay;
 
     S3Service(Environment environment, Settings nodeSettings) {
         webIdentityTokenCredentialsProvider = new CustomWebIdentityTokenCredentialsProvider(
@@ -88,6 +97,7 @@ class S3Service implements Closeable {
             Clock.systemUTC()
         );
         compareAndExchangeTimeToLive = REPOSITORY_S3_CAS_TTL_SETTING.get(nodeSettings);
+        compareAndExchangeAntiContentionDelay = REPOSITORY_S3_CAS_ANTI_CONTENTION_DELAY_SETTING.get(nodeSettings);
     }
 
     /**