diff --git a/docs/changelog/101245.yaml b/docs/changelog/101245.yaml new file mode 100644 index 0000000000000..2f9fef318f31a --- /dev/null +++ b/docs/changelog/101245.yaml @@ -0,0 +1,5 @@ +pr: 101245 +summary: Make S3 anti-contention delay configurable +area: Snapshot/Restore +type: bug +issues: [] diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java index c7dee4f1599c5..d97763ab315c9 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java @@ -751,7 +751,9 @@ void run(BytesReference expected, BytesReference updated, ActionListener 0) { threadPool.scheduleUnlessShuttingDown( - TimeValue.timeValueMillis(TimeValue.timeValueSeconds(uploadIndex).millis() + Randomness.get().nextInt(50)), + TimeValue.timeValueMillis( + uploadIndex * blobStore.getCompareAndExchangeAntiContentionDelay().millis() + Randomness.get().nextInt(50) + ), blobStore.getSnapshotExecutor(), cancelConcurrentUpdates ); diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java index fca005e8de32c..b6cd017c049ee 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobStore.java @@ -157,6 +157,10 @@ public TimeValue getCompareAndExchangeTimeToLive() { return service.compareAndExchangeTimeToLive; } + public TimeValue getCompareAndExchangeAntiContentionDelay() { + return service.compareAndExchangeAntiContentionDelay; + } + // metrics collector that ignores null responses that we interpret as the request not reaching the S3 endpoint due to a network // issue private abstract static class IgnoreNoResponseMetricsCollector extends RequestMetricCollector { diff --git a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java index ddacb24be7118..736e8870f21a2 100644 --- a/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java +++ b/modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java @@ -60,6 +60,14 @@ class S3Service implements Closeable { Setting.Property.NodeScope ); + private static final Setting REPOSITORY_S3_CAS_ANTI_CONTENTION_DELAY_SETTING = Setting.timeSetting( + "repository_s3.compare_and_exchange.anti_contention_delay", + TimeValue.timeValueSeconds(1), + TimeValue.timeValueMillis(1), + TimeValue.timeValueHours(24), + Setting.Property.NodeScope + ); + private volatile Map clientsCache = emptyMap(); /** @@ -79,6 +87,7 @@ class S3Service implements Closeable { final CustomWebIdentityTokenCredentialsProvider webIdentityTokenCredentialsProvider; final TimeValue compareAndExchangeTimeToLive; + final TimeValue compareAndExchangeAntiContentionDelay; S3Service(Environment environment, Settings nodeSettings) { webIdentityTokenCredentialsProvider = new CustomWebIdentityTokenCredentialsProvider( @@ -88,6 +97,7 @@ class S3Service implements Closeable { Clock.systemUTC() ); compareAndExchangeTimeToLive = REPOSITORY_S3_CAS_TTL_SETTING.get(nodeSettings); + compareAndExchangeAntiContentionDelay = REPOSITORY_S3_CAS_ANTI_CONTENTION_DELAY_SETTING.get(nodeSettings); } /**