From 370d261e0f0e8061b4a144c89eb23b5895c37cdf Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Mon, 27 Jul 2020 13:49:51 -0400 Subject: [PATCH] Set timeout of master requests on follower to unbounded (#60070) Today, a follow task will fail if the master node of the follower cluster is temporarily overloaded and unable to process master node requests (such as update mapping, setting, or alias) from a follow-task within the default timeout. This error is transient, and follow-tasks should not abort. We can avoid this problem by setting the timeout of master node requests on the follower cluster to unbounded. Closes #56891 --- .../java/org/elasticsearch/common/unit/TimeValue.java | 1 + .../org/elasticsearch/xpack/ccr/action/CcrRequests.java | 1 + .../xpack/ccr/action/ShardFollowTasksExecutor.java | 8 ++++---- .../elasticsearch/xpack/ccr/repository/CcrRepository.java | 3 +-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/libs/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java b/libs/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java index b5a3003b194d5..54937b159c363 100644 --- a/libs/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java +++ b/libs/core/src/main/java/org/elasticsearch/common/unit/TimeValue.java @@ -30,6 +30,7 @@ public class TimeValue implements Comparable { public static final TimeValue MINUS_ONE = timeValueMillis(-1); public static final TimeValue ZERO = timeValueMillis(0); + public static final TimeValue MAX_VALUE = TimeValue.timeValueNanos(Long.MAX_VALUE); public static TimeValue timeValueNanos(long nanos) { return new TimeValue(nanos, TimeUnit.NANOSECONDS); diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CcrRequests.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CcrRequests.java index 02ee7d1f138c9..70a29eb51352e 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CcrRequests.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/CcrRequests.java @@ -42,6 +42,7 @@ public static PutMappingRequest putMappingRequest(String followerIndex, MappingM putMappingRequest.origin("ccr"); putMappingRequest.type(mappingMetaData.type()); putMappingRequest.source(mappingMetaData.source().string(), XContentType.JSON); + putMappingRequest.masterNodeTimeout(TimeValue.MAX_VALUE); return putMappingRequest; } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java index b28cef1cb0be9..715a083b4c36e 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/ShardFollowTasksExecutor.java @@ -218,10 +218,10 @@ private void closeIndexUpdateSettingsAndOpenIndex(String followIndex, Settings updatedSettings, Runnable handler, Consumer onFailure) { - CloseIndexRequest closeRequest = new CloseIndexRequest(followIndex); CheckedConsumer onResponse = response -> { updateSettingsAndOpenIndex(followIndex, updatedSettings, handler, onFailure); }; + CloseIndexRequest closeRequest = new CloseIndexRequest(followIndex).masterNodeTimeout(TimeValue.MAX_VALUE); followerClient.admin().indices().close(closeRequest, ActionListener.wrap(onResponse, onFailure)); } @@ -229,8 +229,8 @@ private void updateSettingsAndOpenIndex(String followIndex, Settings updatedSettings, Runnable handler, Consumer onFailure) { - final UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(followIndex); - updateSettingsRequest.settings(updatedSettings); + final UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(followIndex) + .masterNodeTimeout(TimeValue.MAX_VALUE).settings(updatedSettings); CheckedConsumer onResponse = response -> openIndex(followIndex, handler, onFailure); followerClient.admin().indices().updateSettings(updateSettingsRequest, ActionListener.wrap(onResponse, onFailure)); } @@ -238,7 +238,7 @@ private void updateSettingsAndOpenIndex(String followIndex, private void openIndex(String followIndex, Runnable handler, Consumer onFailure) { - OpenIndexRequest openIndexRequest = new OpenIndexRequest(followIndex); + OpenIndexRequest openIndexRequest = new OpenIndexRequest(followIndex).masterNodeTimeout(TimeValue.MAX_VALUE); CheckedConsumer onResponse = response -> handler.run(); followerClient.admin().indices().open(openIndexRequest, ActionListener.wrap(onResponse, onFailure)); } diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java index 3415b7657a780..cf1d1a8b3eb25 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/repository/CcrRepository.java @@ -433,8 +433,7 @@ private void updateMappings(Client leaderClient, Index leaderIndex, long leaderM leaderIndexMetadata.getMappings().size() + "]"; MappingMetaData mappingMetaData = leaderIndexMetadata.getMappings().iterator().next().value; if (mappingMetaData != null) { - final PutMappingRequest putMappingRequest = CcrRequests.putMappingRequest(followerIndex.getName(), mappingMetaData) - .masterNodeTimeout(TimeValue.timeValueMinutes(30)); + final PutMappingRequest putMappingRequest = CcrRequests.putMappingRequest(followerIndex.getName(), mappingMetaData); followerClient.admin().indices().putMapping(putMappingRequest).actionGet(ccrSettings.getRecoveryActionTimeout()); } }