-
Notifications
You must be signed in to change notification settings - Fork 25k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Tasks: Retry if task can't be written (#35054)
Adds about a minute worth of backoffs and retries to saving task results so it is *much* more likely that a busy cluster won't lose task results. This isn't an ideal solution to losing task results, but it is an incremental improvement. If all of the retries fail when still log the task result, but that is far from ideal. Closes #33764
- Loading branch information
Showing
3 changed files
with
195 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
127 changes: 127 additions & 0 deletions
127
...r/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TaskStorageRetryIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,127 @@ | ||
| /* | ||
| * Licensed to Elasticsearch under one or more contributor | ||
| * license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright | ||
| * ownership. Elasticsearch licenses this file to you under | ||
| * the Apache License, Version 2.0 (the "License"); you may | ||
| * not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.action.admin.cluster.node.tasks; | ||
|
|
||
| import org.elasticsearch.action.admin.cluster.node.tasks.get.GetTaskResponse; | ||
| import org.elasticsearch.action.support.PlainListenableActionFuture; | ||
| import org.elasticsearch.client.node.NodeClient; | ||
| import org.elasticsearch.common.settings.Settings; | ||
| import org.elasticsearch.plugins.Plugin; | ||
| import org.elasticsearch.tasks.Task; | ||
| import org.elasticsearch.tasks.TaskId; | ||
| import org.elasticsearch.test.ESSingleNodeTestCase; | ||
| import org.elasticsearch.threadpool.ThreadPool; | ||
|
|
||
| import java.util.Arrays; | ||
| import java.util.Collection; | ||
| import java.util.concurrent.CyclicBarrier; | ||
| import java.util.concurrent.TimeUnit; | ||
|
|
||
| import static java.util.Collections.emptyMap; | ||
| import static java.util.Collections.singletonMap; | ||
|
|
||
| /** | ||
| * Makes sure that tasks that attempt to store themselves on completion retry if | ||
| * they don't succeed at first. | ||
| */ | ||
| public class TaskStorageRetryIT extends ESSingleNodeTestCase { | ||
| @Override | ||
| protected Collection<Class<? extends Plugin>> getPlugins() { | ||
| return Arrays.asList(TestTaskPlugin.class); | ||
| } | ||
|
|
||
| /** | ||
| * Lower the queue sizes to be small enough that both bulk and searches will time out and have to be retried. | ||
| */ | ||
| @Override | ||
| protected Settings nodeSettings() { | ||
| return Settings.builder() | ||
| .put(super.nodeSettings()) | ||
| .put("thread_pool.write.size", 2) | ||
| .put("thread_pool.write.queue_size", 0) | ||
| .build(); | ||
| } | ||
|
|
||
| public void testRetry() throws Exception { | ||
| logger.info("block the write executor"); | ||
| CyclicBarrier barrier = new CyclicBarrier(2); | ||
| getInstanceFromNode(ThreadPool.class).executor(ThreadPool.Names.WRITE).execute(() -> { | ||
| try { | ||
| barrier.await(); | ||
| logger.info("blocking the write executor"); | ||
| barrier.await(); | ||
| logger.info("unblocked the write executor"); | ||
| } catch (Exception e) { | ||
| throw new RuntimeException(e); | ||
| } | ||
| }); | ||
| barrier.await(); | ||
| Task task; | ||
| PlainListenableActionFuture<TestTaskPlugin.NodesResponse> future = | ||
| PlainListenableActionFuture.newListenableFuture(); | ||
| try { | ||
| logger.info("start a task that will store its results"); | ||
| TestTaskPlugin.NodesRequest req = new TestTaskPlugin.NodesRequest("foo"); | ||
| req.setShouldStoreResult(true); | ||
| req.setShouldBlock(false); | ||
| task = nodeClient().executeLocally(TestTaskPlugin.TestTaskAction.INSTANCE, req, future); | ||
|
|
||
| logger.info("verify that the task has started and is still running"); | ||
| assertBusy(() -> { | ||
| GetTaskResponse runningTask = client().admin().cluster() | ||
| .prepareGetTask(new TaskId(nodeClient().getLocalNodeId(), task.getId())) | ||
| .get(); | ||
| assertNotNull(runningTask.getTask()); | ||
| assertFalse(runningTask.getTask().isCompleted()); | ||
| assertEquals(emptyMap(), runningTask.getTask().getErrorAsMap()); | ||
| assertEquals(emptyMap(), runningTask.getTask().getResponseAsMap()); | ||
| assertFalse(future.isDone()); | ||
| }); | ||
| } finally { | ||
| logger.info("unblock the write executor"); | ||
| barrier.await(); | ||
| } | ||
|
|
||
| logger.info("wait for the task to finish"); | ||
| future.get(10, TimeUnit.SECONDS); | ||
|
|
||
| logger.info("check that it was written successfully"); | ||
| GetTaskResponse finishedTask = client().admin().cluster() | ||
| .prepareGetTask(new TaskId(nodeClient().getLocalNodeId(), task.getId())) | ||
| .get(); | ||
| assertTrue(finishedTask.getTask().isCompleted()); | ||
| assertEquals(emptyMap(), finishedTask.getTask().getErrorAsMap()); | ||
| assertEquals(singletonMap("failure_count", 0), | ||
| finishedTask.getTask().getResponseAsMap()); | ||
| } | ||
|
|
||
| /** | ||
| * Get the {@linkplain NodeClient} local to the node being tested. | ||
| */ | ||
| private NodeClient nodeClient() { | ||
| /* | ||
| * Luckilly our test infrastructure already returns it, but we can't | ||
| * change the return type in the superclass because it is wrapped other | ||
| * places. | ||
| */ | ||
| return (NodeClient) client(); | ||
| } | ||
| } | ||
|
|
40 changes: 40 additions & 0 deletions
40
server/src/test/java/org/elasticsearch/tasks/TaskResultsServiceTests.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| /* | ||
| * Licensed to Elasticsearch under one or more contributor | ||
| * license agreements. See the NOTICE file distributed with | ||
| * this work for additional information regarding copyright | ||
| * ownership. Elasticsearch licenses this file to you under | ||
| * the Apache License, Version 2.0 (the "License"); you may | ||
| * not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
|
|
||
| package org.elasticsearch.tasks; | ||
|
|
||
| import org.elasticsearch.common.unit.TimeValue; | ||
| import org.elasticsearch.test.ESTestCase; | ||
|
|
||
| import java.util.Iterator; | ||
|
|
||
| /** | ||
| * Makes sure that tasks that attempt to store themselves on completion retry if | ||
| * they don't succeed at first. | ||
| */ | ||
| public class TaskResultsServiceTests extends ESTestCase { | ||
| public void testRetryTotalTime() { | ||
| Iterator<TimeValue> times = TaskResultsService.STORE_BACKOFF_POLICY.iterator(); | ||
| long total = 0; | ||
| while (times.hasNext()) { | ||
| total += times.next().millis(); | ||
| } | ||
| assertEquals(600000L, total); | ||
| } | ||
| } |