From ba62eb3dce438b40f85040c0f5ab3295f5fee53c Mon Sep 17 00:00:00 2001 From: David Turner Date: Fri, 11 Oct 2019 16:43:59 +0200 Subject: [PATCH] Allow truncation of clean translog (#47866) Today the `elasticsearch-shard remove-corrupted-data` tool will only truncate a translog it determines to be corrupt. However there may be other cases in which it is desirable to truncate the translog, for instance if an operation in the translog cannot be replayed for some reason other than corruption. This commit adds a `--truncate-clean-translog` option to skip the corruption check on the translog and blindly truncate it. --- docs/reference/commands/shard-tool.asciidoc | 46 +++++++++++++------ .../RemoveCorruptedShardDataCommand.java | 18 ++++++-- .../RemoveCorruptedShardDataCommandTests.java | 35 ++++++++++++++ 3 files changed, 79 insertions(+), 20 deletions(-) diff --git a/docs/reference/commands/shard-tool.asciidoc b/docs/reference/commands/shard-tool.asciidoc index 6fca1355a27be..5689b26f5dd2c 100644 --- a/docs/reference/commands/shard-tool.asciidoc +++ b/docs/reference/commands/shard-tool.asciidoc @@ -1,22 +1,36 @@ [[shard-tool]] == elasticsearch-shard -In some cases the Lucene index or translog of a shard copy can become -corrupted. The `elasticsearch-shard` command enables you to remove corrupted -parts of the shard if a good copy of the shard cannot be recovered -automatically or restored from backup. +In some cases the Lucene index or translog of a shard copy can become corrupted. +The `elasticsearch-shard` command enables you to remove corrupted parts of the +shard if a good copy of the shard cannot be recovered automatically or restored +from backup. [WARNING] You will lose the corrupted data when you run `elasticsearch-shard`. This tool should only be used as a last resort if there is no way to recover from another copy of the shard or restore a snapshot. -When Elasticsearch detects that a shard's data is corrupted, it fails that -shard copy and refuses to use it. Under normal conditions, the shard is -automatically recovered from another copy. If no good copy of the shard is -available and you cannot restore from backup, you can use `elasticsearch-shard` -to remove the corrupted data and restore access to any remaining data in -unaffected segments. +[float] +=== Synopsis + +[source,shell] +-------------------------------------------------- +bin/elasticsearch-shard remove-corrupted-data + ([--index ] [--shard-id ] | [--dir ]) + [--truncate-clean-translog] + [-E ] + [-h, --help] ([-s, --silent] | [-v, --verbose]) +-------------------------------------------------- + +[float] +=== Description + +When {es} detects that a shard's data is corrupted, it fails that shard copy and +refuses to use it. Under normal conditions, the shard is automatically recovered +from another copy. If no good copy of the shard is available and you cannot +restore one from a snapshot, you can use `elasticsearch-shard` to remove the +corrupted data and restore access to any remaining data in unaffected segments. [WARNING] Stop Elasticsearch before running `elasticsearch-shard`. @@ -31,7 +45,7 @@ There are two ways to specify the path: translog files. [float] -=== Removing corrupted data +==== Removing corrupted data `elasticsearch-shard` analyses the shard copy and provides an overview of the corruption found. To proceed you must then confirm that you want to remove the @@ -91,7 +105,7 @@ POST /_cluster/reroute ] } -You must accept the possibility of data loss by changing parameter `accept_data_loss` to `true`. +You must accept the possibility of data loss by changing the `accept_data_loss` parameter to `true`. Deleted corrupt marker corrupted_FzTSBSuxT7i3Tls_TgwEag from /var/lib/elasticsearchdata/nodes/0/indices/P45vf_YQRhqjfwLMUvSqDw/0/index/ @@ -99,9 +113,11 @@ Deleted corrupt marker corrupted_FzTSBSuxT7i3Tls_TgwEag from /var/lib/elasticsea When you use `elasticsearch-shard` to drop the corrupted data, the shard's allocation ID changes. After restarting the node, you must use the -<> to tell Elasticsearch to use the new -ID. The `elasticsearch-shard` command shows the request that -you need to submit. +<> to tell Elasticsearch to use the new ID. +The `elasticsearch-shard` command shows the request that you need to submit. You can also use the `-h` option to get a list of all options and parameters that the `elasticsearch-shard` tool supports. + +Finally, you can use the `--truncate-clean-translog` option to truncate the +shard's translog even if it does not appear to be corrupt. diff --git a/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java b/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java index d7c03330f0b7e..7466e798c5d00 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java +++ b/server/src/main/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommand.java @@ -80,6 +80,7 @@ public class RemoveCorruptedShardDataCommand extends EnvironmentAwareCommand { private final OptionSpec folderOption; private final OptionSpec indexNameOption; private final OptionSpec shardIdOption; + static final String TRUNCATE_CLEAN_TRANSLOG_FLAG = "truncate-clean-translog"; private final RemoveCorruptedLuceneSegmentsAction removeCorruptedLuceneSegmentsAction; private final TruncateTranslogAction truncateTranslogAction; @@ -99,6 +100,8 @@ public RemoveCorruptedShardDataCommand() { .withRequiredArg() .ofType(Integer.class); + parser.accepts(TRUNCATE_CLEAN_TRANSLOG_FLAG, "Truncate the translog even if it is not corrupt"); + namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables()); removeCorruptedLuceneSegmentsAction = new RemoveCorruptedLuceneSegmentsAction(); @@ -320,8 +323,11 @@ public void write(int b) { terminal.println(""); ////////// Translog - // as translog relies on data stored in an index commit - we have to have non unrecoverable index to truncate translog - if (indexCleanStatus.v1() != CleanStatus.UNRECOVERABLE) { + if (options.has(TRUNCATE_CLEAN_TRANSLOG_FLAG)) { + translogCleanStatus = Tuple.tuple(CleanStatus.OVERRIDDEN, + "Translog was not analysed and will be truncated due to the --" + TRUNCATE_CLEAN_TRANSLOG_FLAG + " flag"); + } else if (indexCleanStatus.v1() != CleanStatus.UNRECOVERABLE) { + // translog relies on data stored in an index commit so we have to have a recoverable index to check the translog terminal.println(""); terminal.println("Opening translog at " + translogPath); terminal.println(""); @@ -344,7 +350,8 @@ public void write(int b) { final CleanStatus translogStatus = translogCleanStatus.v1(); if (indexStatus == CleanStatus.CLEAN && translogStatus == CleanStatus.CLEAN) { - throw new ElasticsearchException("Shard does not seem to be corrupted at " + shardPath.getDataPath()); + throw new ElasticsearchException("Shard does not seem to be corrupted at " + shardPath.getDataPath() + + " (pass --" + TRUNCATE_CLEAN_TRANSLOG_FLAG + " to truncate the translog anyway)"); } if (indexStatus == CleanStatus.UNRECOVERABLE) { @@ -493,7 +500,7 @@ private void printRerouteCommand(ShardPath shardPath, Terminal terminal, boolean terminal.println(""); terminal.println("POST /_cluster/reroute\n" + Strings.toString(commands, true, true)); terminal.println(""); - terminal.println("You must accept the possibility of data loss by changing parameter `accept_data_loss` to `true`."); + terminal.println("You must accept the possibility of data loss by changing the `accept_data_loss` parameter to `true`."); terminal.println(""); } @@ -509,7 +516,8 @@ public enum CleanStatus { CLEAN("clean"), CLEAN_WITH_CORRUPTED_MARKER("marked corrupted, but no corruption detected"), CORRUPTED("corrupted"), - UNRECOVERABLE("corrupted and unrecoverable"); + UNRECOVERABLE("corrupted and unrecoverable"), + OVERRIDDEN("to be truncated regardless of whether it is corrupt"); private final String msg; diff --git a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java index 38d306596ba11..847c4543f63b0 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/RemoveCorruptedShardDataCommandTests.java @@ -56,6 +56,8 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.elasticsearch.index.shard.RemoveCorruptedShardDataCommand.TRUNCATE_CLEAN_TRANSLOG_FLAG; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; @@ -349,6 +351,39 @@ public void testResolveIndexDirectory() throws Exception { shardPath -> assertThat(shardPath.resolveIndex(), equalTo(indexPath))); } + public void testFailsOnCleanIndex() throws Exception { + indexDocs(indexShard, true); + closeShards(indexShard); + + final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand(); + final MockTerminal t = new MockTerminal(); + final OptionParser parser = command.getParser(); + + final OptionSet options = parser.parse("-d", translogPath.toString()); + t.setVerbosity(Terminal.Verbosity.VERBOSE); + assertThat(expectThrows(ElasticsearchException.class, () -> command.execute(t, options, environment)).getMessage(), + allOf(containsString("Shard does not seem to be corrupted"), containsString("--" + TRUNCATE_CLEAN_TRANSLOG_FLAG))); + assertThat(t.getOutput(), containsString("Lucene index is clean")); + assertThat(t.getOutput(), containsString("Translog is clean")); + } + + public void testTruncatesCleanTranslogIfRequested() throws Exception { + indexDocs(indexShard, true); + closeShards(indexShard); + + final RemoveCorruptedShardDataCommand command = new RemoveCorruptedShardDataCommand(); + final MockTerminal t = new MockTerminal(); + final OptionParser parser = command.getParser(); + + final OptionSet options = parser.parse("-d", translogPath.toString(), "--" + TRUNCATE_CLEAN_TRANSLOG_FLAG); + t.addTextInput("y"); + t.setVerbosity(Terminal.Verbosity.VERBOSE); + command.execute(t, options, environment); + assertThat(t.getOutput(), containsString("Lucene index is clean")); + assertThat(t.getOutput(), containsString("Translog was not analysed and will be truncated")); + assertThat(t.getOutput(), containsString("Creating new empty translog")); + } + public void testCleanWithCorruptionMarker() throws Exception { // index some docs in several segments final int numDocs = indexDocs(indexShard, true);