From 67a4fba1a0fdadac19db3688cdb8be7f18f27103 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 29 Jan 2019 16:09:47 +0100 Subject: [PATCH 01/10] detach-cluster tool --- .../coordination/CoordinationMetaData.java | 2 + .../coordination/DetachClusterCommand.java | 79 ++++ .../ElasticsearchNodeCommand.java | 151 ++++++ .../cluster/coordination/NodeToolCli.java | 1 + .../UnsafeBootstrapMasterCommand.java | 100 +--- .../ElasticsearchNodeCommandIT.java | 440 ++++++++++++++++++ .../coordination/UnsafeBootstrapMasterIT.java | 262 ----------- 7 files changed, 693 insertions(+), 342 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java create mode 100644 server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java create mode 100644 server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java delete mode 100644 server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java index 01ef85b656d1e..b63cb07feff99 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/CoordinationMetaData.java @@ -325,6 +325,8 @@ public String toString() { public static class VotingConfiguration implements Writeable, ToXContentFragment { public static final VotingConfiguration EMPTY_CONFIG = new VotingConfiguration(Collections.emptySet()); + public static final VotingConfiguration MUST_JOIN_ELECTED_MASTER = new VotingConfiguration(Collections.singleton( + "_must_join_elected_master_")); private final Set nodeIds; diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java new file mode 100644 index 0000000000000..7bb7629938780 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java @@ -0,0 +1,79 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.coordination; + +import joptsimple.OptionSet; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.cluster.metadata.Manifest; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.env.Environment; + +import java.io.IOException; +import java.nio.file.Path; + +public class DetachClusterCommand extends ElasticsearchNodeCommand { + + static final String NODE_DETACHED_MSG = "Node was successfully detached from the cluster"; + static final String CONFIRMATION_MSG = + "--------------------------------------------------------------------------\n" + + "\n" + + "You should run this tool only if you have permanently lost all\n" + + "your master-eligible nodes, and you cannot restore the cluster\n" + + "from a snapshot, or you have already run `elasticsearch-node unsafe-bootstrap`\n" + + "on master-eligible node that formed cluster with this node.\n" + + "This tool can result in arbitrary data loss and should be\n" + + "the last resort.\n" + + "Do you want to proceed?\n"; + + public DetachClusterCommand() { + super("Detaches this node from the cluster with old UUID, allowing it to join new cluster"); + } + + @Override + protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception { + super.execute(terminal, options, env); + + processNodePathsWithLock(terminal, options, env); + + terminal.println(NODE_DETACHED_MSG); + } + + @Override + protected void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOException { + final Tuple manifestMetaDataTuple = loadMetaData(terminal, dataPaths); + final Manifest manifest = manifestMetaDataTuple.v1(); + final MetaData metaData = manifestMetaDataTuple.v2(); + + confirm(terminal, CONFIRMATION_MSG); + + final CoordinationMetaData coordinationMetaData = CoordinationMetaData.builder() + .lastAcceptedConfiguration(CoordinationMetaData.VotingConfiguration.MUST_JOIN_ELECTED_MASTER) + .lastCommittedConfiguration(CoordinationMetaData.VotingConfiguration.MUST_JOIN_ELECTED_MASTER) + .build(); + final MetaData newMetaData = MetaData.builder(metaData) + .version(0) + .coordinationMetaData(coordinationMetaData) + .clusterUUID(MetaData.UNKNOWN_CLUSTER_UUID) + .clusterUUIDCommitted(false) + .build(); + + writeNewMetaData(terminal, manifest, 0, 0, metaData, newMetaData, dataPaths); + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java new file mode 100644 index 0000000000000..137d46c72b6a1 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java @@ -0,0 +1,151 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.coordination; + +import joptsimple.OptionParser; +import joptsimple.OptionSet; +import joptsimple.OptionSpec; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.store.LockObtainFailedException; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cli.EnvironmentAwareCommand; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.cluster.ClusterModule; +import org.elasticsearch.cluster.metadata.Manifest; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Objects; + +public abstract class ElasticsearchNodeCommand extends EnvironmentAwareCommand { + private static final Logger logger = LogManager.getLogger(ElasticsearchNodeCommand.class); + protected final NamedXContentRegistry namedXContentRegistry; + static final String STOP_WARNING_MSG = + "--------------------------------------------------------------------------\n" + + "\n" + + " WARNING: Elasticsearch MUST be stopped before running this tool." + + "\n"; + static final String FAILED_TO_OBTAIN_NODE_LOCK_MSG = "failed to lock node's directory, is Elasticsearch still running?"; + static final String NO_NODE_FOLDER_FOUND_MSG = "no node folder is found in data folder(s), node has not been started yet?"; + static final String NO_MANIFEST_FILE_FOUND_MSG = "no manifest file is found, do you run pre 7.0 Elasticsearch?"; + static final String GLOBAL_GENERATION_MISSING_MSG = "no metadata is referenced from the manifest file, cluster has never been " + + "bootstrapped?"; + static final String NO_GLOBAL_METADATA_MSG = "failed to find global metadata, metadata corrupted?"; + static final String WRITE_METADATA_EXCEPTION_MSG = "exception occurred when writing new metadata to disk"; + static final String ABORTED_BY_USER_MSG = "aborted by user"; + final OptionSpec nodeOrdinalOption; + + public ElasticsearchNodeCommand(String description) { + super(description); + nodeOrdinalOption = parser.accepts("ordinal", "Optional node ordinal, 0 if not specified") + .withRequiredArg().ofType(Integer.class); + namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables()); + } + + protected void processNodePathsWithLock(Terminal terminal, OptionSet options, Environment env) throws IOException { + terminal.println(Terminal.Verbosity.VERBOSE, "Obtaining lock for node"); + Integer nodeOrdinal = nodeOrdinalOption.value(options); + if (nodeOrdinal == null) { + nodeOrdinal = 0; + } + try (NodeEnvironment.NodeLock lock = new NodeEnvironment.NodeLock(nodeOrdinal, logger, env, Files::exists)) { + final Path[] dataPaths = + Arrays.stream(lock.getNodePaths()).filter(Objects::nonNull).map(p -> p.path).toArray(Path[]::new); + if (dataPaths.length == 0) { + throw new ElasticsearchException(NO_NODE_FOLDER_FOUND_MSG); + } + processNodePaths(terminal, dataPaths); + } catch (LockObtainFailedException ex) { + throw new ElasticsearchException( + FAILED_TO_OBTAIN_NODE_LOCK_MSG + " [" + ex.getMessage() + "]"); + } + } + + protected Tuple loadMetaData(Terminal terminal, Path[] dataPaths) throws IOException { + terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest file"); + final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths); + + if (manifest == null) { + throw new ElasticsearchException(NO_MANIFEST_FILE_FOUND_MSG); + } + if (manifest.isGlobalGenerationMissing()) { + throw new ElasticsearchException(GLOBAL_GENERATION_MISSING_MSG); + } + terminal.println(Terminal.Verbosity.VERBOSE, "Loading global metadata file"); + final MetaData metaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(), + dataPaths); + if (metaData == null) { + throw new ElasticsearchException(NO_GLOBAL_METADATA_MSG + " [generation = " + manifest.getGlobalGeneration() + "]"); + } + + return Tuple.tuple(manifest, metaData); + } + + protected void confirm(Terminal terminal, String msg) { + terminal.println(msg); + String text = terminal.readText("Confirm [y/N] "); + if (text.equalsIgnoreCase("y") == false) { + throw new ElasticsearchException(ABORTED_BY_USER_MSG); + } + } + + @Override + protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception { + terminal.println(STOP_WARNING_MSG); + } + + protected abstract void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOException; + + + protected void writeNewMetaData(Terminal terminal, Manifest oldManifest, long newCurrentTerm, long newVersion, + MetaData oldMetaData, MetaData newMetaData, Path[] dataPaths) { + try { + terminal.println(Terminal.Verbosity.VERBOSE, + "[clusterUUID = " + oldMetaData.clusterUUID() + ", committed = " + oldMetaData.clusterUUIDCommitted() + "] => " + + "[clusterUUID = " + newMetaData.clusterUUID() + ", committed = " + newMetaData.clusterUUIDCommitted() + "]"); + terminal.println(Terminal.Verbosity.VERBOSE, "New coordination metadata is " + newMetaData.coordinationMetaData()); + terminal.println(Terminal.Verbosity.VERBOSE, "Writing new global metadata to disk"); + long newGeneration = MetaData.FORMAT.write(newMetaData, dataPaths); + Manifest newManifest = new Manifest(newCurrentTerm, newVersion, newGeneration, + oldManifest.getIndexGenerations()); + terminal.println(Terminal.Verbosity.VERBOSE, "New manifest is " + newManifest); + terminal.println(Terminal.Verbosity.VERBOSE, "Writing new manifest file to disk"); + Manifest.FORMAT.writeAndCleanup(newManifest, dataPaths); + terminal.println(Terminal.Verbosity.VERBOSE, "Cleaning up old metadata"); + MetaData.FORMAT.cleanupOldFiles(newGeneration, dataPaths); + } catch (Exception e) { + terminal.println(Terminal.Verbosity.VERBOSE, "Cleaning up new metadata"); + MetaData.FORMAT.cleanupOldFiles(oldManifest.getGlobalGeneration(), dataPaths); + throw new ElasticsearchException(WRITE_METADATA_EXCEPTION_MSG, e); + } + } + + //package-private for testing + OptionParser getParser() { + return parser; + } +} diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java index d8fb77433faef..e2a94f1140b92 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeToolCli.java @@ -35,6 +35,7 @@ public NodeToolCli() { super("A CLI tool to unsafely recover a cluster after the permanent loss of too many master-eligible nodes", ()->{}); CommandLoggingConfigurator.configureLoggingWithoutConfig(); subcommands.put("unsafe-bootstrap", new UnsafeBootstrapMasterCommand()); + subcommands.put("detach-cluster", new DetachClusterCommand()); } public static void main(String[] args) throws Exception { diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java index 9db750c2a1f08..7ad38e98ef39c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java @@ -21,40 +21,27 @@ import joptsimple.OptionSet; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.store.LockObtainFailedException; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.cli.EnvironmentAwareCommand; import org.elasticsearch.cli.Terminal; -import org.elasticsearch.cluster.ClusterModule; import org.elasticsearch.cluster.metadata.Manifest; import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.NodeMetaData; import org.elasticsearch.node.Node; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; -import java.util.Arrays; import java.util.Collections; import java.util.Locale; -import java.util.Objects; -public class UnsafeBootstrapMasterCommand extends EnvironmentAwareCommand { +public class UnsafeBootstrapMasterCommand extends ElasticsearchNodeCommand { private static final Logger logger = LogManager.getLogger(UnsafeBootstrapMasterCommand.class); - private final NamedXContentRegistry namedXContentRegistry; - static final String STOP_WARNING_MSG = - "--------------------------------------------------------------------------\n" + - "\n" + - " WARNING: Elasticsearch MUST be stopped before running this tool." + - "\n"; static final String CLUSTER_STATE_TERM_VERSION_MSG_FORMAT = "Current node cluster state (term, version) pair is (%s, %s)"; static final String CONFIRMATION_MSG = @@ -67,30 +54,24 @@ public class UnsafeBootstrapMasterCommand extends EnvironmentAwareCommand { "If you have multiple survived master eligible nodes, consider running\n" + "this tool on the node with the highest cluster state (term, version) pair.\n" + "Do you want to proceed?\n"; - static final String ABORTED_BY_USER_MSG = "aborted by user"; static final String NOT_MASTER_NODE_MSG = "unsafe-bootstrap tool can only be run on master eligible node"; - static final String FAILED_TO_OBTAIN_NODE_LOCK_MSG = "failed to lock node's directory, is Elasticsearch still running?"; - static final String NO_NODE_FOLDER_FOUND_MSG = "no node folder is found in data folder(s), node has not been started yet?"; + static final String NO_NODE_METADATA_FOUND_MSG = "no node meta data is found, node has not been started yet?"; - static final String NO_MANIFEST_FILE_FOUND_MSG = "no manifest file is found, do you run pre 7.0 Elasticsearch?"; - static final String GLOBAL_GENERATION_MISSING_MSG = "no metadata is referenced from the manifest file, cluster has never been " + - "bootstrapped?"; - static final String NO_GLOBAL_METADATA_MSG = "failed to find global metadata, metadata corrupted?"; + static final String EMPTY_LAST_COMMITTED_VOTING_CONFIG_MSG = "last committed voting voting configuration is empty, cluster has never been bootstrapped?"; - static final String WRITE_METADATA_EXCEPTION_MSG = "exception occurred when writing new metadata to disk"; + static final String MASTER_NODE_BOOTSTRAPPED_MSG = "Master node was successfully bootstrapped"; static final Setting UNSAFE_BOOTSTRAP = ClusterService.USER_DEFINED_META_DATA.getConcreteSetting("cluster.metadata.unsafe-bootstrap"); UnsafeBootstrapMasterCommand() { super("Forces the successful election of the current node after the permanent loss of the half or more master-eligible nodes"); - namedXContentRegistry = new NamedXContentRegistry(ClusterModule.getNamedXWriteables()); } @Override protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception { - terminal.println(STOP_WARNING_MSG); + super.execute(terminal, options, env); Settings settings = env.settings(); terminal.println(Terminal.Verbosity.VERBOSE, "Checking node.master setting"); @@ -98,27 +79,13 @@ protected void execute(Terminal terminal, OptionSet options, Environment env) th if (master == false) { throw new ElasticsearchException(NOT_MASTER_NODE_MSG); } - final int nodeOrdinal = 0; - - terminal.println(Terminal.Verbosity.VERBOSE, "Obtaining lock for node"); - try (NodeEnvironment.NodeLock lock = new NodeEnvironment.NodeLock(nodeOrdinal, logger, env, Files::exists)) { - processNodePaths(logger, terminal, lock.getNodePaths()); - } catch (LockObtainFailedException ex) { - throw new ElasticsearchException( - FAILED_TO_OBTAIN_NODE_LOCK_MSG + " [" + ex.getMessage() + "]"); - } + processNodePathsWithLock(terminal, options, env); terminal.println(MASTER_NODE_BOOTSTRAPPED_MSG); } - private void processNodePaths(Logger logger, Terminal terminal, NodeEnvironment.NodePath[] nodePaths) throws IOException { - final Path[] dataPaths = - Arrays.stream(nodePaths).filter(Objects::nonNull).map(p -> p.path).toArray(Path[]::new); - if (dataPaths.length == 0) { - throw new ElasticsearchException(NO_NODE_FOLDER_FOUND_MSG); - } - + protected void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOException { terminal.println(Terminal.Verbosity.VERBOSE, "Loading node metadata"); final NodeMetaData nodeMetaData = NodeMetaData.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths); if (nodeMetaData == null) { @@ -127,21 +94,10 @@ private void processNodePaths(Logger logger, Terminal terminal, NodeEnvironment. String nodeId = nodeMetaData.nodeId(); terminal.println(Terminal.Verbosity.VERBOSE, "Current nodeId is " + nodeId); - terminal.println(Terminal.Verbosity.VERBOSE, "Loading manifest file"); - final Manifest manifest = Manifest.FORMAT.loadLatestState(logger, namedXContentRegistry, dataPaths); - if (manifest == null) { - throw new ElasticsearchException(NO_MANIFEST_FILE_FOUND_MSG); - } - if (manifest.isGlobalGenerationMissing()) { - throw new ElasticsearchException(GLOBAL_GENERATION_MISSING_MSG); - } - terminal.println(Terminal.Verbosity.VERBOSE, "Loading global metadata file"); - final MetaData metaData = MetaData.FORMAT.loadGeneration(logger, namedXContentRegistry, manifest.getGlobalGeneration(), - dataPaths); - if (metaData == null) { - throw new ElasticsearchException(NO_GLOBAL_METADATA_MSG + " [generation = " + manifest.getGlobalGeneration() + "]"); - } + final Tuple manifestMetaDataTuple = loadMetaData(terminal, dataPaths); + final Manifest manifest = manifestMetaDataTuple.v1(); + final MetaData metaData = manifestMetaDataTuple.v2(); final CoordinationMetaData coordinationMetaData = metaData.coordinationMetaData(); if (coordinationMetaData == null || coordinationMetaData.getLastCommittedConfiguration() == null || @@ -151,45 +107,29 @@ private void processNodePaths(Logger logger, Terminal terminal, NodeEnvironment. terminal.println(String.format(Locale.ROOT, CLUSTER_STATE_TERM_VERSION_MSG_FORMAT, coordinationMetaData.term(), metaData.version())); - terminal.println(CONFIRMATION_MSG); - String text = terminal.readText("Confirm [y/N] "); - if (text.equalsIgnoreCase("y") == false) { - throw new ElasticsearchException(ABORTED_BY_USER_MSG); - } + confirm(terminal, CONFIRMATION_MSG); CoordinationMetaData newCoordinationMetaData = CoordinationMetaData.builder(coordinationMetaData) .clearVotingConfigExclusions() .lastAcceptedConfiguration(new CoordinationMetaData.VotingConfiguration(Collections.singleton(nodeId))) .lastCommittedConfiguration(new CoordinationMetaData.VotingConfiguration(Collections.singleton(nodeId))) .build(); - terminal.println(Terminal.Verbosity.VERBOSE, "New coordination metadata is constructed " + newCoordinationMetaData); + Settings persistentSettings = Settings.builder() .put(metaData.persistentSettings()) .put(UNSAFE_BOOTSTRAP.getKey(), true) .build(); MetaData newMetaData = MetaData.builder(metaData) + .clusterUUID(MetaData.UNKNOWN_CLUSTER_UUID) + .generateClusterUuidIfNeeded() + .clusterUUIDCommitted(true) .persistentSettings(persistentSettings) .coordinationMetaData(newCoordinationMetaData) .build(); - writeNewMetaData(terminal, manifest, newMetaData, dataPaths); - } + terminal.println(Terminal.Verbosity.VERBOSE, + "Changing clusterUUID from " + metaData.clusterUUID() + " to " +newMetaData.clusterUUID()); - private void writeNewMetaData(Terminal terminal, Manifest manifest, MetaData newMetaData, Path[] dataPaths) { - try { - terminal.println(Terminal.Verbosity.VERBOSE, "Writing new global metadata to disk"); - long newGeneration = MetaData.FORMAT.write(newMetaData, dataPaths); - long newCurrentTerm = manifest.getCurrentTerm() + 1; - terminal.println(Terminal.Verbosity.VERBOSE, "Incrementing currentTerm. New value is " + newCurrentTerm); - Manifest newManifest = new Manifest(newCurrentTerm, manifest.getClusterStateVersion(), newGeneration, - manifest.getIndexGenerations()); - terminal.println(Terminal.Verbosity.VERBOSE, "Writing new manifest file to disk"); - Manifest.FORMAT.writeAndCleanup(newManifest, dataPaths); - terminal.println(Terminal.Verbosity.VERBOSE, "Cleaning up old metadata"); - MetaData.FORMAT.cleanupOldFiles(newGeneration, dataPaths); - } catch (Exception e) { - terminal.println(Terminal.Verbosity.VERBOSE, "Cleaning up new metadata"); - MetaData.FORMAT.cleanupOldFiles(manifest.getGlobalGeneration(), dataPaths); - throw new ElasticsearchException(WRITE_METADATA_EXCEPTION_MSG, e); - } + long newCurrentTerm = manifest.getCurrentTerm() + 1; + writeNewMetaData(terminal, manifest, newCurrentTerm, manifest.getClusterStateVersion(), metaData, newMetaData, dataPaths); } } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java new file mode 100644 index 0000000000000..461561893199c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java @@ -0,0 +1,440 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.cluster.coordination; + +import joptsimple.OptionSet; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; +import org.elasticsearch.cli.MockTerminal; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.Manifest; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.discovery.DiscoverySettings; +import org.elasticsearch.discovery.zen.ElectMasterService; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.env.NodeMetaData; +import org.elasticsearch.env.TestEnvironment; +import org.elasticsearch.node.Node; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalTestCluster; +import org.elasticsearch.test.junit.annotations.TestLogging; +import org.junit.Before; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false) +@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.discovery.zen:TRACE") +public class ElasticsearchNodeCommandIT extends ESIntegTestCase { + + private int bootstrapNodeId; + + @Before + public void resetBootstrapNodeId() { + bootstrapNodeId = -1; + } + + /** + * Performs cluster bootstrap when node with id bootstrapNodeId is started. + * Any node of the batch could be selected as bootstrap target. + */ + @Override + protected List addExtraClusterBootstrapSettings(List allNodesSettings) { + if (internalCluster().numMasterNodes() + allNodesSettings.size() == bootstrapNodeId) { + List nodeNames = new ArrayList<>(); + Collections.addAll(nodeNames, internalCluster().getNodeNames()); + allNodesSettings.forEach(settings -> nodeNames.add(Node.NODE_NAME_SETTING.get(settings))); + + List newSettings = new ArrayList<>(); + int bootstrapIndex = randomInt(allNodesSettings.size() - 1); + for (int i = 0; i < allNodesSettings.size(); i++) { + Settings nodeSettings = allNodesSettings.get(i); + if (i == bootstrapIndex) { + newSettings.add(Settings.builder().put(nodeSettings) + .putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(), nodeNames) + .build()); + } else { + newSettings.add(nodeSettings); + } + } + + return newSettings; + } + return allNodesSettings; + } + + private MockTerminal executeCommand(ElasticsearchNodeCommand command, Environment environment, int nodeOrdinal, boolean abort) + throws Exception { + final MockTerminal terminal = new MockTerminal(); + final OptionSet options = command.getParser().parse("-ordinal", Integer.toString(nodeOrdinal)); + final String input; + + if (abort) { + input = randomValueOtherThanMany(c -> c.equalsIgnoreCase("y"), () -> randomAlphaOfLength(1)); + } else { + input = randomBoolean() ? "y" : "Y"; + } + + terminal.addTextInput(input); + + try { + command.execute(terminal, options, environment); + } finally { + assertThat(terminal.getOutput(), containsString(ElasticsearchNodeCommand.STOP_WARNING_MSG)); + } + + return terminal; + } + + private MockTerminal unsafeBootstrap(Environment environment, int nodeOrdinal, boolean abort) throws Exception { + final MockTerminal terminal = executeCommand(new UnsafeBootstrapMasterCommand(), environment, nodeOrdinal, abort); + assertThat(terminal.getOutput(), containsString(UnsafeBootstrapMasterCommand.CONFIRMATION_MSG)); + assertThat(terminal.getOutput(), containsString(UnsafeBootstrapMasterCommand.MASTER_NODE_BOOTSTRAPPED_MSG)); + return terminal; + } + + private MockTerminal detachCluster(Environment environment, int nodeOrdinal, boolean abort) throws Exception { + final MockTerminal terminal = executeCommand(new DetachClusterCommand(), environment, nodeOrdinal, abort); + assertThat(terminal.getOutput(), containsString(DetachClusterCommand.CONFIRMATION_MSG)); + assertThat(terminal.getOutput(), containsString(DetachClusterCommand.NODE_DETACHED_MSG)); + return terminal; + } + + private MockTerminal unsafeBootstrap(Environment environment) throws Exception { + return unsafeBootstrap(environment, 0, false); + } + + private MockTerminal detachCluster(Environment environment) throws Exception { + return detachCluster(environment, 0, false); + } + + private void expectThrows(ThrowingRunnable runnable, String message) { + ElasticsearchException ex = expectThrows(ElasticsearchException.class, runnable); + assertThat(ex.getMessage(), containsString(message)); + } + + public void testBootstrapNotMasterEligible() { + final Environment environment = TestEnvironment.newEnvironment(Settings.builder() + .put(internalCluster().getDefaultSettings()) + .put(Node.NODE_MASTER_SETTING.getKey(), false) + .build()); + expectThrows(() -> unsafeBootstrap(environment), UnsafeBootstrapMasterCommand.NOT_MASTER_NODE_MSG); + } + + public void testBootstrapNoDataFolder() { + final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); + expectThrows(() -> unsafeBootstrap(environment), ElasticsearchNodeCommand.NO_NODE_FOLDER_FOUND_MSG); + } + + public void testDetachNoDataFolder() { + final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); + expectThrows(() -> detachCluster(environment), ElasticsearchNodeCommand.NO_NODE_FOLDER_FOUND_MSG); + } + + public void testBootstrapNodeLocked() throws IOException { + Settings envSettings = buildEnvSettings(Settings.EMPTY); + Environment environment = TestEnvironment.newEnvironment(envSettings); + try (NodeEnvironment ignored = new NodeEnvironment(envSettings, environment)) { + expectThrows(() -> unsafeBootstrap(environment), ElasticsearchNodeCommand.FAILED_TO_OBTAIN_NODE_LOCK_MSG); + } + } + + public void testDetachNodeLocked() throws IOException { + Settings envSettings = buildEnvSettings(Settings.EMPTY); + Environment environment = TestEnvironment.newEnvironment(envSettings); + try (NodeEnvironment ignored = new NodeEnvironment(envSettings, environment)) { + expectThrows(() -> detachCluster(environment), ElasticsearchNodeCommand.FAILED_TO_OBTAIN_NODE_LOCK_MSG); + } + } + + public void testBootstrapNoNodeMetaData() throws IOException { + Settings envSettings = buildEnvSettings(Settings.EMPTY); + Environment environment = TestEnvironment.newEnvironment(envSettings); + try (NodeEnvironment nodeEnvironment = new NodeEnvironment(envSettings, environment)) { + NodeMetaData.FORMAT.cleanupOldFiles(-1, nodeEnvironment.nodeDataPaths()); + } + + expectThrows(() -> unsafeBootstrap(environment), UnsafeBootstrapMasterCommand.NO_NODE_METADATA_FOUND_MSG); + } + + public void testBootstrapNotBootstrappedCluster() throws Exception { + internalCluster().startNode( + Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") // to ensure quick node startup + .build()); + assertBusy(() -> { + ClusterState state = client().admin().cluster().prepareState().setLocal(true) + .execute().actionGet().getState(); + assertTrue(state.blocks().hasGlobalBlockWithId(DiscoverySettings.NO_MASTER_BLOCK_ID)); + }); + + internalCluster().stopRandomDataNode(); + + Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); + expectThrows(() -> unsafeBootstrap(environment), ElasticsearchNodeCommand.GLOBAL_GENERATION_MISSING_MSG); + } + + public void testDetachNotBootstrappedCluster() throws Exception { + internalCluster().startNode( + Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") // to ensure quick node startup + .build()); + assertBusy(() -> { + ClusterState state = client().admin().cluster().prepareState().setLocal(true) + .execute().actionGet().getState(); + assertTrue(state.blocks().hasGlobalBlockWithId(DiscoverySettings.NO_MASTER_BLOCK_ID)); + }); + + internalCluster().stopRandomDataNode(); + + Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); + expectThrows(() -> detachCluster(environment), ElasticsearchNodeCommand.GLOBAL_GENERATION_MISSING_MSG); + } + + public void testBootstrapNoManifestFile() throws IOException { + bootstrapNodeId = 1; + internalCluster().startNode(Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); + ensureStableCluster(1); + NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); + internalCluster().stopRandomDataNode(); + Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); + Manifest.FORMAT.cleanupOldFiles(-1, nodeEnvironment.nodeDataPaths()); + + expectThrows(() -> unsafeBootstrap(environment), ElasticsearchNodeCommand.NO_MANIFEST_FILE_FOUND_MSG); + } + + public void testDetachNoManifestFile() throws IOException { + bootstrapNodeId = 1; + internalCluster().startNode(Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); + ensureStableCluster(1); + NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); + internalCluster().stopRandomDataNode(); + Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); + Manifest.FORMAT.cleanupOldFiles(-1, nodeEnvironment.nodeDataPaths()); + + expectThrows(() -> detachCluster(environment), ElasticsearchNodeCommand.NO_MANIFEST_FILE_FOUND_MSG); + } + + public void testBootstrapNoMetaData() throws IOException { + bootstrapNodeId = 1; + internalCluster().startNode(Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); + ensureStableCluster(1); + NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); + internalCluster().stopRandomDataNode(); + + Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); + MetaData.FORMAT.cleanupOldFiles(-1, nodeEnvironment.nodeDataPaths()); + + expectThrows(() -> unsafeBootstrap(environment), ElasticsearchNodeCommand.NO_GLOBAL_METADATA_MSG); + } + + public void testDetachNoMetaData() throws IOException { + bootstrapNodeId = 1; + internalCluster().startNode(Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); + ensureStableCluster(1); + NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); + internalCluster().stopRandomDataNode(); + + Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); + MetaData.FORMAT.cleanupOldFiles(-1, nodeEnvironment.nodeDataPaths()); + + expectThrows(() -> detachCluster(environment), ElasticsearchNodeCommand.NO_GLOBAL_METADATA_MSG); + } + + public void testBootstrapAbortedByUser() throws IOException { + bootstrapNodeId = 1; + internalCluster().startNode(Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); + ensureStableCluster(1); + internalCluster().stopRandomDataNode(); + + Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); + expectThrows(() -> unsafeBootstrap(environment, 0, true), ElasticsearchNodeCommand.ABORTED_BY_USER_MSG); + } + + public void testDetachAbortedByUser() throws IOException { + bootstrapNodeId = 1; + internalCluster().startNode(Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); + ensureStableCluster(1); + internalCluster().stopRandomDataNode(); + + Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); + expectThrows(() -> detachCluster(environment, 0, true), ElasticsearchNodeCommand.ABORTED_BY_USER_MSG); + } + + public void test3MasterNodes2Failed() throws Exception { + bootstrapNodeId = 3; + List masterNodes = new ArrayList<>(); + + logger.info("--> start 1st master-eligible node"); + masterNodes.add(internalCluster().startMasterOnlyNode(Settings.builder() + .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build())); // node ordinal 0 + + logger.info("--> start one data-only node"); + String dataNode = internalCluster().startDataOnlyNode(Settings.builder() + .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); // node ordinal 1 + + logger.info("--> start 2nd and 3rd master-eligible nodes and bootstrap"); + masterNodes.addAll(internalCluster().startMasterOnlyNodes(2, Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build())); // node ordinals 2 and 3 + + logger.info("--> create index test"); + createIndex("test"); + + logger.info("--> stop 2nd and 3d master eligible node"); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(1))); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(2))); + + logger.info("--> ensure NO_MASTER_BLOCK on data-only node"); + assertBusy(() -> { + ClusterState state = internalCluster().client(dataNode).admin().cluster().prepareState().setLocal(true) + .execute().actionGet().getState(); + assertTrue(state.blocks().hasGlobalBlockWithId(DiscoverySettings.NO_MASTER_BLOCK_ID)); + }); + + logger.info("--> try to unsafely bootstrap 1st master-eligible node, while node lock is held"); + final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); + expectThrows(() -> unsafeBootstrap(environment), UnsafeBootstrapMasterCommand.FAILED_TO_OBTAIN_NODE_LOCK_MSG); + + logger.info("--> stop 1st master-eligible node and data-only node"); + NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(0))); + internalCluster().stopRandomDataNode(); + + logger.info("--> unsafely-bootstrap 1st master-eligible node"); + MockTerminal terminal = unsafeBootstrap(environment); + MetaData metaData = MetaData.FORMAT.loadLatestState(logger, xContentRegistry(), nodeEnvironment.nodeDataPaths()); + assertThat(terminal.getOutput(), containsString( + String.format(Locale.ROOT, UnsafeBootstrapMasterCommand.CLUSTER_STATE_TERM_VERSION_MSG_FORMAT, + metaData.coordinationMetaData().term(), metaData.version()))); + + logger.info("--> start 1st master-eligible node"); + internalCluster().startMasterOnlyNode(Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); + + logger.info("--> detach-cluster on data-only node"); + detachCluster(environment, 1, false); + + logger.info("--> start data-only node"); + String dataNode2 = internalCluster().startDataOnlyNode(Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); + + logger.info("--> ensure there is no NO_MASTER_BLOCK and unsafe-bootstrap is reflected in cluster state"); + assertBusy(() -> { + ClusterState state = internalCluster().client(dataNode2).admin().cluster().prepareState().setLocal(true) + .execute().actionGet().getState(); + assertFalse(state.blocks().hasGlobalBlockWithId(DiscoverySettings.NO_MASTER_BLOCK_ID)); + assertTrue(state.metaData().persistentSettings().getAsBoolean(UnsafeBootstrapMasterCommand.UNSAFE_BOOTSTRAP.getKey(), false)); + }); + + logger.info("--> ensure index test is green"); + ensureGreen("test"); + + logger.info("--> detach-cluster on 2nd and 3rd master-eligible nodes"); + detachCluster(environment, 2, false); + detachCluster(environment, 3, false); + + logger.info("--> start 2nd and 3rd master-eligible nodes and ensure 4 nodes stable cluster"); + internalCluster().startMasterOnlyNodes(2, Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); + ensureStableCluster(4); + } + + public void testNoInitialBootstrapAfterDetach() throws Exception { + bootstrapNodeId = 1; + internalCluster().startMasterOnlyNode(Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); + internalCluster().stopCurrentMasterNode(); + + final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); + detachCluster(environment); + + String node = internalCluster().startMasterOnlyNode(Settings.builder() + // give the cluster 2 seconds to elect the master (it should not) + .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "2s") + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); + + ClusterState state = internalCluster().client().admin().cluster().prepareState().setLocal(true) + .execute().actionGet().getState(); + assertTrue(state.blocks().hasGlobalBlockWithId(DiscoverySettings.NO_MASTER_BLOCK_ID)); + + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node)); + } + + public void testCanRunUnsafeBootstrapAfterErroneousDetachWithoutLoosingMetaData() throws Exception { + bootstrapNodeId = 1; + internalCluster().startMasterOnlyNode(Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); + ClusterUpdateSettingsRequest req = new ClusterUpdateSettingsRequest().persistentSettings( + Settings.builder().put(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), "1234kb")); + internalCluster().client().admin().cluster().updateSettings(req).get(); + + ClusterState state = internalCluster().client().admin().cluster().prepareState().execute().actionGet().getState(); + assertThat(state.metaData().persistentSettings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()), + equalTo("1234kb")); + + internalCluster().stopCurrentMasterNode(); + + final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); + detachCluster(environment); + unsafeBootstrap(environment); + + internalCluster().startMasterOnlyNode(Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); + ensureStableCluster(1); + + state = internalCluster().client().admin().cluster().prepareState().execute().actionGet().getState(); + assertThat(state.metaData().settings().get(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey()), + equalTo("1234kb")); + } +} diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java deleted file mode 100644 index 73add5ba83520..0000000000000 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.elasticsearch.cluster.coordination; - -import joptsimple.OptionParser; -import joptsimple.OptionSet; -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.cli.MockTerminal; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.Manifest; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.discovery.DiscoverySettings; -import org.elasticsearch.discovery.zen.ElectMasterService; -import org.elasticsearch.env.Environment; -import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.env.NodeMetaData; -import org.elasticsearch.env.TestEnvironment; -import org.elasticsearch.node.Node; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalTestCluster; -import org.elasticsearch.test.junit.annotations.TestLogging; -import org.junit.Before; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Locale; - -import static org.hamcrest.Matchers.containsString; - -@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, autoMinMasterNodes = false) -@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.discovery.zen:TRACE") -public class UnsafeBootstrapMasterIT extends ESIntegTestCase { - - private int bootstrapNodeId; - - @Before - public void resetBootstrapNodeId() { - bootstrapNodeId = -1; - } - - /** - * Performs cluster bootstrap when node with id bootstrapNodeId is started. - * Any node of the batch could be selected as bootstrap target. - */ - @Override - protected List addExtraClusterBootstrapSettings(List allNodesSettings) { - if (internalCluster().size() + allNodesSettings.size() == bootstrapNodeId) { - List nodeNames = new ArrayList<>(); - Collections.addAll(nodeNames, internalCluster().getNodeNames()); - allNodesSettings.forEach(settings -> nodeNames.add(Node.NODE_NAME_SETTING.get(settings))); - - List newSettings = new ArrayList<>(); - int bootstrapIndex = randomInt(allNodesSettings.size() - 1); - for (int i = 0; i < allNodesSettings.size(); i++) { - Settings nodeSettings = allNodesSettings.get(i); - if (i == bootstrapIndex) { - newSettings.add(Settings.builder().put(nodeSettings) - .putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(), nodeNames) - .build()); - } else { - newSettings.add(nodeSettings); - } - } - - return newSettings; - } - return allNodesSettings; - } - - private MockTerminal executeCommand(Environment environment, boolean abort) throws Exception { - final UnsafeBootstrapMasterCommand command = new UnsafeBootstrapMasterCommand(); - final MockTerminal terminal = new MockTerminal(); - final OptionParser parser = new OptionParser(); - final OptionSet options = parser.parse(); - final String input; - - if (abort) { - input = randomValueOtherThanMany(c -> c.equalsIgnoreCase("y"), () -> randomAlphaOfLength(1)); - } else { - input = randomBoolean() ? "y" : "Y"; - } - - terminal.addTextInput(input); - - try { - command.execute(terminal, options, environment); - assertThat(terminal.getOutput(), containsString(UnsafeBootstrapMasterCommand.MASTER_NODE_BOOTSTRAPPED_MSG)); - } finally { - assertThat(terminal.getOutput(), containsString(UnsafeBootstrapMasterCommand.STOP_WARNING_MSG)); - } - - return terminal; - } - - private MockTerminal executeCommand(Environment environment) throws Exception { - return executeCommand(environment, false); - } - - private void expectThrows(ThrowingRunnable runnable, String message) { - ElasticsearchException ex = expectThrows(ElasticsearchException.class, runnable); - assertThat(ex.getMessage(), containsString(message)); - } - - public void testNotMasterEligible() { - final Environment environment = TestEnvironment.newEnvironment(Settings.builder() - .put(internalCluster().getDefaultSettings()) - .put(Node.NODE_MASTER_SETTING.getKey(), false) - .build()); - expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.NOT_MASTER_NODE_MSG); - } - - public void testNoDataFolder() { - final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); - expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.NO_NODE_FOLDER_FOUND_MSG); - } - - public void testNodeLocked() throws IOException { - Settings envSettings = buildEnvSettings(Settings.EMPTY); - Environment environment = TestEnvironment.newEnvironment(envSettings); - try (NodeEnvironment ignored = new NodeEnvironment(envSettings, environment)) { - expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.FAILED_TO_OBTAIN_NODE_LOCK_MSG); - } - } - - public void testNoNodeMetaData() throws IOException { - Settings envSettings = buildEnvSettings(Settings.EMPTY); - Environment environment = TestEnvironment.newEnvironment(envSettings); - try (NodeEnvironment nodeEnvironment = new NodeEnvironment(envSettings, environment)) { - NodeMetaData.FORMAT.cleanupOldFiles(-1, nodeEnvironment.nodeDataPaths()); - } - - expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.NO_NODE_METADATA_FOUND_MSG); - } - - public void testNotBootstrappedCluster() throws Exception { - internalCluster().startNode( - Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") // to ensure quick node startup - .build()); - assertBusy(() -> { - ClusterState state = client().admin().cluster().prepareState().setLocal(true) - .execute().actionGet().getState(); - assertTrue(state.blocks().hasGlobalBlockWithId(DiscoverySettings.NO_MASTER_BLOCK_ID)); - }); - - internalCluster().stopRandomDataNode(); - - Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); - expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.GLOBAL_GENERATION_MISSING_MSG); - } - - public void testNoManifestFile() throws IOException { - bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); - ensureStableCluster(1); - NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); - internalCluster().stopRandomDataNode(); - Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); - Manifest.FORMAT.cleanupOldFiles(-1, nodeEnvironment.nodeDataPaths()); - - expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.NO_MANIFEST_FILE_FOUND_MSG); - } - - public void testNoMetaData() throws IOException { - bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); - ensureStableCluster(1); - NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); - internalCluster().stopRandomDataNode(); - - Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); - MetaData.FORMAT.cleanupOldFiles(-1, nodeEnvironment.nodeDataPaths()); - - expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.NO_GLOBAL_METADATA_MSG); - } - - public void testAbortedByUser() throws IOException { - bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); - ensureStableCluster(1); - internalCluster().stopRandomDataNode(); - - Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); - expectThrows(() -> executeCommand(environment, true), UnsafeBootstrapMasterCommand.ABORTED_BY_USER_MSG); - } - - public void test3MasterNodes2Failed() throws Exception { - bootstrapNodeId = 3; - List masterNodes = internalCluster().startMasterOnlyNodes(3, Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); - - String dataNode = internalCluster().startDataOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); - createIndex("test"); - - Client dataNodeClient = internalCluster().client(dataNode); - - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(1))); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(2))); - - assertBusy(() -> { - ClusterState state = dataNodeClient.admin().cluster().prepareState().setLocal(true) - .execute().actionGet().getState(); - assertTrue(state.blocks().hasGlobalBlockWithId(DiscoverySettings.NO_MASTER_BLOCK_ID)); - }); - - final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); - expectThrows(() -> executeCommand(environment), UnsafeBootstrapMasterCommand.FAILED_TO_OBTAIN_NODE_LOCK_MSG); - - NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); - internalCluster().stopRandomNode(InternalTestCluster.nameFilter(masterNodes.get(0))); - - MockTerminal terminal = executeCommand(environment); - - MetaData metaData = MetaData.FORMAT.loadLatestState(logger, xContentRegistry(), nodeEnvironment.nodeDataPaths()); - assertThat(terminal.getOutput(), containsString( - String.format(Locale.ROOT, UnsafeBootstrapMasterCommand.CLUSTER_STATE_TERM_VERSION_MSG_FORMAT, - metaData.coordinationMetaData().term(), metaData.version()))); - - internalCluster().startMasterOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); - - assertBusy(() -> { - ClusterState state = dataNodeClient.admin().cluster().prepareState().setLocal(true) - .execute().actionGet().getState(); - assertFalse(state.blocks().hasGlobalBlockWithId(DiscoverySettings.NO_MASTER_BLOCK_ID)); - assertTrue(state.metaData().persistentSettings().getAsBoolean(UnsafeBootstrapMasterCommand.UNSAFE_BOOTSTRAP.getKey(), false)); - }); - - ensureGreen("test"); - } -} From f2050061cc74245d48c483d67acf3ea0aa4d31d8 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Tue, 29 Jan 2019 16:43:07 +0100 Subject: [PATCH 02/10] Remove terminal log line --- .../cluster/coordination/UnsafeBootstrapMasterCommand.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java index 7ad38e98ef39c..25c00ef068a1e 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java @@ -126,8 +126,6 @@ protected void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOEx .persistentSettings(persistentSettings) .coordinationMetaData(newCoordinationMetaData) .build(); - terminal.println(Terminal.Verbosity.VERBOSE, - "Changing clusterUUID from " + metaData.clusterUUID() + " to " +newMetaData.clusterUUID()); long newCurrentTerm = manifest.getCurrentTerm() + 1; writeNewMetaData(terminal, manifest, newCurrentTerm, manifest.getClusterStateVersion(), metaData, newMetaData, dataPaths); From 5572a835be478d6e370f5a3c57c6e59be8e6895c Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 30 Jan 2019 12:37:16 +0100 Subject: [PATCH 03/10] Make minimal disruption to cluster state UnsafeBootstrap - do not increment current term DetachCluster - do not reset MetaData and cluster state versions, do not change clusterUUID --- .../cluster/coordination/DetachClusterCommand.java | 5 ++--- .../cluster/coordination/ElasticsearchNodeCommand.java | 4 ++-- .../cluster/coordination/UnsafeBootstrapMasterCommand.java | 3 +-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java index 7bb7629938780..a7346bd2a56a9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java @@ -66,14 +66,13 @@ protected void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOEx final CoordinationMetaData coordinationMetaData = CoordinationMetaData.builder() .lastAcceptedConfiguration(CoordinationMetaData.VotingConfiguration.MUST_JOIN_ELECTED_MASTER) .lastCommittedConfiguration(CoordinationMetaData.VotingConfiguration.MUST_JOIN_ELECTED_MASTER) + .term(0) .build(); final MetaData newMetaData = MetaData.builder(metaData) - .version(0) .coordinationMetaData(coordinationMetaData) - .clusterUUID(MetaData.UNKNOWN_CLUSTER_UUID) .clusterUUIDCommitted(false) .build(); - writeNewMetaData(terminal, manifest, 0, 0, metaData, newMetaData, dataPaths); + writeNewMetaData(terminal, manifest, 0, metaData, newMetaData, dataPaths); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java index 137d46c72b6a1..9ef75879e9275 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommand.java @@ -121,7 +121,7 @@ protected void execute(Terminal terminal, OptionSet options, Environment env) th protected abstract void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOException; - protected void writeNewMetaData(Terminal terminal, Manifest oldManifest, long newCurrentTerm, long newVersion, + protected void writeNewMetaData(Terminal terminal, Manifest oldManifest, long newCurrentTerm, MetaData oldMetaData, MetaData newMetaData, Path[] dataPaths) { try { terminal.println(Terminal.Verbosity.VERBOSE, @@ -130,7 +130,7 @@ protected void writeNewMetaData(Terminal terminal, Manifest oldManifest, long ne terminal.println(Terminal.Verbosity.VERBOSE, "New coordination metadata is " + newMetaData.coordinationMetaData()); terminal.println(Terminal.Verbosity.VERBOSE, "Writing new global metadata to disk"); long newGeneration = MetaData.FORMAT.write(newMetaData, dataPaths); - Manifest newManifest = new Manifest(newCurrentTerm, newVersion, newGeneration, + Manifest newManifest = new Manifest(newCurrentTerm, oldManifest.getClusterStateVersion(), newGeneration, oldManifest.getIndexGenerations()); terminal.println(Terminal.Verbosity.VERBOSE, "New manifest is " + newManifest); terminal.println(Terminal.Verbosity.VERBOSE, "Writing new manifest file to disk"); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java index 25c00ef068a1e..b30a58fa73734 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java @@ -127,7 +127,6 @@ protected void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOEx .coordinationMetaData(newCoordinationMetaData) .build(); - long newCurrentTerm = manifest.getCurrentTerm() + 1; - writeNewMetaData(terminal, manifest, newCurrentTerm, manifest.getClusterStateVersion(), metaData, newMetaData, dataPaths); + writeNewMetaData(terminal, manifest, manifest.getCurrentTerm(), metaData, newMetaData, dataPaths); } } From 28940db934dd851bb989054a33a7a71a52063efb Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 30 Jan 2019 12:57:35 +0100 Subject: [PATCH 04/10] testAllMasterEligibleNodesFailed --- .../ElasticsearchNodeCommandIT.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java index 461561893199c..5b00b87d8f457 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java @@ -386,6 +386,40 @@ public void test3MasterNodes2Failed() throws Exception { ensureStableCluster(4); } + public void testAllMasterEligibleNodesFailed() throws Exception { + bootstrapNodeId = 1; + + logger.info("--> start master-eligible node and bootstrap cluster"); + String masterNode = internalCluster().startMasterOnlyNode(Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); // node ordinal 0 + + logger.info("--> start data-only node and ensure 2 nodes stable cluster"); + String dataNode = internalCluster().startDataOnlyNode(Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); // node ordinal 1 + ensureStableCluster(2); + + logger.info("--> stop data-only node and detach it from the old cluster"); + internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataNode)); + final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); + detachCluster(environment, 1, false); + + logger.info("--> stop master-eligible node, clear its data and start it again - new cluster should form"); + internalCluster().restartNode(masterNode, new InternalTestCluster.RestartCallback(){ + @Override + public boolean clearData(String nodeName) { + return true; + } + }); + + logger.info("--> start data-only only node and ensure 2 nodes stable cluster"); + internalCluster().startDataOnlyNode(Settings.builder() + .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) + .build()); + ensureStableCluster(2); + } + public void testNoInitialBootstrapAfterDetach() throws Exception { bootstrapNodeId = 1; internalCluster().startMasterOnlyNode(Settings.builder() From d1c3c2bfa93d713764d4a9f7cd8bfdf02e2640bd Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 30 Jan 2019 13:40:21 +0100 Subject: [PATCH 05/10] Extend testAllMasterEligibleNodesFailed Replaces testIndexImportedFromDataOnlyNodesIfMasterLostDataFolder and testDanglingIndices --- .../ElasticsearchNodeCommandIT.java | 26 +++++++-- .../discovery/ClusterDisruptionIT.java | 23 -------- .../gateway/GatewayIndexStateIT.java | 53 ------------------- 3 files changed, 23 insertions(+), 79 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java index 5b00b87d8f457..93b50f1d89cd7 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java @@ -44,7 +44,10 @@ import java.util.List; import java.util.Locale; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.elasticsearch.indices.recovery.RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -386,11 +389,11 @@ public void test3MasterNodes2Failed() throws Exception { ensureStableCluster(4); } - public void testAllMasterEligibleNodesFailed() throws Exception { + public void testAllMasterEligibleNodesFailedDanglingIndexImport() throws Exception { bootstrapNodeId = 1; - logger.info("--> start master-eligible node and bootstrap cluster"); - String masterNode = internalCluster().startMasterOnlyNode(Settings.builder() + logger.info("--> start mixed data and master-eligible node and bootstrap cluster"); + String masterNode = internalCluster().startNode(Settings.builder() .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) .build()); // node ordinal 0 @@ -400,6 +403,14 @@ public void testAllMasterEligibleNodesFailed() throws Exception { .build()); // node ordinal 1 ensureStableCluster(2); + logger.info("--> index 1 doc and ensure index is green"); + client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get(); + ensureGreen("test"); + + logger.info("--> verify 1 doc in the index"); + assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L); + assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(true)); + logger.info("--> stop data-only node and detach it from the old cluster"); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataNode)); final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); @@ -418,6 +429,15 @@ public boolean clearData(String nodeName) { .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) .build()); ensureStableCluster(2); + + logger.info("--> verify that the dangling index exists and has green status"); + assertBusy(() -> { + assertThat(client().admin().indices().prepareExists("test").execute().actionGet().isExists(), equalTo(true)); + }); + ensureGreen("test"); + + logger.info("--> verify the doc is there"); + assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(true)); } public void testNoInitialBootstrapAfterDetach() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java b/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java index 330c73b9c02c5..21dd102768e0f 100644 --- a/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java +++ b/server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java @@ -38,7 +38,6 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; -import org.elasticsearch.test.discovery.TestZenDiscovery; import org.elasticsearch.test.disruption.NetworkDisruption; import org.elasticsearch.test.disruption.NetworkDisruption.Bridge; import org.elasticsearch.test.disruption.NetworkDisruption.NetworkDisconnect; @@ -62,7 +61,6 @@ import static org.elasticsearch.action.DocWriteResponse.Result.CREATED; import static org.elasticsearch.action.DocWriteResponse.Result.UPDATED; -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -357,27 +355,6 @@ public void onFailure(Exception e) { } } - public void testIndexImportedFromDataOnlyNodesIfMasterLostDataFolder() throws Exception { - // test for https://github.com/elastic/elasticsearch/issues/8823 - Settings zen1Settings = Settings.builder().put(TestZenDiscovery.USE_ZEN2.getKey(), false).build(); // TODO: needs adaptions for Zen2 - String masterNode = internalCluster().startMasterOnlyNode(zen1Settings); - internalCluster().startDataOnlyNode(zen1Settings); - ensureStableCluster(2); - assertAcked(prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 0))); - index("index", "_doc", "1", jsonBuilder().startObject().field("text", "some text").endObject()); - ensureGreen(); - - internalCluster().restartNode(masterNode, new InternalTestCluster.RestartCallback() { - @Override - public boolean clearData(String nodeName) { - return true; - } - }); - - ensureGreen("index"); - assertTrue(client().prepareGet("index", "_doc", "1").get().isExists()); - } - public void testCannotJoinIfMasterLostDataFolder() throws Exception { String masterNode = internalCluster().startMasterOnlyNode(); String dataNode = internalCluster().startDataOnlyNode(); diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java index 0cddb929472b7..e13f252c52064 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayIndexStateIT.java @@ -37,7 +37,6 @@ import org.elasticsearch.cluster.routing.ShardRoutingState; import org.elasticsearch.common.Priority; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.discovery.zen.ElectMasterService; @@ -49,7 +48,6 @@ import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import org.elasticsearch.test.InternalTestCluster.RestartCallback; -import org.elasticsearch.test.discovery.TestZenDiscovery; import java.io.IOException; import java.util.List; @@ -275,57 +273,6 @@ public void testTwoNodesSingleDoc() throws Exception { } } - public void testDanglingIndices() throws Exception { - /*TODO This test test does not work with Zen2, because once master node looses its cluster state during restart - it will start with term = 1, which is the same as the term data node has. Data node won't accept cluster state from master - after the restart, because the term is the same, but version of the cluster state is greater on the data node. - Consider adding term to JoinRequest, so that master node can bump its term if its current term is less than JoinRequest#term. - */ - logger.info("--> starting two nodes"); - - final String node_1 = internalCluster().startNodes(2, - Settings.builder().put(TestZenDiscovery.USE_ZEN2.getKey(), false).build()).get(0); - - logger.info("--> indexing a simple document"); - client().prepareIndex("test", "type1", "1").setSource("field1", "value1").setRefreshPolicy(IMMEDIATE).get(); - - logger.info("--> waiting for green status"); - ensureGreen(); - - logger.info("--> verify 1 doc in the index"); - for (int i = 0; i < 10; i++) { - assertHitCount(client().prepareSearch().setQuery(matchAllQuery()).get(), 1L); - } - assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(true)); - - logger.info("--> restarting the nodes"); - internalCluster().fullRestart(new RestartCallback() { - @Override - public boolean clearData(String nodeName) { - return node_1.equals(nodeName); - } - }); - - logger.info("--> waiting for green status"); - ensureGreen(); - - // spin a bit waiting for the index to exists - long time = System.currentTimeMillis(); - while ((System.currentTimeMillis() - time) < TimeValue.timeValueSeconds(10).millis()) { - if (client().admin().indices().prepareExists("test").execute().actionGet().isExists()) { - break; - } - } - - logger.info("--> verify that the dangling index exists"); - assertThat(client().admin().indices().prepareExists("test").execute().actionGet().isExists(), equalTo(true)); - logger.info("--> waiting for green status"); - ensureGreen(); - - logger.info("--> verify the doc is there"); - assertThat(client().prepareGet("test", "type1", "1").execute().actionGet().isExists(), equalTo(true)); - } - /** * This test ensures that when an index deletion takes place while a node is offline, when that * node rejoins the cluster, it deletes the index locally instead of importing it as a dangling index. From a122221a0986982a692c0eb2dc673d93cc001365 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 30 Jan 2019 18:06:19 +0100 Subject: [PATCH 06/10] Special message for must join elected master in ClusterFormationHelper --- .../ClusterFormationFailureHelper.java | 6 ++++ .../ClusterFormationFailureHelperTests.java | 36 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java index cc58628b53893..67d2103ce672d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelper.java @@ -167,6 +167,12 @@ String getDescription() { assert clusterState.getLastCommittedConfiguration().isEmpty() == false; + if (clusterState.getLastCommittedConfiguration().equals(VotingConfiguration.MUST_JOIN_ELECTED_MASTER)) { + return String.format(Locale.ROOT, + "master not discovered yet and this node was detached from its previous cluster, have discovered %s; %s", + foundPeers, discoveryWillContinueDescription); + } + final String quorumDescription; if (clusterState.getLastAcceptedConfiguration().equals(clusterState.getLastCommittedConfiguration())) { quorumDescription = describeQuorum(clusterState.getLastAcceptedConfiguration()); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java index cf8e1737a7708..6e90aed5f74bf 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ClusterFormationFailureHelperTests.java @@ -201,6 +201,42 @@ private static ClusterState state(DiscoveryNode localNode, String[] acceptedConf .lastCommittedConfiguration(config(committedConfig)).build())).build(); } + + public void testDescriptionAfterDetachCluster() { + final DiscoveryNode localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); + + final ClusterState clusterState = state(localNode, + VotingConfiguration.MUST_JOIN_ELECTED_MASTER.getNodeIds().toArray(new String[0])); + + assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), emptyList(), 0L).getDescription(), + is("master not discovered yet and this node was detached from its previous cluster, " + + "have discovered []; " + + "discovery will continue using [] from hosts providers and [" + localNode + + "] from last-known cluster state; node term 0, last-accepted version 0 in term 0")); + + final TransportAddress otherAddress = buildNewFakeTransportAddress(); + assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, singletonList(otherAddress), emptyList(), 0L).getDescription(), + is("master not discovered yet and this node was detached from its previous cluster, " + + "have discovered []; " + + "discovery will continue using [" + otherAddress + "] from hosts providers and [" + localNode + + "] from last-known cluster state; node term 0, last-accepted version 0 in term 0")); + + final DiscoveryNode otherNode = new DiscoveryNode("otherNode", buildNewFakeTransportAddress(), Version.CURRENT); + assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), singletonList(otherNode), 0L).getDescription(), + is("master not discovered yet and this node was detached from its previous cluster, " + + "have discovered [" + otherNode + "]; " + + "discovery will continue using [] from hosts providers and [" + localNode + + "] from last-known cluster state; node term 0, last-accepted version 0 in term 0")); + + final DiscoveryNode yetAnotherNode = new DiscoveryNode("yetAnotherNode", buildNewFakeTransportAddress(), Version.CURRENT); + assertThat(new ClusterFormationState(Settings.EMPTY, clusterState, emptyList(), singletonList(yetAnotherNode), 0L).getDescription(), + is("master not discovered yet and this node was detached from its previous cluster, " + + "have discovered [" + yetAnotherNode + "]; " + + "discovery will continue using [] from hosts providers and [" + localNode + + "] from last-known cluster state; node term 0, last-accepted version 0 in term 0")); + + } + public void testDescriptionAfterBootstrapping() { final DiscoveryNode localNode = new DiscoveryNode("local", buildNewFakeTransportAddress(), Version.CURRENT); From 80ad2248d1469b1ae602e891e91f63c594076734 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Wed, 30 Jan 2019 18:13:24 +0100 Subject: [PATCH 07/10] Wording --- .../cluster/coordination/DetachClusterCommand.java | 9 ++++----- .../coordination/UnsafeBootstrapMasterCommand.java | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java index a7346bd2a56a9..9822a7780eeea 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java @@ -32,18 +32,17 @@ public class DetachClusterCommand extends ElasticsearchNodeCommand { static final String NODE_DETACHED_MSG = "Node was successfully detached from the cluster"; static final String CONFIRMATION_MSG = - "--------------------------------------------------------------------------\n" + + "-------------------------------------------------------------------------------\n" + "\n" + "You should run this tool only if you have permanently lost all\n" + "your master-eligible nodes, and you cannot restore the cluster\n" + "from a snapshot, or you have already run `elasticsearch-node unsafe-bootstrap`\n" + - "on master-eligible node that formed cluster with this node.\n" + - "This tool can result in arbitrary data loss and should be\n" + - "the last resort.\n" + + "on a master-eligible node that formed a cluster with this node.\n" + + "This tool can cause arbitrary data loss and its use should be your last resort.\n" + "Do you want to proceed?\n"; public DetachClusterCommand() { - super("Detaches this node from the cluster with old UUID, allowing it to join new cluster"); + super("Detaches this node from its cluster, allowing it to unsafely join a new cluster"); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java index b30a58fa73734..72afe8ec70428 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterCommand.java @@ -49,8 +49,8 @@ public class UnsafeBootstrapMasterCommand extends ElasticsearchNodeCommand { "\n" + "You should run this tool only if you have permanently lost half\n" + "or more of the master-eligible nodes, and you cannot restore the cluster\n" + - "from a snapshot. This tool can result in arbitrary data loss and\n" + - "should be the last resort.\n" + + "from a snapshot. This tool can cause arbitrary data loss and its use " + + "should be your last resort.\n" + "If you have multiple survived master eligible nodes, consider running\n" + "this tool on the node with the highest cluster state (term, version) pair.\n" + "Do you want to proceed?\n"; From b12f50b0be08f08b72ea6d47e312dd0d415443c6 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 31 Jan 2019 16:56:20 +0100 Subject: [PATCH 08/10] Address TODO in CoordinatorTests --- .../cluster/coordination/DetachClusterCommand.java | 12 ++++++++++-- .../cluster/coordination/CoordinatorTests.java | 10 ++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java b/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java index 9822a7780eeea..6bd41ccf37f0c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/DetachClusterCommand.java @@ -62,16 +62,24 @@ protected void processNodePaths(Terminal terminal, Path[] dataPaths) throws IOEx confirm(terminal, CONFIRMATION_MSG); + writeNewMetaData(terminal, manifest, updateCurrentTerm(), metaData, updateMetaData(metaData), dataPaths); + } + + // package-private for tests + static MetaData updateMetaData(MetaData oldMetaData) { final CoordinationMetaData coordinationMetaData = CoordinationMetaData.builder() .lastAcceptedConfiguration(CoordinationMetaData.VotingConfiguration.MUST_JOIN_ELECTED_MASTER) .lastCommittedConfiguration(CoordinationMetaData.VotingConfiguration.MUST_JOIN_ELECTED_MASTER) .term(0) .build(); - final MetaData newMetaData = MetaData.builder(metaData) + return MetaData.builder(oldMetaData) .coordinationMetaData(coordinationMetaData) .clusterUUIDCommitted(false) .build(); + } - writeNewMetaData(terminal, manifest, 0, metaData, newMetaData, dataPaths); + //package-private for tests + static long updateCurrentTerm() { + return 0; } } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java index c3028de1801da..93c89cfafabd5 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/CoordinatorTests.java @@ -1048,15 +1048,9 @@ public void testCannotJoinClusterWithDifferentUUID() throws IllegalAccessExcepti } assertTrue(newNode.getLastAppliedClusterState().version() == 0); - // reset clusterUUIDCommitted (and node / cluster state term) to let node join again - // TODO: use elasticsearch-node detach-cluster tool once it's implemented final ClusterNode detachedNode = newNode.restartedNode( - metaData -> MetaData.builder(metaData) - .clusterUUIDCommitted(false) - .coordinationMetaData(CoordinationMetaData.builder(metaData.coordinationMetaData()) - .term(0L).build()) - .build(), - term -> 0L); + metaData -> DetachClusterCommand.updateMetaData(metaData), + term -> DetachClusterCommand.updateCurrentTerm()); cluster1.clusterNodes.replaceAll(cn -> cn == newNode ? detachedNode : cn); cluster1.stabilise(); } From e1cfd09cbf57b3e62cb7c12b895f5dccd85b51b8 Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Thu, 31 Jan 2019 17:06:53 +0100 Subject: [PATCH 09/10] Fix merge, remove min master nodes setting from tests --- .../ElasticsearchNodeCommandIT.java | 70 +++++-------------- .../coordination/UnsafeBootstrapMasterIT.java | 0 2 files changed, 16 insertions(+), 54 deletions(-) delete mode 100644 server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java index 93b50f1d89cd7..ac66ff42d77ca 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java @@ -27,7 +27,6 @@ import org.elasticsearch.cluster.metadata.MetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.discovery.DiscoverySettings; -import org.elasticsearch.discovery.zen.ElectMasterService; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.env.NodeMetaData; @@ -188,7 +187,6 @@ public void testBootstrapNoNodeMetaData() throws IOException { public void testBootstrapNotBootstrappedCluster() throws Exception { internalCluster().startNode( Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") // to ensure quick node startup .build()); assertBusy(() -> { @@ -206,7 +204,6 @@ public void testBootstrapNotBootstrappedCluster() throws Exception { public void testDetachNotBootstrappedCluster() throws Exception { internalCluster().startNode( Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") // to ensure quick node startup .build()); assertBusy(() -> { @@ -223,9 +220,7 @@ public void testDetachNotBootstrappedCluster() throws Exception { public void testBootstrapNoManifestFile() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); internalCluster().stopRandomDataNode(); @@ -237,9 +232,7 @@ public void testBootstrapNoManifestFile() throws IOException { public void testDetachNoManifestFile() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); internalCluster().stopRandomDataNode(); @@ -251,9 +244,7 @@ public void testDetachNoManifestFile() throws IOException { public void testBootstrapNoMetaData() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); internalCluster().stopRandomDataNode(); @@ -266,9 +257,7 @@ public void testBootstrapNoMetaData() throws IOException { public void testDetachNoMetaData() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); internalCluster().stopRandomDataNode(); @@ -281,9 +270,7 @@ public void testDetachNoMetaData() throws IOException { public void testBootstrapAbortedByUser() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(); ensureStableCluster(1); internalCluster().stopRandomDataNode(); @@ -293,9 +280,7 @@ public void testBootstrapAbortedByUser() throws IOException { public void testDetachAbortedByUser() throws IOException { bootstrapNodeId = 1; - internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startNode(); ensureStableCluster(1); internalCluster().stopRandomDataNode(); @@ -310,19 +295,15 @@ public void test3MasterNodes2Failed() throws Exception { logger.info("--> start 1st master-eligible node"); masterNodes.add(internalCluster().startMasterOnlyNode(Settings.builder() .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) .build())); // node ordinal 0 logger.info("--> start one data-only node"); String dataNode = internalCluster().startDataOnlyNode(Settings.builder() .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "0s") - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) .build()); // node ordinal 1 logger.info("--> start 2nd and 3rd master-eligible nodes and bootstrap"); - masterNodes.addAll(internalCluster().startMasterOnlyNodes(2, Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build())); // node ordinals 2 and 3 + masterNodes.addAll(internalCluster().startMasterOnlyNodes(2)); // node ordinals 2 and 3 logger.info("--> create index test"); createIndex("test"); @@ -355,17 +336,13 @@ public void test3MasterNodes2Failed() throws Exception { metaData.coordinationMetaData().term(), metaData.version()))); logger.info("--> start 1st master-eligible node"); - internalCluster().startMasterOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startMasterOnlyNode(); logger.info("--> detach-cluster on data-only node"); detachCluster(environment, 1, false); logger.info("--> start data-only node"); - String dataNode2 = internalCluster().startDataOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + String dataNode2 = internalCluster().startDataOnlyNode(); logger.info("--> ensure there is no NO_MASTER_BLOCK and unsafe-bootstrap is reflected in cluster state"); assertBusy(() -> { @@ -383,9 +360,7 @@ public void test3MasterNodes2Failed() throws Exception { detachCluster(environment, 3, false); logger.info("--> start 2nd and 3rd master-eligible nodes and ensure 4 nodes stable cluster"); - internalCluster().startMasterOnlyNodes(2, Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startMasterOnlyNodes(2); ensureStableCluster(4); } @@ -393,14 +368,10 @@ public void testAllMasterEligibleNodesFailedDanglingIndexImport() throws Excepti bootstrapNodeId = 1; logger.info("--> start mixed data and master-eligible node and bootstrap cluster"); - String masterNode = internalCluster().startNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); // node ordinal 0 + String masterNode = internalCluster().startNode(); // node ordinal 0 logger.info("--> start data-only node and ensure 2 nodes stable cluster"); - String dataNode = internalCluster().startDataOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); // node ordinal 1 + String dataNode = internalCluster().startDataOnlyNode(); // node ordinal 1 ensureStableCluster(2); logger.info("--> index 1 doc and ensure index is green"); @@ -425,9 +396,7 @@ public boolean clearData(String nodeName) { }); logger.info("--> start data-only only node and ensure 2 nodes stable cluster"); - internalCluster().startDataOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startDataOnlyNode(); ensureStableCluster(2); logger.info("--> verify that the dangling index exists and has green status"); @@ -442,9 +411,7 @@ public boolean clearData(String nodeName) { public void testNoInitialBootstrapAfterDetach() throws Exception { bootstrapNodeId = 1; - internalCluster().startMasterOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startMasterOnlyNode(); internalCluster().stopCurrentMasterNode(); final Environment environment = TestEnvironment.newEnvironment(internalCluster().getDefaultSettings()); @@ -453,7 +420,6 @@ public void testNoInitialBootstrapAfterDetach() throws Exception { String node = internalCluster().startMasterOnlyNode(Settings.builder() // give the cluster 2 seconds to elect the master (it should not) .put(DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "2s") - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) .build()); ClusterState state = internalCluster().client().admin().cluster().prepareState().setLocal(true) @@ -465,9 +431,7 @@ public void testNoInitialBootstrapAfterDetach() throws Exception { public void testCanRunUnsafeBootstrapAfterErroneousDetachWithoutLoosingMetaData() throws Exception { bootstrapNodeId = 1; - internalCluster().startMasterOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startMasterOnlyNode(); ClusterUpdateSettingsRequest req = new ClusterUpdateSettingsRequest().persistentSettings( Settings.builder().put(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), "1234kb")); internalCluster().client().admin().cluster().updateSettings(req).get(); @@ -482,9 +446,7 @@ public void testCanRunUnsafeBootstrapAfterErroneousDetachWithoutLoosingMetaData( detachCluster(environment); unsafeBootstrap(environment); - internalCluster().startMasterOnlyNode(Settings.builder() - .put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), Integer.MAX_VALUE) - .build()); + internalCluster().startMasterOnlyNode(); ensureStableCluster(1); state = internalCluster().client().admin().cluster().prepareState().execute().actionGet().getState(); diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java deleted file mode 100644 index e69de29bb2d1d..0000000000000 From f84d472b2acceeb2314be58eacc575459048a74f Mon Sep 17 00:00:00 2001 From: Andrey Ershov Date: Fri, 1 Feb 2019 11:47:41 +0100 Subject: [PATCH 10/10] Use internalCluster.setBootstrapMasterNodeIndex --- .../ElasticsearchNodeCommandIT.java | 58 ++++--------------- 1 file changed, 10 insertions(+), 48 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java index ac66ff42d77ca..ae8eba050020a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/ElasticsearchNodeCommandIT.java @@ -35,11 +35,9 @@ import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.junit.annotations.TestLogging; -import org.junit.Before; import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Locale; @@ -54,42 +52,6 @@ @TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.discovery.zen:TRACE") public class ElasticsearchNodeCommandIT extends ESIntegTestCase { - private int bootstrapNodeId; - - @Before - public void resetBootstrapNodeId() { - bootstrapNodeId = -1; - } - - /** - * Performs cluster bootstrap when node with id bootstrapNodeId is started. - * Any node of the batch could be selected as bootstrap target. - */ - @Override - protected List addExtraClusterBootstrapSettings(List allNodesSettings) { - if (internalCluster().numMasterNodes() + allNodesSettings.size() == bootstrapNodeId) { - List nodeNames = new ArrayList<>(); - Collections.addAll(nodeNames, internalCluster().getNodeNames()); - allNodesSettings.forEach(settings -> nodeNames.add(Node.NODE_NAME_SETTING.get(settings))); - - List newSettings = new ArrayList<>(); - int bootstrapIndex = randomInt(allNodesSettings.size() - 1); - for (int i = 0; i < allNodesSettings.size(); i++) { - Settings nodeSettings = allNodesSettings.get(i); - if (i == bootstrapIndex) { - newSettings.add(Settings.builder().put(nodeSettings) - .putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(), nodeNames) - .build()); - } else { - newSettings.add(nodeSettings); - } - } - - return newSettings; - } - return allNodesSettings; - } - private MockTerminal executeCommand(ElasticsearchNodeCommand command, Environment environment, int nodeOrdinal, boolean abort) throws Exception { final MockTerminal terminal = new MockTerminal(); @@ -219,7 +181,7 @@ public void testDetachNotBootstrappedCluster() throws Exception { } public void testBootstrapNoManifestFile() throws IOException { - bootstrapNodeId = 1; + internalCluster().setBootstrapMasterNodeIndex(0); internalCluster().startNode(); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); @@ -231,7 +193,7 @@ public void testBootstrapNoManifestFile() throws IOException { } public void testDetachNoManifestFile() throws IOException { - bootstrapNodeId = 1; + internalCluster().setBootstrapMasterNodeIndex(0); internalCluster().startNode(); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); @@ -243,7 +205,7 @@ public void testDetachNoManifestFile() throws IOException { } public void testBootstrapNoMetaData() throws IOException { - bootstrapNodeId = 1; + internalCluster().setBootstrapMasterNodeIndex(0); internalCluster().startNode(); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); @@ -256,7 +218,7 @@ public void testBootstrapNoMetaData() throws IOException { } public void testDetachNoMetaData() throws IOException { - bootstrapNodeId = 1; + internalCluster().setBootstrapMasterNodeIndex(0); internalCluster().startNode(); ensureStableCluster(1); NodeEnvironment nodeEnvironment = internalCluster().getMasterNodeInstance(NodeEnvironment.class); @@ -269,7 +231,7 @@ public void testDetachNoMetaData() throws IOException { } public void testBootstrapAbortedByUser() throws IOException { - bootstrapNodeId = 1; + internalCluster().setBootstrapMasterNodeIndex(0); internalCluster().startNode(); ensureStableCluster(1); internalCluster().stopRandomDataNode(); @@ -279,7 +241,7 @@ public void testBootstrapAbortedByUser() throws IOException { } public void testDetachAbortedByUser() throws IOException { - bootstrapNodeId = 1; + internalCluster().setBootstrapMasterNodeIndex(0); internalCluster().startNode(); ensureStableCluster(1); internalCluster().stopRandomDataNode(); @@ -289,7 +251,7 @@ public void testDetachAbortedByUser() throws IOException { } public void test3MasterNodes2Failed() throws Exception { - bootstrapNodeId = 3; + internalCluster().setBootstrapMasterNodeIndex(2); List masterNodes = new ArrayList<>(); logger.info("--> start 1st master-eligible node"); @@ -365,7 +327,7 @@ public void test3MasterNodes2Failed() throws Exception { } public void testAllMasterEligibleNodesFailedDanglingIndexImport() throws Exception { - bootstrapNodeId = 1; + internalCluster().setBootstrapMasterNodeIndex(0); logger.info("--> start mixed data and master-eligible node and bootstrap cluster"); String masterNode = internalCluster().startNode(); // node ordinal 0 @@ -410,7 +372,7 @@ public boolean clearData(String nodeName) { } public void testNoInitialBootstrapAfterDetach() throws Exception { - bootstrapNodeId = 1; + internalCluster().setBootstrapMasterNodeIndex(0); internalCluster().startMasterOnlyNode(); internalCluster().stopCurrentMasterNode(); @@ -430,7 +392,7 @@ public void testNoInitialBootstrapAfterDetach() throws Exception { } public void testCanRunUnsafeBootstrapAfterErroneousDetachWithoutLoosingMetaData() throws Exception { - bootstrapNodeId = 1; + internalCluster().setBootstrapMasterNodeIndex(0); internalCluster().startMasterOnlyNode(); ClusterUpdateSettingsRequest req = new ClusterUpdateSettingsRequest().persistentSettings( Settings.builder().put(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), "1234kb"));