From 30ea08e730a4c17f60d013079b20d832df16caa0 Mon Sep 17 00:00:00 2001 From: David Turner Date: Sat, 2 Oct 2021 15:18:05 +0100 Subject: [PATCH] Prevent downgrades from 8.x to 7.x Users sometimes attempt to downgrade a node in place, but downgrades are totally untested and unsupported and generally don't work. We protect against this by recording the node version in the metadata and refusing to start if we encounter metadata written by a future version. However in 8.0 (#42489) we changed the directory layout so that a 7.x node won't find the upgraded metadata, or indeed any other data, and will proceed as if it's a fresh node. That's almost certainly not what the user wants, so with this commit we create a file at `${path.data}/nodes` at each startup, preventing an older node from starting. Closes #52414 --- .../elasticsearch/env/NodeEnvironmentIT.java | 10 ++++-- .../elasticsearch/env/NodeEnvironment.java | 35 ++++++++++--------- .../env/NodeEnvironmentTests.java | 30 ++++++++++------ 3 files changed, 45 insertions(+), 30 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java index 0fc7a9a03d942..d18dc6c6c9746 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java @@ -14,6 +14,7 @@ import org.elasticsearch.core.PathUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.gateway.PersistedClusterStateService; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESIntegTestCase; @@ -138,8 +139,11 @@ public void testUpgradeDataFolder() throws IOException, InterruptedException { // simulate older data path layout by moving data under "nodes/0" folder final List dataPaths = List.of(PathUtils.get(Environment.PATH_DATA_SETTING.get(dataPathSettings))); dataPaths.forEach(path -> { - final Path targetPath = path.resolve("nodes").resolve("0"); + final Path nodesPath = path.resolve("nodes"); + final Path targetPath = nodesPath.resolve("0"); try { + assertTrue(Files.isRegularFile(nodesPath)); + Files.delete(nodesPath); Files.createDirectories(targetPath); try (DirectoryStream stream = Files.newDirectoryStream(path)) { @@ -192,9 +196,9 @@ public void testUpgradeDataFolder() throws IOException, InterruptedException { } // check that upgrade works - dataPaths.forEach(path -> assertTrue(Files.exists(path.resolve("nodes")))); + dataPaths.forEach(path -> assertTrue(Files.isDirectory(path.resolve("nodes")))); internalCluster().startNode(dataPathSettings); - dataPaths.forEach(path -> assertFalse(Files.exists(path.resolve("nodes")))); + dataPaths.forEach(path -> assertTrue(Files.isRegularFile(path.resolve("nodes")))); assertEquals(nodeId, client().admin().cluster().prepareState().get().getState().nodes().getMasterNodeId()); assertTrue(indexExists("test")); ensureYellow("test"); diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 9bfe598dc0566..80ee60015b65d 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -55,10 +55,12 @@ import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.DirectoryStream; import java.nio.file.FileStore; import java.nio.file.Files; +import java.nio.file.OpenOption; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.util.ArrayList; @@ -269,6 +271,16 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce assertCanWrite(); } + // versions 7.x and earlier put their data under ${path.data}/nodes/; leave a file at that location to prevent downgrades + final Path legacyNodesPath = environment.dataFile().resolve("nodes"); + if (Files.isRegularFile(legacyNodesPath) == false) { + final String content = "written by Elasticsearch v" + Version.CURRENT + + " to prevent a downgrade to a version prior to v8.0.0 which would result in data loss"; + Files.write(legacyNodesPath, content.getBytes(StandardCharsets.UTF_8)); + IOUtils.fsync(legacyNodesPath, false); + IOUtils.fsync(environment.dataFile(), true); + } + if (DiscoveryNode.canContainData(settings) == false) { if (DiscoveryNode.isMasterNode(settings) == false) { ensureNoIndexMetadata(nodePath); @@ -347,7 +359,6 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings // move contents from legacy path to new path try { - final List> upgradeActions = new ArrayList<>(); final NodePath legacyNodePath = legacyNodeLock.getNodePath(); final NodePath nodePath = nodeLock.getNodePath(); @@ -398,22 +409,14 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings assert Sets.difference(folderNames, expectedFolderNames).isEmpty() : "expected indices and/or state dir folder but was " + folderNames; - upgradeActions.add(() -> { - for (String folderName : folderNames) { - final Path sourceSubFolderPath = legacyNodePath.path.resolve(folderName); - final Path targetSubFolderPath = nodePath.path.resolve(folderName); - Files.move(sourceSubFolderPath, targetSubFolderPath, StandardCopyOption.ATOMIC_MOVE); - logger.info("data folder upgrade: moved from [{}] to [{}]", sourceSubFolderPath, targetSubFolderPath); - } - IOUtils.fsync(nodePath.path, true); - }); - - // now do the actual upgrade. start by upgrading the node metadata file before moving anything, since a downgrade in an - // intermediate state would be pretty disastrous - loadNodeMetadata(settings, logger, legacyNodeLock.getNodePath()); - for (CheckedRunnable upgradeAction : upgradeActions) { - upgradeAction.run(); + // now do the actual upgrade + for (String folderName : folderNames) { + final Path sourceSubFolderPath = legacyNodePath.path.resolve(folderName); + final Path targetSubFolderPath = nodePath.path.resolve(folderName); + Files.move(sourceSubFolderPath, targetSubFolderPath, StandardCopyOption.ATOMIC_MOVE); + logger.info("data folder upgrade: moved from [{}] to [{}]", sourceSubFolderPath, targetSubFolderPath); } + IOUtils.fsync(nodePath.path, true); } finally { legacyNodeLock.close(); } diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index 5cf9c24aa2529..016329eca28c9 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.test.NodeRoles; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -41,6 +42,7 @@ import static org.elasticsearch.test.NodeRoles.nonDataNode; import static org.elasticsearch.test.NodeRoles.nonMasterNode; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.startsWith; @@ -439,6 +441,23 @@ public void testEnsureNoShardDataOrIndexMetadata() throws IOException { verifyFailsOnShardData(noDataNoMasterSettings, indexPath, shardDataDirName); } + public void testBlocksDowngradeToVersionWithMultipleNodesInDataPath() throws IOException { + final Settings settings = buildEnvSettings(Settings.EMPTY); + for (int i = 0; i < 2; i++) { // ensure the file gets created again if missing + try (NodeEnvironment env = newNodeEnvironment(settings)) { + final Path nodesPath = env.nodeDataPath().resolve("nodes"); + assertTrue(Files.isRegularFile(nodesPath)); + assertThat( + Files.readString(nodesPath, StandardCharsets.UTF_8), + allOf( + containsString("written by Elasticsearch"), + containsString("prevent a downgrade"), + containsString("data loss"))); + Files.delete(nodesPath); + } + } + } + private void verifyFailsOnShardData(Settings settings, Path indexPath, String shardDataDirName) { IllegalStateException ex = expectThrows(IllegalStateException.class, "Must fail creating NodeEnvironment on a data path that has shard data if node does not have data role", @@ -459,17 +478,6 @@ private void verifyFailsOnMetadata(Settings settings, Path indexPath) { assertThat(ex.getMessage(), startsWith("node does not have the data and master roles but has index metadata")); } - /** - * Converts an array of Strings to an array of Paths, adding an additional child if specified - */ - private Path[] stringsToPaths(String[] strings, String additional) { - Path[] locations = new Path[strings.length]; - for (int i = 0; i < strings.length; i++) { - locations[i] = PathUtils.get(strings[i], additional); - } - return locations; - } - @Override public NodeEnvironment newNodeEnvironment() throws IOException { return newNodeEnvironment(Settings.EMPTY);