From edb29e008b09d7e38b50a8f31259411722bd10be Mon Sep 17 00:00:00 2001 From: David Turner Date: Thu, 14 Oct 2021 10:14:21 +0100 Subject: [PATCH] Prevent downgrades from 8.x to 7.x (#78638) Users sometimes attempt to downgrade a node in place, but downgrades are totally untested and unsupported and generally don't work. We protect against this by recording the node version in the metadata and refusing to start if we encounter metadata written by a future version. However in 8.0 (#42489) we changed the directory layout so that a 7.x node won't find the upgraded metadata, or indeed any other data, and will proceed as if it's a fresh node. That's almost certainly not what the user wants, so with this commit we create a file at `${path.data}/nodes` at each startup, preventing an older node from starting. Closes #52414 --- .../elasticsearch/env/NodeEnvironmentIT.java | 13 +++++--- .../elasticsearch/env/NodeEnvironment.java | 32 +++++++++++++------ .../env/NodeEnvironmentTests.java | 25 +++++++++++++-- .../cluster/DiskUsageIntegTestCase.java | 5 +++ 4 files changed, 58 insertions(+), 17 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java index 58f6482cdba6b..d15254134f5c0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java @@ -10,15 +10,15 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.PathUtils; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.gateway.PersistedClusterStateService; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.NodeRoles; +import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.io.UncheckedIOException; @@ -140,8 +140,11 @@ public void testUpgradeDataFolder() throws IOException, InterruptedException { final List dataPaths = Environment.PATH_DATA_SETTING.get(dataPathSettings) .stream().map(PathUtils::get).collect(Collectors.toList()); dataPaths.forEach(path -> { - final Path targetPath = path.resolve("nodes").resolve("0"); + final Path nodesPath = path.resolve("nodes"); + final Path targetPath = nodesPath.resolve("0"); try { + assertTrue(Files.isRegularFile(nodesPath)); + Files.delete(nodesPath); Files.createDirectories(targetPath); try (DirectoryStream stream = Files.newDirectoryStream(path)) { @@ -194,9 +197,9 @@ public void testUpgradeDataFolder() throws IOException, InterruptedException { } // check that upgrade works - dataPaths.forEach(path -> assertTrue(Files.exists(path.resolve("nodes")))); + dataPaths.forEach(path -> assertTrue(Files.isDirectory(path.resolve("nodes")))); internalCluster().startNode(dataPathSettings); - dataPaths.forEach(path -> assertFalse(Files.exists(path.resolve("nodes")))); + dataPaths.forEach(path -> assertTrue(Files.isRegularFile(path.resolve("nodes")))); assertEquals(nodeId, client().admin().cluster().prepareState().get().getState().nodes().getMasterNodeId()); assertTrue(indexExists("test")); ensureYellow("test"); diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java index f1ee49c6ce8f3..29d68612f7e33 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -25,21 +25,20 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; -import org.elasticsearch.core.CheckedFunction; -import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.common.Randomness; -import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.common.UUIDs; -import org.elasticsearch.core.Tuple; import org.elasticsearch.common.io.FileSystemUtils; -import org.elasticsearch.core.Releasable; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.util.set.Sets; -import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.core.CheckedRunnable; +import org.elasticsearch.core.Releasable; +import org.elasticsearch.core.SuppressForbidden; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.core.Tuple; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.gateway.MetadataStateFormat; import org.elasticsearch.gateway.PersistedClusterStateService; @@ -51,10 +50,12 @@ import org.elasticsearch.monitor.fs.FsInfo; import org.elasticsearch.monitor.fs.FsProbe; import org.elasticsearch.monitor.jvm.JvmInfo; +import org.elasticsearch.xcontent.NamedXContentRegistry; import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.DirectoryStream; import java.nio.file.FileStore; @@ -281,6 +282,18 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce assertCanWrite(); } + // versions 7.x and earlier put their data under ${path.data}/nodes/; leave a file at that location to prevent downgrades + for (Path dataPath : environment.dataFiles()) { + final Path legacyNodesPath = dataPath.resolve("nodes"); + if (Files.isRegularFile(legacyNodesPath) == false) { + final String content = "written by Elasticsearch v" + Version.CURRENT + + " to prevent a downgrade to a version prior to v8.0.0 which would result in data loss"; + Files.write(legacyNodesPath, content.getBytes(StandardCharsets.UTF_8)); + IOUtils.fsync(legacyNodesPath, false); + IOUtils.fsync(dataPath, true); + } + } + if (DiscoveryNode.canContainData(settings) == false) { if (DiscoveryNode.isMasterNode(settings) == false) { ensureNoIndexMetadata(nodePaths); @@ -424,12 +437,11 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings IOUtils.fsync(nodePath.path, true); }); } - // now do the actual upgrade. start by upgrading the node metadata file before moving anything, since a downgrade in an - // intermediate state would be pretty disastrous - loadNodeMetadata(settings, logger, legacyNodeLock.getNodePaths()); + // now do the actual upgrade for (CheckedRunnable upgradeAction : upgradeActions) { upgradeAction.run(); } + } finally { legacyNodeLock.close(); } diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index c3cd335ed2da9..b927d402de3ba 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -11,11 +11,11 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.SetOnce; import org.elasticsearch.cluster.node.DiscoveryNodeRole; -import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.core.PathUtils; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.core.PathUtils; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.gateway.MetadataStateFormat; import org.elasticsearch.index.Index; @@ -26,6 +26,7 @@ import org.elasticsearch.test.NodeRoles; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -41,6 +42,7 @@ import static org.elasticsearch.test.NodeRoles.nonDataNode; import static org.elasticsearch.test.NodeRoles.nonMasterNode; import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; @@ -468,6 +470,25 @@ public void testEnsureNoShardDataOrIndexMetadata() throws IOException { verifyFailsOnShardData(noDataNoMasterSettings, indexPath, shardDataDirName); } + public void testBlocksDowngradeToVersionWithMultipleNodesInDataPath() throws IOException { + final Settings settings = buildEnvSettings(Settings.EMPTY); + for (int i = 0; i < 2; i++) { // ensure the file gets created again if missing + try (NodeEnvironment env = newNodeEnvironment(settings)) { + for (Path dataPath : env.nodeDataPaths()) { + final Path nodesPath = dataPath.resolve("nodes"); + assertTrue(Files.isRegularFile(nodesPath)); + assertThat( + Files.readString(nodesPath, StandardCharsets.UTF_8), + allOf( + containsString("written by Elasticsearch"), + containsString("prevent a downgrade"), + containsString("data loss"))); + Files.delete(nodesPath); + } + } + } + } + private void verifyFailsOnShardData(Settings settings, Path indexPath, String shardDataDirName) { IllegalStateException ex = expectThrows(IllegalStateException.class, "Must fail creating NodeEnvironment on a data path that has shard data if node does not have data role", diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java index d1d49d6cde6bd..653817b420e59 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java @@ -26,6 +26,7 @@ import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.FileStore; import java.nio.file.FileSystem; @@ -148,6 +149,10 @@ public long getUnallocatedSpace() throws IOException { private static long getTotalFileSize(Path path) throws IOException { if (Files.isRegularFile(path)) { + if (path.getFileName().toString().equals("nodes") + && Files.readString(path, StandardCharsets.UTF_8).contains("prevent a downgrade")) { + return 0; + } try { return Files.size(path); } catch (NoSuchFileException | FileNotFoundException e) {