From 4f0d137c5fc19aa879a3afe2d73c1d31c9bf55a7 Mon Sep 17 00:00:00 2001 From: David Turner Date: Mon, 4 Oct 2021 19:43:01 +0100 Subject: [PATCH] Revert "Prevent downgrades from 8.x to 7.x (#78586)" This reverts commit 02b7b170a51d7c822040f12cd0d2970864b94940. --- .../elasticsearch/env/NodeEnvironmentIT.java | 13 ++++---- .../elasticsearch/env/NodeEnvironment.java | 28 ++++++----------- .../env/NodeEnvironmentTests.java | 30 +++++++------------ .../cluster/DiskUsageIntegTestCase.java | 5 ---- 4 files changed, 25 insertions(+), 51 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java index 967aaed752851..0fc7a9a03d942 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java @@ -10,10 +10,10 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNodeRole; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.PathUtils; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.gateway.PersistedClusterStateService; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.test.ESIntegTestCase; @@ -138,11 +138,8 @@ public void testUpgradeDataFolder() throws IOException, InterruptedException { // simulate older data path layout by moving data under "nodes/0" folder final List dataPaths = List.of(PathUtils.get(Environment.PATH_DATA_SETTING.get(dataPathSettings))); dataPaths.forEach(path -> { - final Path nodesPath = path.resolve("nodes"); - final Path targetPath = nodesPath.resolve("0"); + final Path targetPath = path.resolve("nodes").resolve("0"); try { - assertTrue(Files.isRegularFile(nodesPath)); - Files.delete(nodesPath); Files.createDirectories(targetPath); try (DirectoryStream stream = Files.newDirectoryStream(path)) { @@ -195,9 +192,9 @@ public void testUpgradeDataFolder() throws IOException, InterruptedException { } // check that upgrade works - dataPaths.forEach(path -> assertTrue(Files.isDirectory(path.resolve("nodes")))); + dataPaths.forEach(path -> assertTrue(Files.exists(path.resolve("nodes")))); internalCluster().startNode(dataPathSettings); - dataPaths.forEach(path -> assertTrue(Files.isRegularFile(path.resolve("nodes")))); + dataPaths.forEach(path -> assertFalse(Files.exists(path.resolve("nodes")))); assertEquals(nodeId, client().admin().cluster().prepareState().get().getState().nodes().getMasterNodeId()); assertTrue(indexExists("test")); ensureYellow("test"); diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java index 1c229299aa22a..9bfe598dc0566 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -25,21 +25,21 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeRole; +import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.common.Randomness; +import org.elasticsearch.core.SuppressForbidden; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.core.Tuple; import org.elasticsearch.common.io.FileSystemUtils; +import org.elasticsearch.core.Releasable; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.core.CheckedFunction; -import org.elasticsearch.core.CheckedRunnable; -import org.elasticsearch.core.Releasable; -import org.elasticsearch.core.SuppressForbidden; -import org.elasticsearch.core.TimeValue; -import org.elasticsearch.core.Tuple; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.gateway.MetadataStateFormat; import org.elasticsearch.gateway.PersistedClusterStateService; @@ -55,7 +55,6 @@ import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.DirectoryStream; import java.nio.file.FileStore; @@ -270,16 +269,6 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce assertCanWrite(); } - // versions 7.x and earlier put their data under ${path.data}/nodes/; leave a file at that location to prevent downgrades - final Path legacyNodesPath = environment.dataFile().resolve("nodes"); - if (Files.isRegularFile(legacyNodesPath) == false) { - final String content = "written by Elasticsearch v" + Version.CURRENT + - " to prevent a downgrade to a version prior to v8.0.0 which would result in data loss"; - Files.write(legacyNodesPath, content.getBytes(StandardCharsets.UTF_8)); - IOUtils.fsync(legacyNodesPath, false); - IOUtils.fsync(environment.dataFile(), true); - } - if (DiscoveryNode.canContainData(settings) == false) { if (DiscoveryNode.isMasterNode(settings) == false) { ensureNoIndexMetadata(nodePath); @@ -419,11 +408,12 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings IOUtils.fsync(nodePath.path, true); }); - // now do the actual upgrade + // now do the actual upgrade. start by upgrading the node metadata file before moving anything, since a downgrade in an + // intermediate state would be pretty disastrous + loadNodeMetadata(settings, logger, legacyNodeLock.getNodePath()); for (CheckedRunnable upgradeAction : upgradeActions) { upgradeAction.run(); } - } finally { legacyNodeLock.close(); } diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index 016329eca28c9..5cf9c24aa2529 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.test.NodeRoles; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -42,7 +41,6 @@ import static org.elasticsearch.test.NodeRoles.nonDataNode; import static org.elasticsearch.test.NodeRoles.nonMasterNode; import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.startsWith; @@ -441,23 +439,6 @@ public void testEnsureNoShardDataOrIndexMetadata() throws IOException { verifyFailsOnShardData(noDataNoMasterSettings, indexPath, shardDataDirName); } - public void testBlocksDowngradeToVersionWithMultipleNodesInDataPath() throws IOException { - final Settings settings = buildEnvSettings(Settings.EMPTY); - for (int i = 0; i < 2; i++) { // ensure the file gets created again if missing - try (NodeEnvironment env = newNodeEnvironment(settings)) { - final Path nodesPath = env.nodeDataPath().resolve("nodes"); - assertTrue(Files.isRegularFile(nodesPath)); - assertThat( - Files.readString(nodesPath, StandardCharsets.UTF_8), - allOf( - containsString("written by Elasticsearch"), - containsString("prevent a downgrade"), - containsString("data loss"))); - Files.delete(nodesPath); - } - } - } - private void verifyFailsOnShardData(Settings settings, Path indexPath, String shardDataDirName) { IllegalStateException ex = expectThrows(IllegalStateException.class, "Must fail creating NodeEnvironment on a data path that has shard data if node does not have data role", @@ -478,6 +459,17 @@ private void verifyFailsOnMetadata(Settings settings, Path indexPath) { assertThat(ex.getMessage(), startsWith("node does not have the data and master roles but has index metadata")); } + /** + * Converts an array of Strings to an array of Paths, adding an additional child if specified + */ + private Path[] stringsToPaths(String[] strings, String additional) { + Path[] locations = new Path[strings.length]; + for (int i = 0; i < strings.length; i++) { + locations[i] = PathUtils.get(strings[i], additional); + } + return locations; + } + @Override public NodeEnvironment newNodeEnvironment() throws IOException { return newNodeEnvironment(Settings.EMPTY); diff --git a/test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java index 2bca8d4ddccb2..96bf7f8e5eb34 100644 --- a/test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java @@ -26,7 +26,6 @@ import java.io.FileNotFoundException; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.FileStore; import java.nio.file.FileSystem; @@ -149,10 +148,6 @@ public long getUnallocatedSpace() throws IOException { private static long getTotalFileSize(Path path) throws IOException { if (Files.isRegularFile(path)) { - if (path.getFileName().toString().equals("nodes") - && Files.readString(path, StandardCharsets.UTF_8).contains("prevent a downgrade")) { - return 0; - } try { return Files.size(path); } catch (NoSuchFileException | FileNotFoundException e) {