Revert "Prevent downgrades from 8.x to 7.x (elastic#78586)"

This reverts commit 02b7b17.

server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java

@@ -10,10 +10,10 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.gateway.PersistedClusterStateService;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESIntegTestCase;
@@ -138,11 +138,8 @@ public void testUpgradeDataFolder() throws IOException, InterruptedException {
         // simulate older data path layout by moving data under "nodes/0" folder
         final List<Path> dataPaths = List.of(PathUtils.get(Environment.PATH_DATA_SETTING.get(dataPathSettings)));
         dataPaths.forEach(path -> {
-                final Path nodesPath = path.resolve("nodes");
-                final Path targetPath = nodesPath.resolve("0");
+                final Path targetPath = path.resolve("nodes").resolve("0");
                 try {
-                    assertTrue(Files.isRegularFile(nodesPath));
-                    Files.delete(nodesPath);
                     Files.createDirectories(targetPath);
 
                     try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
@@ -195,9 +192,9 @@ public void testUpgradeDataFolder() throws IOException, InterruptedException {
         }
 
         // check that upgrade works
-        dataPaths.forEach(path -> assertTrue(Files.isDirectory(path.resolve("nodes"))));
+        dataPaths.forEach(path -> assertTrue(Files.exists(path.resolve("nodes"))));
         internalCluster().startNode(dataPathSettings);
-        dataPaths.forEach(path -> assertTrue(Files.isRegularFile(path.resolve("nodes"))));
+        dataPaths.forEach(path -> assertFalse(Files.exists(path.resolve("nodes"))));
         assertEquals(nodeId, client().admin().cluster().prepareState().get().getState().nodes().getMasterNodeId());
         assertTrue(indexExists("test"));
         ensureYellow("test");

server/src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -25,21 +25,21 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.common.Randomness;
+import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.common.io.FileSystemUtils;
+import org.elasticsearch.core.Releasable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
-import org.elasticsearch.core.CheckedFunction;
-import org.elasticsearch.core.CheckedRunnable;
-import org.elasticsearch.core.Releasable;
-import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.gateway.MetadataStateFormat;
 import org.elasticsearch.gateway.PersistedClusterStateService;
@@ -55,7 +55,6 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.UncheckedIOException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileStore;
@@ -270,16 +269,6 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
                 assertCanWrite();
             }
 
-            // versions 7.x and earlier put their data under ${path.data}/nodes/; leave a file at that location to prevent downgrades
-            final Path legacyNodesPath = environment.dataFile().resolve("nodes");
-            if (Files.isRegularFile(legacyNodesPath) == false) {
-                final String content = "written by Elasticsearch v" + Version.CURRENT +
-                    " to prevent a downgrade to a version prior to v8.0.0 which would result in data loss";
-                Files.write(legacyNodesPath, content.getBytes(StandardCharsets.UTF_8));
-                IOUtils.fsync(legacyNodesPath, false);
-                IOUtils.fsync(environment.dataFile(), true);
-            }
-
             if (DiscoveryNode.canContainData(settings) == false) {
                 if (DiscoveryNode.isMasterNode(settings) == false) {
                     ensureNoIndexMetadata(nodePath);
@@ -419,11 +408,12 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings
                 IOUtils.fsync(nodePath.path, true);
             });
 
-            // now do the actual upgrade
+            // now do the actual upgrade. start by upgrading the node metadata file before moving anything, since a downgrade in an
+            // intermediate state would be pretty disastrous
+            loadNodeMetadata(settings, logger, legacyNodeLock.getNodePath());
             for (CheckedRunnable<IOException> upgradeAction : upgradeActions) {
                 upgradeAction.run();
             }
-
         } finally {
             legacyNodeLock.close();
         }

server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java

@@ -26,7 +26,6 @@
 import org.elasticsearch.test.NodeRoles;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -42,7 +41,6 @@
 import static org.elasticsearch.test.NodeRoles.nonDataNode;
 import static org.elasticsearch.test.NodeRoles.nonMasterNode;
 import static org.hamcrest.CoreMatchers.equalTo;
-import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.startsWith;
@@ -441,23 +439,6 @@ public void testEnsureNoShardDataOrIndexMetadata() throws IOException {
         verifyFailsOnShardData(noDataNoMasterSettings, indexPath, shardDataDirName);
     }
 
-    public void testBlocksDowngradeToVersionWithMultipleNodesInDataPath() throws IOException {
-        final Settings settings = buildEnvSettings(Settings.EMPTY);
-        for (int i = 0; i < 2; i++) { // ensure the file gets created again if missing
-            try (NodeEnvironment env = newNodeEnvironment(settings)) {
-                final Path nodesPath = env.nodeDataPath().resolve("nodes");
-                assertTrue(Files.isRegularFile(nodesPath));
-                assertThat(
-                    Files.readString(nodesPath, StandardCharsets.UTF_8),
-                    allOf(
-                        containsString("written by Elasticsearch"),
-                        containsString("prevent a downgrade"),
-                        containsString("data loss")));
-                Files.delete(nodesPath);
-            }
-        }
-    }
-
     private void verifyFailsOnShardData(Settings settings, Path indexPath, String shardDataDirName) {
         IllegalStateException ex = expectThrows(IllegalStateException.class,
             "Must fail creating NodeEnvironment on a data path that has shard data if node does not have data role",
@@ -478,6 +459,17 @@ private void verifyFailsOnMetadata(Settings settings, Path indexPath) {
         assertThat(ex.getMessage(), startsWith("node does not have the data and master roles but has index metadata"));
     }
 
+    /**
+     * Converts an array of Strings to an array of Paths, adding an additional child if specified
+     */
+    private Path[] stringsToPaths(String[] strings, String additional) {
+        Path[] locations = new Path[strings.length];
+        for (int i = 0; i < strings.length; i++) {
+            locations[i] = PathUtils.get(strings[i], additional);
+        }
+        return locations;
+    }
+
     @Override
     public NodeEnvironment newNodeEnvironment() throws IOException {
         return newNodeEnvironment(Settings.EMPTY);

test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java

@@ -26,7 +26,6 @@
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileStore;
 import java.nio.file.FileSystem;
@@ -149,10 +148,6 @@ public long getUnallocatedSpace() throws IOException {
 
         private static long getTotalFileSize(Path path) throws IOException {
             if (Files.isRegularFile(path)) {
-                if (path.getFileName().toString().equals("nodes")
-                    && Files.readString(path, StandardCharsets.UTF_8).contains("prevent a downgrade")) {
-                    return 0;
-                }
                 try {
                     return Files.size(path);
                 } catch (NoSuchFileException | FileNotFoundException e) {