Prevent downgrades from 8.x to 7.x (#78638)

Users sometimes attempt to downgrade a node in place, but downgrades are
totally untested and unsupported and generally don't work. We protect
against this by recording the node version in the metadata and refusing
to start if we encounter metadata written by a future version.

However in 8.0 (#42489) we changed the directory layout so that a 7.x
node won't find the upgraded metadata, or indeed any other data, and
will proceed as if it's a fresh node. That's almost certainly not what
the user wants, so with this commit we create a file at
`${path.data}/nodes` at each startup, preventing an older node from
starting.

Closes #52414

server/src/internalClusterTest/java/org/elasticsearch/env/NodeEnvironmentIT.java

@@ -10,15 +10,15 @@
 
 import org.elasticsearch.Version;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.core.PathUtils;
-import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.gateway.PersistedClusterStateService;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.test.InternalTestCluster;
 import org.elasticsearch.test.NodeRoles;
+import org.elasticsearch.xcontent.XContentType;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -140,8 +140,11 @@ public void testUpgradeDataFolder() throws IOException, InterruptedException {
         final List<Path> dataPaths = Environment.PATH_DATA_SETTING.get(dataPathSettings)
             .stream().map(PathUtils::get).collect(Collectors.toList());
         dataPaths.forEach(path -> {
-                final Path targetPath = path.resolve("nodes").resolve("0");
+                final Path nodesPath = path.resolve("nodes");
+                final Path targetPath = nodesPath.resolve("0");
                 try {
+                    assertTrue(Files.isRegularFile(nodesPath));
+                    Files.delete(nodesPath);
                     Files.createDirectories(targetPath);
 
                     try (DirectoryStream<Path> stream = Files.newDirectoryStream(path)) {
@@ -194,9 +197,9 @@ public void testUpgradeDataFolder() throws IOException, InterruptedException {
         }
 
         // check that upgrade works
-        dataPaths.forEach(path -> assertTrue(Files.exists(path.resolve("nodes"))));
+        dataPaths.forEach(path -> assertTrue(Files.isDirectory(path.resolve("nodes"))));
         internalCluster().startNode(dataPathSettings);
-        dataPaths.forEach(path -> assertFalse(Files.exists(path.resolve("nodes"))));
+        dataPaths.forEach(path -> assertTrue(Files.isRegularFile(path.resolve("nodes"))));
         assertEquals(nodeId, client().admin().cluster().prepareState().get().getState().nodes().getMasterNodeId());
         assertTrue(indexExists("test"));
         ensureYellow("test");

server/src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -25,21 +25,20 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.core.CheckedFunction;
-import org.elasticsearch.core.CheckedRunnable;
 import org.elasticsearch.common.Randomness;
-import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.common.UUIDs;
-import org.elasticsearch.core.Tuple;
 import org.elasticsearch.common.io.FileSystemUtils;
-import org.elasticsearch.core.Releasable;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
-import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.common.util.set.Sets;
-import org.elasticsearch.xcontent.NamedXContentRegistry;
+import org.elasticsearch.core.CheckedFunction;
+import org.elasticsearch.core.CheckedRunnable;
+import org.elasticsearch.core.Releasable;
+import org.elasticsearch.core.SuppressForbidden;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.gateway.MetadataStateFormat;
 import org.elasticsearch.gateway.PersistedClusterStateService;
@@ -51,10 +50,12 @@
 import org.elasticsearch.monitor.fs.FsInfo;
 import org.elasticsearch.monitor.fs.FsProbe;
 import org.elasticsearch.monitor.jvm.JvmInfo;
+import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileStore;
@@ -281,6 +282,18 @@ public NodeEnvironment(Settings settings, Environment environment) throws IOExce
                 assertCanWrite();
             }
 
+            // versions 7.x and earlier put their data under ${path.data}/nodes/; leave a file at that location to prevent downgrades
+            for (Path dataPath : environment.dataFiles()) {
+                final Path legacyNodesPath = dataPath.resolve("nodes");
+                if (Files.isRegularFile(legacyNodesPath) == false) {
+                    final String content = "written by Elasticsearch v" + Version.CURRENT +
+                        " to prevent a downgrade to a version prior to v8.0.0 which would result in data loss";
+                    Files.write(legacyNodesPath, content.getBytes(StandardCharsets.UTF_8));
+                    IOUtils.fsync(legacyNodesPath, false);
+                    IOUtils.fsync(dataPath, true);
+                }
+            }
+
             if (DiscoveryNode.canContainData(settings) == false) {
                 if (DiscoveryNode.isMasterNode(settings) == false) {
                     ensureNoIndexMetadata(nodePaths);
@@ -424,12 +437,11 @@ private static boolean upgradeLegacyNodeFolders(Logger logger, Settings settings
                     IOUtils.fsync(nodePath.path, true);
                 });
             }
-            // now do the actual upgrade. start by upgrading the node metadata file before moving anything, since a downgrade in an
-            // intermediate state would be pretty disastrous
-            loadNodeMetadata(settings, logger, legacyNodeLock.getNodePaths());
+            // now do the actual upgrade
             for (CheckedRunnable<IOException> upgradeAction : upgradeActions) {
                 upgradeAction.run();
             }
+
         } finally {
             legacyNodeLock.close();
         }

server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java

@@ -11,11 +11,11 @@
 import org.apache.lucene.util.LuceneTestCase;
 import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.cluster.node.DiscoveryNodeRole;
-import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.PathUtils;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.common.util.set.Sets;
+import org.elasticsearch.core.PathUtils;
+import org.elasticsearch.core.SuppressForbidden;
 import org.elasticsearch.core.internal.io.IOUtils;
 import org.elasticsearch.gateway.MetadataStateFormat;
 import org.elasticsearch.index.Index;
@@ -26,6 +26,7 @@
 import org.elasticsearch.test.NodeRoles;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
@@ -41,6 +42,7 @@
 import static org.elasticsearch.test.NodeRoles.nonDataNode;
 import static org.elasticsearch.test.NodeRoles.nonMasterNode;
 import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.Matchers.allOf;
 import static org.hamcrest.Matchers.arrayWithSize;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.empty;
@@ -468,6 +470,25 @@ public void testEnsureNoShardDataOrIndexMetadata() throws IOException {
         verifyFailsOnShardData(noDataNoMasterSettings, indexPath, shardDataDirName);
     }
 
+    public void testBlocksDowngradeToVersionWithMultipleNodesInDataPath() throws IOException {
+        final Settings settings = buildEnvSettings(Settings.EMPTY);
+        for (int i = 0; i < 2; i++) { // ensure the file gets created again if missing
+            try (NodeEnvironment env = newNodeEnvironment(settings)) {
+                for (Path dataPath : env.nodeDataPaths()) {
+                    final Path nodesPath = dataPath.resolve("nodes");
+                    assertTrue(Files.isRegularFile(nodesPath));
+                    assertThat(
+                        Files.readString(nodesPath, StandardCharsets.UTF_8),
+                        allOf(
+                            containsString("written by Elasticsearch"),
+                            containsString("prevent a downgrade"),
+                            containsString("data loss")));
+                    Files.delete(nodesPath);
+                }
+            }
+        }
+    }
+
     private void verifyFailsOnShardData(Settings settings, Path indexPath, String shardDataDirName) {
         IllegalStateException ex = expectThrows(IllegalStateException.class,
             "Must fail creating NodeEnvironment on a data path that has shard data if node does not have data role",

test/framework/src/main/java/org/elasticsearch/cluster/DiskUsageIntegTestCase.java

@@ -26,6 +26,7 @@
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.DirectoryStream;
 import java.nio.file.FileStore;
 import java.nio.file.FileSystem;
@@ -148,6 +149,10 @@ public long getUnallocatedSpace() throws IOException {
 
         private static long getTotalFileSize(Path path) throws IOException {
             if (Files.isRegularFile(path)) {
+                if (path.getFileName().toString().equals("nodes")
+                    && Files.readString(path, StandardCharsets.UTF_8).contains("prevent a downgrade")) {
+                    return 0;
+                }
                 try {
                     return Files.size(path);
                 } catch (NoSuchFileException | FileNotFoundException e) {