Re-define index.mapper.dynamic setting in 8.x (elastic#109341)

Currently when upgrading a 7.x cluster to 8.x with
`index.mapper.dynamic` index setting defined the following happens:

- In case of a full cluster restart upgrade, then the index setting gets archived and after the upgrade the cluster is in a green health.
- In case of a rolling cluster restart upgrade, then shards of indices with the index setting fail to allocate as nodes start with 8.x version. The result is that the cluster has a red health and the index setting isn't archived. Closing and opening the index should archive the index setting and allocate the shards.

The change is about ensuring the same behavior happens when upgrading a
cluster from 7.x to 8.x with indices that have the
`index.mapper.dynamic` index setting defined.  By re-defining the
`index.mapper.dynamic `index setting with
`IndexSettingDeprecatedInV7AndRemovedInV8` property, the index is
allowed to exist in 7.x indices, but can't be defined in new indices
after the upgrade. This way we don't have to rely on index archiving and
upgrading via full cluster restart or rolling restart will yield the
same outcome.

Based on the test in elastic#109301. Relates to elastic#109160 and elastic#96075

docs/changelog/109341.yaml

@@ -0,0 +1,5 @@
+pr: 109341
+summary: Re-define `index.mapper.dynamic` setting in 8.x for a better 7.x to 8.x upgrade if this setting is used.
+area: Mapping
+type: bug
+issues: []

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.upgrades;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+
+import org.elasticsearch.client.Request;
+import org.elasticsearch.client.ResponseException;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.support.XContentMapValues;
+import org.elasticsearch.test.cluster.ElasticsearchCluster;
+import org.elasticsearch.test.cluster.FeatureFlag;
+import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider;
+import org.elasticsearch.test.cluster.local.distribution.DistributionType;
+import org.junit.ClassRule;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+
+public class UpgradeWithOldIndexSettingsIT extends ParameterizedFullClusterRestartTestCase {
+
+    protected static LocalClusterConfigProvider clusterConfig = c -> {};
+
+    @ClassRule
+    public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
+        .distribution(DistributionType.DEFAULT)
+        .version(getOldClusterTestVersion())
+        .nodes(2)
+        .setting("xpack.security.enabled", "false")
+        .feature(FeatureFlag.FAILURE_STORE_ENABLED)
+        .apply(() -> clusterConfig)
+        .build();
+
+    @Override
+    protected ElasticsearchCluster getUpgradeCluster() {
+        return cluster;
+    }
+
+    public UpgradeWithOldIndexSettingsIT(@Name("cluster") FullClusterRestartUpgradeStatus upgradeStatus) {
+        super(upgradeStatus);
+    }
+
+    public void testMapperDynamicIndexSetting() throws IOException {
+        assumeTrue(
+            "Setting deprecated in 6.x, but remained in 7.x and is no longer defined in 8.x",
+            getOldClusterTestVersion().before("8.0.0")
+        );
+        String indexName = "my-index";
+        if (isRunningAgainstOldCluster()) {
+            createIndex(indexName);
+
+            var request = new Request("PUT", "/my-index/_settings");
+            request.setJsonEntity(org.elasticsearch.common.Strings.toString(Settings.builder().put("index.mapper.dynamic", true).build()));
+            request.setOptions(
+                expectWarnings(
+                    "[index.mapper.dynamic] setting was deprecated in Elasticsearch and will be removed in a future release! "
+                        + "See the breaking changes documentation for the next major version."
+                )
+            );
+            assertOK(client().performRequest(request));
+        } else {
+            var indexSettings = getIndexSettings(indexName);
+            assertThat(XContentMapValues.extractValue(indexName + ".settings.index.mapper.dynamic", indexSettings), equalTo("true"));
+            ensureGreen(indexName);
+            // New indices can never define the index.mapper.dynamic setting.
+            Exception e = expectThrows(
+                ResponseException.class,
+                () -> createIndex("my-index2", Settings.builder().put("index.mapper.dynamic", true).build())
+            );
+            assertThat(e.getMessage(), containsString("unknown setting [index.mapper.dynamic]"));
+        }
+    }
+
+}

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/UpgradeWithOldIndexSettingsIT.java

@@ -22,6 +22,8 @@
 import java.util.Map;
 
 import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 
 public class UpgradeWithOldIndexSettingsIT extends AbstractRollingUpgradeTestCase {
@@ -102,6 +104,38 @@ public void testOldIndexSettings() throws Exception {
         }
     }
 
+    public void testMapperDynamicIndexSetting() throws IOException {
+        assumeTrue(
+            "Setting deprecated in 6.x, but remained in 7.x and is no longer defined in 8.x",
+            getOldClusterTestVersion().before("8.0.0")
+        );
+        String indexName = "my-index";
+        if (isOldCluster()) {
+            createIndex(indexName);
+            Request request = new Request("PUT", "/" + indexName + "/_settings");
+            request.setJsonEntity(org.elasticsearch.common.Strings.toString(Settings.builder().put("index.mapper.dynamic", true).build()));
+            request.setOptions(
+                expectWarnings(
+                    "[index.mapper.dynamic] setting was deprecated in Elasticsearch and will be removed in a future release! "
+                        + "See the breaking changes documentation for the next major version."
+                )
+            );
+            assertOK(client().performRequest(request));
+        } else {
+            if (isUpgradedCluster()) {
+                var indexSettings = getIndexSettings(indexName);
+                assertThat(XContentMapValues.extractValue(indexName + ".settings.index.mapper.dynamic", indexSettings), equalTo("true"));
+                ensureGreen(indexName);
+                // New indices can never define the index.mapper.dynamic setting.
+                Exception e = expectThrows(
+                    ResponseException.class,
+                    () -> createIndex("my-index2", Settings.builder().put("index.mapper.dynamic", true).build())
+                );
+                assertThat(e.getMessage(), containsString("unknown setting [index.mapper.dynamic]"));
+            }
+        }
+    }
+
     private void assertCount(String index, int countAtLeast) throws IOException {
         Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search");
         searchTestIndexRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true");

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -158,6 +158,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
         MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING,
         MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING,
         MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING,
+        MapperService.INDEX_MAPPER_DYNAMIC_SETTING,
         BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,
         IndexModule.INDEX_STORE_TYPE_SETTING,
         IndexModule.INDEX_STORE_PRE_LOAD_SETTING,

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.AbstractIndexComponent;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
@@ -145,6 +146,19 @@ public boolean isAutoUpdate() {
         Property.Dynamic,
         Property.IndexScope
     );
+    /**
+     * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use.
+     * TODO: Remove in 9.0
+     */
+    @Deprecated
+    @UpdateForV9
+    public static final Setting<Boolean> INDEX_MAPPER_DYNAMIC_SETTING = Setting.boolSetting(
+        "index.mapper.dynamic",
+        true,
+        Property.Dynamic,
+        Property.IndexScope,
+        Property.IndexSettingDeprecatedInV7AndRemovedInV8
+    );
 
     private final IndexAnalyzers indexAnalyzers;
     private final MappingParser mappingParser;