Merge branch 'main' of https://github.com/elastic/elasticsearch into …

…esql-kql-function

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java

@@ -137,7 +137,7 @@ public void skipTest(String fullTestName, String reason) {
         // However, the folder can be arbitrarily nest so, a == a1/a2/a3, and the test name can include forward slashes, so c == c1/c2/c3
         // So we also need to support a1/a2/a3/b/c1/c2/c3
 
-        String[] testParts = fullTestName.split("/", 3);
+        String[] testParts = fullTestName.split("/");
         if (testParts.length < 3) {
             throw new IllegalArgumentException(
                 "To skip tests, all 3 parts [folder/file/test name] must be defined. found [" + fullTestName + "]"

muted-tests.yml

@@ -245,36 +245,6 @@ tests:
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=snapshot/10_basic/Failed to snapshot indices with synthetic source}
   issue: https://github.com/elastic/elasticsearch/issues/117082
-- class: org.elasticsearch.upgrades.FullClusterRestartDownsampleIT
-  method: testRollupIndex {cluster=OLD}
-  issue: https://github.com/elastic/elasticsearch/issues/117084
-- class: org.elasticsearch.upgrades.FullClusterRestartDownsampleIT
-  method: testRollupIndex {cluster=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/117086
-- class: org.elasticsearch.upgrades.FullClusterRestartIT
-  method: testNewReplicasTimeSeriesMode {cluster=OLD}
-  issue: https://github.com/elastic/elasticsearch/issues/117087
-- class: org.elasticsearch.upgrades.FullClusterRestartIT
-  method: testNewReplicasTimeSeriesMode {cluster=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/117088
-- class: org.elasticsearch.upgrades.FullClusterRestartIT
-  method: testSearchTimeSeriesMode {cluster=OLD}
-  issue: https://github.com/elastic/elasticsearch/issues/117089
-- class: org.elasticsearch.upgrades.FullClusterRestartIT
-  method: testSearchTimeSeriesMode {cluster=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/117090
-- class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT
-  method: testNewReplicasTimeSeriesMode {cluster=OLD}
-  issue: https://github.com/elastic/elasticsearch/issues/117091
-- class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT
-  method: testSearchTimeSeriesMode {cluster=OLD}
-  issue: https://github.com/elastic/elasticsearch/issues/117092
-- class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT
-  method: testNewReplicasTimeSeriesMode {cluster=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/117093
-- class: org.elasticsearch.xpack.restart.CoreFullClusterRestartIT
-  method: testSearchTimeSeriesMode {cluster=UPGRADED}
-  issue: https://github.com/elastic/elasticsearch/issues/117094
 - class: org.elasticsearch.discovery.ClusterDisruptionIT
   method: testAckedIndexing
   issue: https://github.com/elastic/elasticsearch/issues/117024
@@ -284,6 +254,41 @@ tests:
 - class: org.elasticsearch.xpack.apmdata.APMYamlTestSuiteIT
   method: test {yaml=/10_apm/Test template reinstallation}
   issue: https://github.com/elastic/elasticsearch/issues/116445
+- class: org.elasticsearch.upgrades.DownsampleIT
+  method: testRollupIndex {upgradedNodes=3}
+  issue: https://github.com/elastic/elasticsearch/issues/117122
+- class: org.elasticsearch.upgrades.DownsampleIT
+  method: testRollupIndex {upgradedNodes=1}
+  issue: https://github.com/elastic/elasticsearch/issues/117123
+- class: org.elasticsearch.upgrades.DownsampleIT
+  method: testRollupIndex {upgradedNodes=2}
+  issue: https://github.com/elastic/elasticsearch/issues/117124
+- class: org.elasticsearch.upgrades.IndexingIT
+  method: testAutoIdWithOpTypeCreate {upgradedNodes=1}
+  issue: https://github.com/elastic/elasticsearch/issues/117125
+- class: org.elasticsearch.upgrades.IndexingIT
+  method: testTsdb {upgradedNodes=1}
+  issue: https://github.com/elastic/elasticsearch/issues/117126
+- class: org.elasticsearch.upgrades.IndexingIT
+  method: testIndexing {upgradedNodes=1}
+  issue: https://github.com/elastic/elasticsearch/issues/117127
+- class: org.elasticsearch.upgrades.IndexingIT
+  method: testSyntheticSource {upgradedNodes=1}
+  issue: https://github.com/elastic/elasticsearch/issues/117128
+- class: org.elasticsearch.upgrades.IndexingIT
+  method: testIndexing {upgradedNodes=3}
+  issue: https://github.com/elastic/elasticsearch/issues/117135
+- class: org.elasticsearch.upgrades.IndexingIT
+  method: testTsdb {upgradedNodes=3}
+  issue: https://github.com/elastic/elasticsearch/issues/117136
+- class: org.elasticsearch.upgrades.IndexingIT
+  method: testIndexing {upgradedNodes=2}
+  issue: https://github.com/elastic/elasticsearch/issues/117137
+- class: org.elasticsearch.upgrades.IndexingIT
+  method: testTsdb {upgradedNodes=2}
+  issue: https://github.com/elastic/elasticsearch/issues/117138
+- class: org.elasticsearch.upgrades.IndexingIT
+  issue: https://github.com/elastic/elasticsearch/issues/117140
 
 # Examples:
 #

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartDownsampleIT.java

@@ -268,6 +268,7 @@ private String getRollupIndexName() throws IOException {
     }
 
     public void testRollupIndex() throws Exception {
+        assumeTrue("Downsample got many stability improvements in 8.10.0", oldClusterHasFeature("gte_v8.10.0"));
         if (isRunningAgainstOldCluster()) {
             createIlmPolicy();
             createIndex();

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -261,6 +261,7 @@ public void testNewReplicas() throws Exception {
     }
 
     public void testSearchTimeSeriesMode() throws Exception {
+        assumeTrue("indexing time series indices changed in 8.2.0", oldClusterHasFeature("gte_v8.2.0"));
         int numDocs;
         if (isRunningAgainstOldCluster()) {
             numDocs = createTimeSeriesModeIndex(1);
@@ -298,6 +299,7 @@ public void testSearchTimeSeriesMode() throws Exception {
     }
 
     public void testNewReplicasTimeSeriesMode() throws Exception {
+        assumeTrue("indexing time series indices changed in 8.2.0", oldClusterHasFeature("gte_v8.2.0"));
         if (isRunningAgainstOldCluster()) {
             createTimeSeriesModeIndex(0);
         } else {

qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.FormatNames;
+import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.hamcrest.Matcher;
@@ -29,6 +30,9 @@
 import java.util.Map;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.test.MapMatcher.assertMap;
+import static org.elasticsearch.test.MapMatcher.matchesMap;
+
 public class LogsIndexModeFullClusterRestartIT extends ParameterizedFullClusterRestartTestCase {
 
     @ClassRule
@@ -165,16 +169,22 @@ public void testLogsIndexing() throws IOException {
             assertOK(bulkIndexResponse);
             assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false));
 
-            assertIndexSettings(0, Matchers.nullValue());
-            assertIndexSettings(1, Matchers.equalTo("logsdb"));
+            assertIndexMappingsAndSettings(0, Matchers.nullValue(), matchesMap().extraOk());
+            assertIndexMappingsAndSettings(
+                1,
+                Matchers.equalTo("logsdb"),
+                matchesMap().extraOk().entry("_source", Map.of("mode", "synthetic"))
+            );
         }
     }
 
-    private void assertIndexSettings(int backingIndex, final Matcher<Object> indexModeMatcher) throws IOException {
+    private void assertIndexMappingsAndSettings(int backingIndex, final Matcher<Object> indexModeMatcher, final MapMatcher mappingsMatcher)
+        throws IOException {
         assertThat(
             getSettings(client(), getWriteBackingIndex(client(), "logs-apache-production", backingIndex)).get("index.mode"),
             indexModeMatcher
         );
+        assertMap(getIndexMappingAsMap(getWriteBackingIndex(client(), "logs-apache-production", backingIndex)), mappingsMatcher);
     }
 
     private static Request createDataStream(final String dataStreamName) {

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.FormatNames;
+import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.hamcrest.Matcher;
@@ -29,6 +30,9 @@
 import java.util.Map;
 import java.util.function.Supplier;
 
+import static org.elasticsearch.test.MapMatcher.assertMap;
+import static org.elasticsearch.test.MapMatcher.matchesMap;
+
 public class LogsIndexModeRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
 
     @ClassRule()
@@ -156,10 +160,14 @@ public void testLogsIndexing() throws IOException {
             assertOK(bulkIndexResponse);
             assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false));
 
-            assertIndexSettings(0, Matchers.nullValue());
-            assertIndexSettings(1, Matchers.nullValue());
-            assertIndexSettings(2, Matchers.nullValue());
-            assertIndexSettings(3, Matchers.equalTo("logsdb"));
+            assertIndexMappingsAndSettings(0, Matchers.nullValue(), matchesMap().extraOk());
+            assertIndexMappingsAndSettings(1, Matchers.nullValue(), matchesMap().extraOk());
+            assertIndexMappingsAndSettings(2, Matchers.nullValue(), matchesMap().extraOk());
+            assertIndexMappingsAndSettings(
+                3,
+                Matchers.equalTo("logsdb"),
+                matchesMap().extraOk().entry("_source", Map.of("mode", "synthetic"))
+            );
         }
     }
 
@@ -175,11 +183,13 @@ static void enableLogsdbByDefault() throws IOException {
         assertOK(client().performRequest(request));
     }
 
-    private void assertIndexSettings(int backingIndex, final Matcher<Object> indexModeMatcher) throws IOException {
+    private void assertIndexMappingsAndSettings(int backingIndex, final Matcher<Object> indexModeMatcher, final MapMatcher mappingsMatcher)
+        throws IOException {
         assertThat(
             getSettings(client(), getWriteBackingIndex(client(), "logs-apache-production", backingIndex)).get("index.mode"),
             indexModeMatcher
         );
+        assertMap(getIndexMappingAsMap(getWriteBackingIndex(client(), "logs-apache-production", backingIndex)), mappingsMatcher);
     }
 
     private static Request createDataStream(final String dataStreamName) {

rest-api-spec/build.gradle

@@ -60,10 +60,4 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy")
   task.skipTest("cat.shards/10_basic/Help", "sync_id is removed in 9.0")
   task.skipTest("search/500_date_range/from, to, include_lower, include_upper deprecated", "deprecated parameters are removed in 9.0")
-  task.skipTest("tsdb/20_mapping/stored source is supported", "no longer serialize source_mode")
-  task.skipTest("tsdb/20_mapping/Synthetic source", "no longer serialize source_mode")
-  task.skipTest("logsdb/10_settings/create logs index", "no longer serialize source_mode")
-  task.skipTest("logsdb/20_source_mapping/stored _source mode is supported", "no longer serialize source_mode")
-  task.skipTest("logsdb/20_source_mapping/include/exclude is supported with stored _source", "no longer serialize source_mode")
-  task.skipTest("logsdb/20_source_mapping/synthetic _source is default", "no longer serialize source_mode")
 })

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml

@@ -76,6 +76,11 @@ create logs index:
   - is_true: test
   - match: { test.settings.index.mode: "logsdb" }
 
+  - do:
+      indices.get_mapping:
+        index: test
+  - match: { test.mappings._source.mode: synthetic }
+
 ---
 using default timestamp field mapping:
   - requires:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml

@@ -13,10 +13,10 @@ synthetic _source is default:
             index:
               mode: logsdb
   - do:
-      indices.get_settings:
+      indices.get:
         index: test-default-source
-  - match: { test-default-source.settings.index.mode: logsdb }
-  - match: { test-default-source.settings.index.mapping.source.mode: null }
+
+  - match: { test-default-source.mappings._source.mode: "synthetic" }
 
 ---
 stored _source mode is supported:
@@ -28,12 +28,11 @@ stored _source mode is supported:
             index:
               mode: logsdb
               mapping.source.mode: stored
-
   - do:
-      indices.get_settings:
+      indices.get:
         index: test-stored-source
-  - match: { test-stored-source.settings.index.mode: logsdb }
-  - match: { test-stored-source.settings.index.mapping.source.mode: stored }
+
+  - match: { test-stored-source.mappings._source.mode: "stored" }
 
 ---
 disabled _source is not supported:
@@ -111,6 +110,7 @@ include/exclude is supported with stored _source:
       indices.get:
         index: test-includes
 
+  - match: { test-includes.mappings._source.mode: "stored" }
   - match: { test-includes.mappings._source.includes: ["a"] }
 
   - do:
@@ -129,4 +129,5 @@ include/exclude is supported with stored _source:
       indices.get:
         index: test-excludes
 
+  - match: { test-excludes.mappings._source.mode: "stored" }
   - match: { test-excludes.mappings._source.excludes: ["b"] }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml

@@ -450,6 +450,11 @@ nested fields:
                 type: long
                 time_series_metric: gauge
 
+  - do:
+      indices.get_mapping: {}
+
+  - match: {tsdb-synthetic.mappings._source.mode: synthetic}
+
 ---
 stored source is supported:
   - requires:
@@ -481,6 +486,12 @@ stored source is supported:
                         type: keyword
                         time_series_dimension: true
 
+  - do:
+      indices.get:
+        index: tsdb_index
+
+  - match: { tsdb_index.mappings._source.mode: "stored" }
+
 ---
 disabled source is not supported:
   - requires:

server/src/main/java/org/elasticsearch/action/admin/cluster/node/features/TransportNodesFeaturesAction.java

@@ -16,7 +16,7 @@
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.core.UpdateForV9;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.injection.guice.Inject;
 import org.elasticsearch.tasks.Task;
@@ -27,8 +27,7 @@
 import java.io.IOException;
 import java.util.List;
 
-@UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-// @UpdateForV10 // this can be removed in v10. It may be called by v8 nodes to v9 nodes.
+@UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA) // this can be removed in v10. It may be called by v8 nodes to v9 nodes.
 public class TransportNodesFeaturesAction extends TransportNodesAction<
     NodesFeaturesRequest,
     NodesFeaturesResponse,