Merge branch 'main' into feature/esql-match-options

# Conflicts:
#	x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

docs/changelog/120392.yaml

@@ -0,0 +1,6 @@
+pr: 120392
+summary: Test/107515 restore template with match only text mapper it fail
+area: Search
+type: bug
+issues:
+ - 107515

docs/changelog/120617.yaml

@@ -0,0 +1,5 @@
+pr: 120617
+summary: Fix queries with document level security on lookup indexes
+area: ES|QL
+type: bug
+issues: [120509]

docs/changelog/120717.yaml

@@ -0,0 +1,6 @@
+pr: 120717
+summary: Fix LTR rescorer throws 'local model reference is null' on multi-shards index when explained is enabled
+area: Ranking
+type: bug
+issues:
+ - 120739

docs/changelog/120748.yaml

@@ -0,0 +1,15 @@
+pr: 120748
+summary: Removing support for types field in watcher search
+area: Watcher
+type: breaking
+issues: []
+breaking:
+  title: Removing support for types field in watcher search
+  area: REST API
+  details: >-
+    Previously, setting the `input.search.request.types` field in the payload when creating a watcher to an empty array
+    was  allowed, although it resulted in a deprecation warning and had no effect (and any value other than an empty
+    array would result in an error). Now, support for this field is entirely removed, and the empty array will also
+    result in an error.
+  impact: Users should stop setting this field (which did not have any effect anyway).
+  notable: false

docs/changelog/120752.yaml

@@ -0,0 +1,6 @@
+pr: 120752
+summary: Refresh source index before reindexing data stream index
+area: Data streams
+type: bug
+issues:
+ - 120314

docs/changelog/120753.yaml

@@ -0,0 +1,5 @@
+pr: 120753
+summary: Optimize `IngestDocMetadata` `isAvailable`
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/120781.yaml

@@ -0,0 +1,5 @@
+pr: 120781
+summary: Add back `keep_alive` to `async_search.submit` rest-api-spec
+area: Search
+type: bug
+issues: []

docs/changelog/120809.yaml

@@ -0,0 +1,6 @@
+pr: 120809
+summary: LTR sometines throw `NullPointerException:` Cannot read field "approximation"
+  because "top" is null
+area: Ranking
+type: bug
+issues: []

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/APMAgentSettings.java

@@ -90,6 +90,11 @@ public void initAgentSystemProperties(Settings settings) {
      */
     @SuppressForbidden(reason = "Need to be able to manipulate APM agent-related properties to set them dynamically")
     public void setAgentSetting(String key, String value) {
+        if (key.startsWith("global_labels.")) {
+            // Invalid agent setting, leftover from flattening global labels in APMJVMOptions
+            // https://github.com/elastic/elasticsearch/issues/120791
+            return;
+        }
         final String completeKey = "elastic.apm." + Objects.requireNonNull(key);
         AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
             if (value == null || value.isEmpty()) {
@@ -242,8 +247,8 @@ private static Setting<String> concreteAgentSetting(String namespace, String qua
         return new Setting<>(qualifiedKey, "", (value) -> {
             if (qualifiedKey.equals("_na_") == false && PERMITTED_AGENT_KEYS.contains(namespace) == false) {
                 if (namespace.startsWith("global_labels.")) {
-                    // The nested labels syntax is transformed in APMJvmOptions.
-                    // Ignore these here to not fail if not correctly removed.
+                    // Invalid agent setting, leftover from flattening global labels in APMJVMOptions
+                    // https://github.com/elastic/elasticsearch/issues/120791
                     return value;
                 }
                 throw new IllegalArgumentException("Configuration [" + qualifiedKey + "] is either prohibited or unknown.");

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java

@@ -1137,11 +1137,8 @@ public void testPartialRestoreSnapshotThatIncludesDataStream() {
 
     /**
      * This test is a copy of the {@link #testPartialRestoreSnapshotThatIncludesDataStream()} the only difference
-     * is that one include the global state and one doesn't. In general this shouldn't matter that's why it used to be
-     * a random parameter of the test, but because of #107515 it fails when we include the global state. Keep them
-     * separate until this is fixed.
+     * is that one include the global state and one doesn't.
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107515")
     public void testPartialRestoreSnapshotThatIncludesDataStreamWithGlobalState() {
         final String snapshot = "test-snapshot";
         final String indexWithoutDataStream = "test-idx-no-ds";
@@ -1307,11 +1304,8 @@ public void testExcludeDSFromSnapshotWhenExcludingAnyOfItsIndices() {
 
     /**
      * This test is a copy of the {@link #testExcludeDSFromSnapshotWhenExcludingAnyOfItsIndices()} ()} the only difference
-     * is that one include the global state and one doesn't. In general this shouldn't matter that's why it used to be
-     * a random parameter of the test, but because of #107515 it fails when we include the global state. Keep them
-     * separate until this is fixed.
+     * is that one include the global state and one doesn't.
      */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107515")
     public void testExcludeDSFromSnapshotWhenExcludingItsIndicesWithGlobalState() {
         final String snapshot = "test-snapshot";
         final String indexWithoutDataStream = "test-idx-no-ds";
@@ -1477,10 +1471,6 @@ public void testWarningHeaderAbsentOnRestoreWithTemplates() throws Exception {
 
     }
 
-    /**
-     * This test is muted as it's awaiting the same fix as {@link #testPartialRestoreSnapshotThatIncludesDataStreamWithGlobalState()}
-     */
-    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107515")
     public void testWarningHeaderOnRestoreTemplateFromSnapshot() throws Exception {
         String datastreamName = "ds";
 

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java

@@ -14,7 +14,6 @@
 import org.elasticsearch.ingest.RandomDocumentPicks;
 import org.elasticsearch.ingest.TestIngestDocument;
 import org.elasticsearch.ingest.TestTemplateService;
-import org.elasticsearch.script.Metadata;
 import org.elasticsearch.test.ESTestCase;
 
 import java.util.ArrayList;
@@ -140,42 +139,40 @@ public void testRenameExistingFieldNullValue() throws Exception {
 
     public void testRenameAtomicOperationSetFails() throws Exception {
         Map<String, Object> metadata = new HashMap<>();
-        metadata.put("list", List.of("item"));
-
-        IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(
-            metadata,
-            Map.of("new_field", new Metadata.FieldProperty<>(Object.class, true, true, (k, v) -> {
-                if (v != null) {
-                    throw new UnsupportedOperationException();
-                }
-            }), "list", new Metadata.FieldProperty<>(Object.class, true, true, null))
-        );
-        Processor processor = createRenameProcessor("list", "new_field", false, false);
+        metadata.put("_index", "foobar");
+
+        IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(metadata);
+        Processor processor = createRenameProcessor("_index", "_version_type", false, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
-        } catch (UnsupportedOperationException e) {
+        } catch (IllegalArgumentException e) {
             // the set failed, the old field has not been removed
-            assertThat(ingestDocument.getSourceAndMetadata().containsKey("list"), equalTo(true));
-            assertThat(ingestDocument.getSourceAndMetadata().containsKey("new_field"), equalTo(false));
+            assertThat(
+                e.getMessage(),
+                equalTo(
+                    "_version_type must be a null or one of [internal, external, external_gte] "
+                        + "but was [foobar] with type [java.lang.String]"
+                )
+            );
+            assertThat(ingestDocument.getSourceAndMetadata().containsKey("_index"), equalTo(true));
+            assertThat(ingestDocument.getSourceAndMetadata().containsKey("_version_type"), equalTo(false));
         }
     }
 
     public void testRenameAtomicOperationRemoveFails() throws Exception {
         Map<String, Object> metadata = new HashMap<>();
-        metadata.put("list", List.of("item"));
+        metadata.put("foo", "bar");
 
-        IngestDocument ingestDocument = TestIngestDocument.ofMetadataWithValidator(
-            metadata,
-            Map.of("list", new Metadata.FieldProperty<>(Object.class, false, true, null))
-        );
-        Processor processor = createRenameProcessor("list", "new_field", false, false);
+        IngestDocument ingestDocument = TestIngestDocument.withDefaultVersion(metadata);
+        Processor processor = createRenameProcessor("_version", "new_field", false, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
         } catch (IllegalArgumentException e) {
-            // the set failed, the old field has not been removed
-            assertThat(ingestDocument.getSourceAndMetadata().containsKey("list"), equalTo(true));
+            // the remove failed, the old field has not been removed
+            assertThat(e.getMessage(), equalTo("_version cannot be removed"));
+            assertThat(ingestDocument.getSourceAndMetadata().containsKey("_version"), equalTo(true));
             assertThat(ingestDocument.getSourceAndMetadata().containsKey("new_field"), equalTo(false));
         }
     }

muted-tests.yml

@@ -131,9 +131,6 @@ tests:
 - class: org.elasticsearch.xpack.ml.integration.RegressionIT
   method: testTwoJobsWithSameRandomizeSeedUseSameTrainingSet
   issue: https://github.com/elastic/elasticsearch/issues/117805
-- class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS2UnavailableRemotesIT
-  method: testEsqlRcs2UnavailableRemoteScenarios
-  issue: https://github.com/elastic/elasticsearch/issues/117419
 - class: org.elasticsearch.xpack.esql.action.EsqlActionTaskIT
   method: testCancelRequestWhenFailingFetchingPages
   issue: https://github.com/elastic/elasticsearch/issues/118193
@@ -149,9 +146,6 @@ tests:
 - class: org.elasticsearch.action.search.SearchQueryThenFetchAsyncActionTests
   method: testBottomFieldSort
   issue: https://github.com/elastic/elasticsearch/issues/118214
-- class: org.elasticsearch.xpack.remotecluster.CrossClusterEsqlRCS1UnavailableRemotesIT
-  method: testEsqlRcs1UnavailableRemoteScenarios
-  issue: https://github.com/elastic/elasticsearch/issues/118350
 - class: org.elasticsearch.xpack.searchablesnapshots.RetrySearchIntegTests
   method: testSearcherId
   issue: https://github.com/elastic/elasticsearch/issues/118374
@@ -205,9 +199,6 @@ tests:
 - class: org.elasticsearch.oldrepos.OldRepositoryAccessIT
   method: testOldSourceOnlyRepoAccess
   issue: https://github.com/elastic/elasticsearch/issues/120080
-- class: org.elasticsearch.xpack.migrate.action.ReindexDatastreamIndexTransportActionIT
-  method: testTsdbStartEndSet
-  issue: https://github.com/elastic/elasticsearch/issues/120314
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=snapshot/10_basic/Failed to snapshot indices with synthetic source}
   issue: https://github.com/elastic/elasticsearch/issues/120332
@@ -246,6 +237,14 @@ tests:
 - class: org.elasticsearch.action.search.SearchProgressActionListenerIT
   method: testSearchProgressWithHits
   issue: https://github.com/elastic/elasticsearch/issues/120671
+- class: org.elasticsearch.xpack.ml.integration.LearningToRankExplainIT
+  method: testLtrExplainWithMultipleShardsAndReplicas
+  issue: https://github.com/elastic/elasticsearch/issues/120805
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  method: test {p0=ml/3rd_party_deployment/Test start deployment fails while model download in progress}
+  issue: https://github.com/elastic/elasticsearch/issues/120810
+- class: org.elasticsearch.xpack.test.rest.XPackRestIT
+  issue: https://github.com/elastic/elasticsearch/issues/120816
 
 # Examples:
 #

rest-api-spec/src/main/resources/rest-api-spec/api/async_search.submit.json

@@ -43,6 +43,11 @@
         "description":"Control whether the response should be stored in the cluster if it completed within the provided [wait_for_completion] time (default: false)",
         "default":false
       },
+      "keep_alive": {
+        "type": "time",
+        "description": "Update the time interval in which the results (partial or final) for this search will be available",
+        "default": "5d"
+      },
       "batched_reduce_size":{
         "type":"number",
         "description":"The number of shard results that should be reduced at once on the coordinating node. This value should be used as the granularity at which progress results will be made available.",

server/src/main/java/org/elasticsearch/ReleaseVersions.java

@@ -10,7 +10,7 @@
 package org.elasticsearch;
 
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.core.UpdateForV9;
+import org.elasticsearch.core.UpdateForV10;
 import org.elasticsearch.internal.BuildExtension;
 import org.elasticsearch.plugins.ExtensionLoader;
 
@@ -114,8 +114,7 @@ private static IntFunction<String> lookupFunction(NavigableMap<Integer, List<Ver
                     // this will no longer be the case with ES 10 (which won't know about ES v8.x where we introduced separated versions)
                     // maybe keep the release mapping around in the csv file?
                     // SEP for now
-                    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-                    // @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA)
+                    @UpdateForV10(owner = UpdateForV10.Owner.CORE_INFRA)
                     Version oldVersion = Version.fromId(id);
                     return oldVersion.toString();
                 }

server/src/main/java/org/elasticsearch/Version.java

@@ -14,8 +14,8 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.Assertions;
+import org.elasticsearch.core.RestApiVersion;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.monitor.jvm.JvmInfo;
 import org.elasticsearch.xcontent.ToXContentFragment;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -242,17 +242,15 @@ public class Version implements VersionId<Version>, ToXContentFragment {
         VERSION_STRINGS = Map.copyOf(builderByString);
     }
 
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-    // Re-enable this assertion once the rest api version is bumped
     private static void assertRestApiVersion() {
-        // assert RestApiVersion.current().major == CURRENT.major && RestApiVersion.previous().major == CURRENT.major - 1
-        // : "RestApiVersion must be upgraded "
-        // + "to reflect major from Version.CURRENT ["
-        // + CURRENT.major
-        // + "]"
-        // + " but is still set to ["
-        // + RestApiVersion.current().major
-        // + "]";
+        assert RestApiVersion.current().major == CURRENT.major && RestApiVersion.previous().major == CURRENT.major - 1
+            : "RestApiVersion must be upgraded "
+                + "to reflect major from Version.CURRENT ["
+                + CURRENT.major
+                + "]"
+                + " but is still set to ["
+                + RestApiVersion.current().major
+                + "]";
     }
 
     public static Version readVersion(StreamInput in) throws IOException {

server/src/main/java/org/elasticsearch/cluster/ClusterState.java

@@ -47,7 +47,6 @@
 import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.UpdateForV9;
 import org.elasticsearch.index.shard.IndexLongFieldRange;
 import org.elasticsearch.indices.SystemIndexDescriptor;
 import org.elasticsearch.xcontent.ToXContent;
@@ -1039,12 +1038,6 @@ public static ClusterState readFrom(StreamInput in, DiscoveryNode localNode) thr
         return builder.build();
     }
 
-    /**
-     * If the cluster state does not contain transport version information, this is the version
-     * that is inferred for all nodes on version 8.8.0 or above.
-     */
-    @UpdateForV9(owner = UpdateForV9.Owner.CORE_INFRA)
-    public static final TransportVersion INFERRED_TRANSPORT_VERSION = TransportVersions.V_8_8_0;
     public static final Version VERSION_INTRODUCING_TRANSPORT_VERSIONS = Version.V_8_8_0;
 
     @Override

server/src/main/java/org/elasticsearch/env/BuildVersion.java

@@ -73,6 +73,12 @@ public abstract class BuildVersion implements ToXContentFragment, Writeable {
      */
     public abstract String toNodeMetadata();
 
+    /**
+     * Returns the minimum compatible build version based on the current version.
+     * Ie a node needs to have at least the return version in order to communicate with a node running the current version.
+     */
+    public abstract BuildVersion minimumCompatibilityVersion();
+
     /**
      * Create a {@link BuildVersion} from a version ID number.
      *

server/src/main/java/org/elasticsearch/env/DefaultBuildVersion.java

@@ -73,6 +73,11 @@ public String toNodeMetadata() {
         return Integer.toString(version.id());
     }
 
+    @Override
+    public BuildVersion minimumCompatibilityVersion() {
+        return fromVersionId(version.minimumCompatibilityVersion().id);
+    }
+
     @Override
     public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
         return builder.value(version.id());