Merge branch 'elastic:main' into fix-116487

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -145,7 +145,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
 
     doLast {
       ['main', 'test'].each { sourceSet ->
-        modifyXml(".idea/modules/libs/native/elasticsearch.libs.${project.project(':libs:native').name}.${sourceSet}.iml") { xml ->
+        modifyXml(".idea/modules/libs/native/elasticsearch.libs.native.${sourceSet}.iml") { xml ->
           xml.component.find { it.'@name' == 'NewModuleRootManager' }?.'@LANGUAGE_LEVEL' = 'JDK_21_PREVIEW'
         }
       }

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java

@@ -137,7 +137,7 @@ public void skipTest(String fullTestName, String reason) {
         // However, the folder can be arbitrarily nest so, a == a1/a2/a3, and the test name can include forward slashes, so c == c1/c2/c3
         // So we also need to support a1/a2/a3/b/c1/c2/c3
 
-        String[] testParts = fullTestName.split("/");
+        String[] testParts = fullTestName.split("/", 3);
         if (testParts.length < 3) {
             throw new IllegalArgumentException(
                 "To skip tests, all 3 parts [folder/file/test name] must be defined. found [" + fullTestName + "]"

docs/changelog/115091.yaml

@@ -0,0 +1,7 @@
+pr: 115091
+summary: Added stricter range type checks and runtime warnings for ENRICH
+area: ES|QL
+type: bug
+issues:
+ - 107357
+ - 116799

docs/changelog/116515.yaml

@@ -0,0 +1,5 @@
+pr: 116515
+summary: Esql/lookup join grammar
+area: ES|QL
+type: feature
+issues: []

docs/changelog/116689.yaml

@@ -0,0 +1,10 @@
+pr: 116689
+summary: Deprecate `_source.mode` in mappings
+area: Mapping
+type: deprecation
+issues: []
+deprecation:
+  title: Deprecate `_source.mode` in mappings
+  area: Mapping
+  details: Configuring `_source.mode` in mappings is deprecated and will be removed in future versions. Use `index.mapping.source.mode` index setting instead.
+  impact: Use `index.mapping.source.mode` index setting instead

docs/changelog/116944.yaml

@@ -0,0 +1,11 @@
+pr: 116944
+summary: "Remove support for type, fields, `copy_to` and boost in metadata field definition"
+area: Mapping
+type: breaking
+issues: []
+breaking:
+  title: "Remove support for type, fields, copy_to and boost in metadata field definition"
+  area: Mapping
+  details: The type, fields, copy_to and boost parameters are no longer supported in metadata field definition
+  impact: Users providing type, fields, copy_to or boost as part of metadata field definition should remove them from their mappings.
+  notable: false

docs/changelog/116970.yaml

@@ -0,0 +1,11 @@
+pr: 116970
+summary: Remove legacy params from range query
+area: Search
+type: breaking
+issues: []
+breaking:
+  title: Remove legacy params from range query
+  area: REST API
+  details: The deprecated range query parameters `to`, `from`, `include_lower`, and `include_upper` are no longer supported.
+  impact: Users should use `lt`, `lte`, `gt`, and `gte` query parameters instead.
+  notable: false

docs/reference/cluster/allocation-explain.asciidoc

@@ -159,6 +159,8 @@ node.
 <5> The decider which led to the `no` decision for the node.
 <6> An explanation as to why the decider returned a `no` decision, with a helpful hint pointing to the setting that led to the decision. In this example, a newly created index has <<indices-get-settings,an index setting>> that requires that it only be allocated to a node named `nonexistent_node`, which does not exist, so the index is unable to allocate.
 
+See https://www.youtube.com/watch?v=5z3n2VgusLE[this video] for a walkthrough of troubleshooting a node and index setting mismatch.
+
 [[maximum-number-of-retries-exceeded]]
 ====== Maximum number of retries exceeded
 
@@ -235,7 +237,9 @@ primary shard that was previously allocated.
 ----
 // NOTCONSOLE
 
-TIP: If a shard is unassigned with an allocation status of `no_valid_shard_copy`, then you should <<fix-cluster-status-recover-nodes,make sure that all nodes are in the cluster>>. If all the nodes containing in-sync copies of a shard are lost, then you can <<fix-cluster-status-restore,recover the data for the shard>>.
+If a shard is unassigned with an allocation status of `no_valid_shard_copy`, then you should <<fix-cluster-status-recover-nodes,make sure that all nodes are in the cluster>>. If all the nodes containing in-sync copies of a shard are lost, then you can <<fix-cluster-status-restore,recover the data for the shard>>. 
+
+See https://www.youtube.com/watch?v=6OAg9IyXFO4[this video] for a walkthrough of troubleshooting `no_valid_shard_copy`.
 
 ===== Unassigned replica shard
 

docs/reference/esql/esql-commands.asciidoc

@@ -38,12 +38,12 @@ image::images/esql/processing-command.svg[A processing command changing an input
 * <<esql-eval>>
 * <<esql-grok>>
 ifeval::["{release-state}"=="unreleased"]
-* experimental:[] <<esql-inlinestats-by>>
+//* experimental:[] <<esql-inlinestats-by>>
 endif::[]
 * <<esql-keep>>
 * <<esql-limit>>
 ifeval::["{release-state}"=="unreleased"]
-* experimental:[] <<esql-lookup>>
+//* experimental:[] <<esql-lookup>>
 endif::[]
 * experimental:[] <<esql-mv_expand>>
 * <<esql-rename>>
@@ -63,12 +63,12 @@ include::processing-commands/enrich.asciidoc[]
 include::processing-commands/eval.asciidoc[]
 include::processing-commands/grok.asciidoc[]
 ifeval::["{release-state}"=="unreleased"]
-include::processing-commands/inlinestats.asciidoc[]
+//include::processing-commands/inlinestats.asciidoc[]
 endif::[]
 include::processing-commands/keep.asciidoc[]
 include::processing-commands/limit.asciidoc[]
 ifeval::["{release-state}"=="unreleased"]
-include::processing-commands/lookup.asciidoc[]
+//include::processing-commands/lookup.asciidoc[]
 endif::[]
 include::processing-commands/mv_expand.asciidoc[]
 include::processing-commands/rename.asciidoc[]

docs/reference/esql/esql-enrich-data.asciidoc

@@ -138,8 +138,33 @@ include::{es-ref-dir}/ingest/apis/enrich/execute-enrich-policy.asciidoc[tag=upda
 
 include::../ingest/enrich.asciidoc[tag=update-enrich-policy]
 
-==== Limitations
+==== Enrich Policy Types and Limitations
+The {esql} `ENRICH` command supports all three enrich policy types:
+
+`geo_match`::
+Matches enrich data to incoming documents based on a <<query-dsl-geo-shape-query,`geo_shape` query>>.
+For an example, see <<geo-match-enrich-policy-type>>.
+
+`match`::
+Matches enrich data to incoming documents based on a <<query-dsl-term-query,`term` query>>.
+For an example, see <<match-enrich-policy-type>>.
+
+`range`::
+Matches a number, date, or IP address in incoming documents to a range in the
+enrich index based on a <<query-dsl-term-query,`term` query>>. For an example,
+see <<range-enrich-policy-type>>.
+
 // tag::limitations[]
-The {esql} `ENRICH` command only supports enrich policies of type `match`.
-Furthermore, `ENRICH` only supports enriching on a column of type `keyword`.
+While all three enrich policy types are supported, there are some limitations to be aware of:
+
+* The `geo_match` enrich policy type only supports the `intersects` spatial relation.
+* It is required that the `match_field` in the `ENRICH` command is of the correct type.
+For example, if the enrich policy is of type `geo_match`, the `match_field` in the `ENRICH`
+command must be of type `geo_point` or `geo_shape`.
+Likewise, a `range` enrich policy requires a `match_field` of type `integer`, `long`, `date`, or `ip`,
+depending on the type of the range field in the original enrich index.
+* However, this constraint is relaxed for `range` policies when the `match_field` is of type `KEYWORD`.
+In this case the field values will be parsed during query execution, row by row.
+If any value fails to parse, the output values for that row will be set to `null`,
+an appropriate warning will be produced and the query will continue to execute.
 // end::limitations[]

docs/reference/esql/esql-query-api.asciidoc

@@ -92,8 +92,8 @@ https://en.wikipedia.org/wiki/Query_plan[EXPLAIN PLAN].
 
 
 ifeval::["{release-state}"=="unreleased"]
-`table`::
-(Optional, object) Named "table" parameters that can be referenced by the <<esql-lookup>> command.
+//`table`::
+//(Optional, object) Named "table" parameters that can be referenced by the <<esql-lookup>> command.
 endif::[]
 
 [discrete]

docs/reference/how-to.asciidoc

@@ -1,23 +1,21 @@
 [[how-to]]
-= How to
+= Optimizations
 
-[partintro]
---
-Elasticsearch ships with defaults which are intended to give a good out of
-the box experience. Full text search, highlighting, aggregations, and indexing
-should all just work without the user having to change anything.
+Elasticsearch's default settings provide a good out-of-box experience for basic operations like full text search, highlighting, aggregations, and indexing. 
 
-Once you better understand how you want to use Elasticsearch, however,
-there are a number of optimizations you can make to improve performance
-for your use case.
+However, there are a number of optimizations you can make to improve performance for your use case.
 
-This section provides guidance about which changes should and shouldn't be
-made.
---
+This section provides recommendations for various use cases.
 
-include::how-to/general.asciidoc[]
+* <<general-recommendations>>
+* <<tune-for-indexing-speed>>
+* <<tune-for-search-speed>>
+* <<tune-knn-search>>
+* <<tune-for-disk-usage>>
+* <<size-your-shards>>
+* <<use-elasticsearch-for-time-series-data>>
 
-include::how-to/recipes.asciidoc[]
+include::how-to/general.asciidoc[]
 
 include::how-to/indexing-speed.asciidoc[]
 

docs/reference/how-to/recipes.asciidoc

@@ -1,7 +1,7 @@
 [[recipes]]
-== Recipes
+== Search relevance optimizations
 
-This section includes a few recipes to help with common problems:
+This section includes a few recipes to help with common search relevance issues:
 
 * <<mixing-exact-search-with-stemming,Mixing exact search with stemming>>
 * <<consistent-scoring,Getting consistent scores>>

docs/reference/how-to/recipes/scoring.asciidoc

@@ -88,8 +88,9 @@ pages independently of the query.
 
 There are two main queries that allow combining static score contributions with
 textual relevance, eg. as computed with BM25:
- - <<query-dsl-script-score-query,`script_score` query>>
- - <<query-dsl-rank-feature-query,`rank_feature` query>>
+
+* <<query-dsl-script-score-query,`script_score` query>>
+* <<query-dsl-rank-feature-query,`rank_feature` query>>
 
 For instance imagine that you have a `pagerank` field that you wish to
 combine with the BM25 score so that the final score is equal to

docs/reference/ilm/error-handling.asciidoc

@@ -8,8 +8,9 @@ When this happens, {ilm-init} moves the index to an `ERROR` step.
 If {ilm-init} cannot resolve the error automatically, execution is halted  
 until you resolve the underlying issues with the policy, index, or cluster.
 
-See this https://www.youtube.com/watch?v=VCIqkji3IwY[{ilm-init} health video] 
-for example troubleshooting walkthrough.
+See https://www.youtube.com/watch?v=VCIqkji3IwY[this video] 
+for a walkthrough of troubleshooting current {ilm-init} health issues, and https://www.youtube.com/watch?v=onrnnwjYWSQ[this video]
+for a walkthrough of troubleshooting historical {ilm-init} issues.
 
 For example, you might have a `shrink-index` policy that shrinks an index to four shards once it
 is at least five days old: 

docs/reference/search/search-your-data/search-your-data.asciidoc

@@ -43,10 +43,12 @@ DSL, with a simplified user experience. Create search applications based on your
 results directly in the Kibana Search UI.
 
 include::search-api.asciidoc[]
+include::../../how-to/recipes.asciidoc[]
+// ☝️ search relevance recipes
 include::retrievers-overview.asciidoc[]
 include::knn-search.asciidoc[]
 include::semantic-search.asciidoc[]
 include::search-across-clusters.asciidoc[]
 include::search-with-synonyms.asciidoc[]
 include::search-application-overview.asciidoc[]
-include::behavioral-analytics/behavioral-analytics-overview.asciidoc[]
+include::behavioral-analytics/behavioral-analytics-overview.asciidoc[]

docs/reference/snapshot-restore/repository-s3.asciidoc

@@ -6,6 +6,9 @@ You can use AWS S3 as a repository for {ref}/snapshot-restore.html[Snapshot/Rest
 *If you are looking for a hosted solution of Elasticsearch on AWS, please visit
 https://www.elastic.co/cloud/.*
 
+See https://www.youtube.com/watch?v=ACqfyzWf-xs[this video]
+for a walkthrough of connecting an AWS S3 repository.
+
 [[repository-s3-usage]]
 ==== Getting started
 

docs/reference/troubleshooting/common-issues/diagnose-unassigned-shards.asciidoc

@@ -8,5 +8,6 @@ In order to diagnose the unassigned shards in your deployment use the following
 
 include::{es-ref-dir}/tab-widgets/troubleshooting/data/diagnose-unassigned-shards-widget.asciidoc[]
 
-
+See https://www.youtube.com/watch?v=v2mbeSd1vTQ[this video]
+for a walkthrough of monitoring allocation health.
 

docs/reference/troubleshooting/common-issues/red-yellow-cluster-status.asciidoc

@@ -19,6 +19,8 @@ operate to have a green health status.
 In many cases, your cluster will recover to green health status automatically.
 If the cluster doesn't automatically recover, then you must <<fix-red-yellow-cluster-status,manually address>>
 the remaining problems so management and cleanup activities can proceed.
+See https://www.youtube.com/watch?v=v2mbeSd1vTQ[this video]
+for a walkthrough of monitoring allocation health.
 
 [discrete]
 [[diagnose-cluster-status]]
@@ -90,6 +92,8 @@ PUT _cluster/settings
 }
 ----
 
+See https://www.youtube.com/watch?v=MiKKUdZvwnI[this video] for walkthrough of troubleshooting "no allocations are allowed".
+
 [discrete]
 [[fix-cluster-status-recover-nodes]]
 ===== Recover lost nodes
@@ -262,3 +266,5 @@ POST _cluster/reroute
 ----
 // TEST[s/^/PUT my-index\n/]
 // TEST[catch:bad_request]
+
+See https://www.youtube.com/watch?v=6OAg9IyXFO4[this video] for a walkthrough of troubleshooting `no_valid_shard_copy`.

docs/reference/troubleshooting/diagnostic.asciidoc

@@ -13,6 +13,8 @@ This information can be used to troubleshoot problems with your cluster. For exa
 You can generate diagnostic information using this tool before you contact https://support.elastic.co[Elastic Support] or 
 https://discuss.elastic.co[Elastic Discuss] to minimize turnaround time. 
 
+See this https://www.youtube.com/watch?v=Bb6SaqhqYHw[this video] for a walkthrough of capturing an {es} diagnostic.
+
 [discrete]
 [[diagnostic-tool-requirements]]
 === Requirements

gradle/build.versions.toml

@@ -16,7 +16,7 @@ checkstyle = "com.puppycrawl.tools:checkstyle:10.3"
 commons-codec = "commons-codec:commons-codec:1.11"
 commmons-io = "commons-io:commons-io:2.2"
 docker-compose = "com.avast.gradle:gradle-docker-compose-plugin:0.17.5"
-forbiddenApis = "de.thetaphi:forbiddenapis:3.6"
+forbiddenApis = "de.thetaphi:forbiddenapis:3.8"
 gradle-enterprise = "com.gradle:develocity-gradle-plugin:3.18.1"
 hamcrest = "org.hamcrest:hamcrest:2.1"
 httpcore = "org.apache.httpcomponents:httpcore:4.4.12"

gradle/verification-metadata.xml

@@ -1336,6 +1336,11 @@
             <sha256 value="7b478e69fb613a05c2e1cb0ecfd257618cfb477333660914440a30b5afc712ff" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="de.thetaphi" name="forbiddenapis" version="3.8">
+         <artifact name="forbiddenapis-3.8.jar">
+            <sha256 value="c8f7d13e5ac1cbbd605616b8882bf1693047a0351ac9e70c7b42fb6d43b3240d" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="dev.equo.ide" name="solstice" version="1.7.4">
          <artifact name="solstice-1.7.4.jar">
             <sha256 value="1288c1010299cb8434218e370f2291f2d6ea49b3ccb34b5082080f7c270a254c" origin="Generated by Gradle"/>

libs/logstash-bridge/src/main/java/org/elasticsearch/logstashbridge/ingest/PipelineConfigurationBridge.java

@@ -28,8 +28,12 @@ public String getId() {
         return delegate.getId();
     }
 
-    public Map<String, Object> getConfigAsMap() {
-        return delegate.getConfigAsMap();
+    public Map<String, Object> getConfig() {
+        return delegate.getConfig();
+    }
+
+    public Map<String, Object> getConfig(final boolean unmodifiable) {
+        return delegate.getConfig(unmodifiable);
     }
 
     @Override

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/AbstractDataStreamIT.java

@@ -45,6 +45,7 @@ public abstract class AbstractDataStreamIT extends ESRestTestCase {
         // tests such as testIgnoreDynamicBeyondLimit.
         .setting("xpack.apm_data.enabled", "false")
         .setting("xpack.otel_data.registry.enabled", "false")
+        .setting("cluster.logsdb.enabled", "false")
         .build();
     protected RestClient client;
 

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloaderTaskExecutor.java

@@ -268,7 +268,7 @@ private static Set<String> pipelinesWithGeoIpProcessor(ClusterState clusterState
         Set<String> ids = new HashSet<>();
         // note: this loop is unrolled rather than streaming-style because it's hot enough to show up in a flamegraph
         for (PipelineConfiguration configuration : configurations) {
-            List<Map<String, Object>> processors = (List<Map<String, Object>>) configuration.getConfigAsMap().get(Pipeline.PROCESSORS_KEY);
+            List<Map<String, Object>> processors = (List<Map<String, Object>>) configuration.getConfig().get(Pipeline.PROCESSORS_KEY);
             if (hasAtLeastOneGeoipProcessor(processors, downloadDatabaseOnPipelineCreation)) {
                 ids.add(configuration.getId());
             }

modules/rank-eval/build.gradle

@@ -25,7 +25,3 @@ testClusters.configureEach {
   // Modules who's integration is explicitly tested in integration tests
   module ':modules:lang-mustache'
 }
-
-tasks.named("yamlRestCompatTestTransform").configure({ task ->
-  task.skipTest("rank_eval/30_failures/Response format", "warning does not exist for compatibility")
-})

modules/runtime-fields-common/build.gradle

@@ -22,7 +22,3 @@ dependencies {
   api project(':libs:grok')
   api project(':libs:dissect')
 }
-
-tasks.named("yamlRestCompatTestTransform").configure({ task ->
-  task.skipTestsByFilePattern("**/runtime_fields/110_composite.yml", "warning does not exist for compatibility")
-})