Merge branch 'main' into mormalizeGeoTile

.backportrc.json

@@ -1,10 +1,10 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.x", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.x", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
     "^v9.0.0$" : "main",
-    "^v8.16.0$" : "8.x",
+    "^v8.17.0$" : "8.x",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/intake.template.yml

@@ -75,6 +75,7 @@ steps:
   - trigger: elasticsearch-dra-workflow
     label: Trigger DRA snapshot workflow
     async: true
+    branches: "main 8.* 7.17"
     build:
       branch: "$BUILDKITE_BRANCH"
       commit: "$BUILDKITE_COMMIT"

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -304,6 +304,22 @@ steps:
         env:
           BWC_VERSION: 8.16.0
 
+      - label: "{{matrix.image}} / 8.17.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.17.0
+
       - label: "{{matrix.image}} / 9.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -325,6 +325,25 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 8.17.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: 8.17.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
       - label: 9.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.0#bwcTest
         timeout_in_minutes: 300
@@ -410,7 +429,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -452,7 +471,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -554,7 +573,7 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.ci/bwcVersions

@@ -16,4 +16,5 @@ BWC_VERSION:
   - "8.14.3"
   - "8.15.3"
   - "8.16.0"
+  - "8.17.0"
   - "9.0.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,5 @@
 BWC_VERSION:
   - "8.15.3"
   - "8.16.0"
+  - "8.17.0"
   - "9.0.0"

branches.json

@@ -4,6 +4,9 @@
     {
       "branch": "main"
     },
+    {
+      "branch": "8.16"
+    },
     {
       "branch": "8.x"
     },

docs/changelog/114271.yaml

@@ -0,0 +1,5 @@
+pr: 114271
+summary: "[ES|QL] Skip validating remote cluster index names in parser"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/114784.yaml

@@ -0,0 +1,5 @@
+pr: 114784
+summary: "[ES|QL] make named parameter for identifier and pattern snapshot"
+area: ES|QL
+type: bug
+issues: []

docs/changelog/114854.yaml

@@ -0,0 +1,10 @@
+pr: 114854
+summary: Adding deprecation warnings for rrf using rank and `sub_searches`
+area: Search
+type: deprecation
+issues: []
+deprecation:
+  title: Adding deprecation warnings for rrf using rank and `sub_searches`
+  area: REST API
+  details: Search API parameter `sub_searches` will no longer be a supported and will be removed in future releases. Similarly, `rrf` can only be used through the specified `retriever` and no longer though the `rank` parameter
+  impact: Requests specifying rrf through `rank` and/or `sub_searches` elements will be disallowed in a future version. Users should instead utilize the new `retriever` parameter.

docs/changelog/114869.yaml

@@ -0,0 +1,5 @@
+pr: 114869
+summary: Standardize error code when bulk body is invalid
+area: CRUD
+type: bug
+issues: []

docs/changelog/114924.yaml

@@ -0,0 +1,5 @@
+pr: 114924
+summary: Reducing error-level stack trace logging for normal events in `GeoIpDownloader`
+area: Ingest Node
+type: bug
+issues: []

docs/reference/ilm/error-handling.asciidoc

@@ -8,6 +8,9 @@ When this happens, {ilm-init} moves the index to an `ERROR` step.
 If {ilm-init} cannot resolve the error automatically, execution is halted  
 until you resolve the underlying issues with the policy, index, or cluster.
 
+See this https://www.youtube.com/watch?v=VCIqkji3IwY[{ilm-init} health video] 
+for example troubleshooting walkthrough.
+
 For example, you might have a `shrink-index` policy that shrinks an index to four shards once it
 is at least five days old: 
 
@@ -183,6 +186,8 @@ The rollover action then manages setting and updating the alias to
 
 Do not explicitly configure this same alias in the aliases section of an index template.
 
+See this https://www.youtube.com/watch?v=Ww5POq4zZtY[resolving `duplicate alias` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== index.lifecycle.rollover_alias [x] does not point to index [y]
 
@@ -191,20 +196,26 @@ Either the index is using the wrong alias or the alias does not exist.
 Check the `index.lifecycle.rollover_alias` <<indices-get-settings, index setting>>.
 To see what aliases are configured, use <<cat-alias, _cat/aliases>>.
 
+See this https://www.youtube.com/watch?v=NKSe67x7aw8[resolving `not point to index` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== Setting [index.lifecycle.rollover_alias] for index [y] is empty or not defined
 
 The `index.lifecycle.rollover_alias` setting must be configured for the rollover action to work.
 
 Update the index settings to set `index.lifecycle.rollover_alias`.
 
+See this https://www.youtube.com/watch?v=LRpMC2GS_FQ[resolving `empty or not defined` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== Alias [x] has more than one write index [y,z]
 
 Only one index can be designated as the write index for a particular alias.
 
 Use the <<indices-aliases, aliases>> API to set `is_write_index:false` for all but one index.
 
+See this https://www.youtube.com/watch?v=jCUvZCT5Hm4[resolving `more than one write index` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== index name [x] does not match pattern ^.*-\d+
 
@@ -214,6 +225,8 @@ For example, `my-index` does not match the pattern requirement.
 
 Append a numeric value to the index name, for example `my-index-000001`.
 
+See this https://www.youtube.com/watch?v=9sp1zF6iL00[resolving `does not match pattern` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== CircuitBreakingException: [x] data too large, data for [y]
 
@@ -227,8 +240,7 @@ For more information, see <<circuit-breaker-errors>>.
 
 This indicates that the cluster is running out of disk space.
 This can happen when you don't have {ilm} set up to roll over from hot to warm nodes.
-
-Consider adding nodes, upgrading your hardware, or deleting unneeded indices.
+For more information, see <<fix-watermark-errors,Fix watermark errors>>.
 
 [discrete]
 ==== security_exception: action [<action-name>] is unauthorized for user [<user-name>] with roles [<role-name>], this action is granted by the index privileges [manage_follow_index,manage,all]

docs/reference/migration/index.asciidoc

@@ -1,5 +1,6 @@
 include::migration_intro.asciidoc[]
 
+* <<migrating-8.17,Migrating to 8.17>>
 * <<migrating-8.16,Migrating to 8.16>>
 * <<migrating-8.15,Migrating to 8.15>>
 * <<migrating-8.14,Migrating to 8.14>>
@@ -18,6 +19,7 @@ include::migration_intro.asciidoc[]
 * <<migrating-8.1,Migrating to 8.1>>
 * <<migrating-8.0,Migrating to 8.0>>
 
+include::migrate_8_17.asciidoc[]
 include::migrate_8_16.asciidoc[]
 include::migrate_8_15.asciidoc[]
 include::migrate_8_14.asciidoc[]

docs/reference/migration/migrate_8_17.asciidoc

@@ -0,0 +1,20 @@
+[[migrating-8.17]]
+== Migrating to 8.17
+++++
+<titleabbrev>8.17</titleabbrev>
+++++
+
+This section discusses the changes that you need to be aware of when migrating
+your application to {es} 8.17.
+
+See also <<release-highlights>> and <<es-release-notes>>.
+
+coming::[8.17.0]
+
+
+[discrete]
+[[breaking-changes-8.17]]
+=== Breaking changes
+
+There are no breaking changes in {es} 8.17.
+

docs/reference/release-notes.asciidoc

@@ -6,6 +6,7 @@
 
 This section summarizes the changes in each release.
 
+* <<release-notes-8.17.0>>
 * <<release-notes-8.16.0>>
 * <<release-notes-8.15.1>>
 * <<release-notes-8.15.0>>
@@ -72,6 +73,7 @@ This section summarizes the changes in each release.
 
 --
 
+include::release-notes/8.17.0.asciidoc[]
 include::release-notes/8.16.0.asciidoc[]
 include::release-notes/8.15.1.asciidoc[]
 include::release-notes/8.15.0.asciidoc[]

docs/reference/release-notes/8.17.0.asciidoc

@@ -0,0 +1,8 @@
+[[release-notes-8.17.0]]
+== {es} version 8.17.0
+
+coming[8.17.0]
+
+Also see <<breaking-changes-8.17,Breaking changes in 8.17>>.
+
+

docs/reference/release-notes/highlights.asciidoc

@@ -11,7 +11,8 @@ For detailed information about this release, see the <<es-release-notes>> and
 // Add previous release to the list
 Other versions:
 
-{ref-bare}/8.15/release-highlights.html[8.15]
+{ref-bare}/8.16/release-highlights.html[8.16]
+| {ref-bare}/8.15/release-highlights.html[8.15]
 | {ref-bare}/8.14/release-highlights.html[8.14]
 | {ref-bare}/8.13/release-highlights.html[8.13]
 | {ref-bare}/8.12/release-highlights.html[8.12]
@@ -30,6 +31,8 @@ Other versions:
 
 endif::[]
 
+// The notable-highlights tag marks entries that
+// should be featured in the Stack Installation and Upgrade Guide:
 // tag::notable-highlights[]
 
 [discrete]
@@ -97,29 +100,3 @@ ZStandard offers ~12% lower storage usage and a ~14% higher indexing throughput
 // end::notable-highlights[]
 
 
-[discrete]
-[[esql_multi_value_fields_supported_in_geospatial_predicates]]
-=== ESQL: Multi-value fields supported in Geospatial predicates
-Supporting multi-value fields in `WHERE` predicates is a challenge due to not knowing whether `ALL` or `ANY`
-of the values in the field should pass the predicate.
-For example, should the field `age:[10,30]` pass the predicate `WHERE age>20` or not?
-This ambiguity does not exist with the spatial predicates
-`ST_INTERSECTS` and `ST_DISJOINT`, because the choice between `ANY` or `ALL`
-is implied by the predicate itself.
-Consider a predicate checking a field named `location` against a test geometry named `shape`:
-
-* `ST_INTERSECTS(field, shape)` - true if `ANY` value can intersect the shape
-* `ST_DISJOINT(field, shape)` - true only if `ALL` values are disjoint from the shape
-
-This works even if the shape argument is itself a complex or compound geometry.
-
-Similar logic exists for `ST_CONTAINS` and `ST_WITHIN` predicates, but these are not as easily solved
-with `ANY` or `ALL`, because a collection of geometries contains another collection if each of the contained
-geometries is within at least one of the containing geometries. Evaluating this requires that the multi-value
-field is first combined into a single geometry before performing the predicate check.
-
-* `ST_CONTAINS(field, shape)` - true if the combined geometry contains the shape
-* `ST_WITHIN(field, shape)` - true if the combined geometry is within the shape
-
-{es-pull}112063[#112063]
-

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDownloader.java

@@ -11,7 +11,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.flush.FlushRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
@@ -139,11 +138,18 @@ void updateDatabases() throws IOException {
         if (geoipIndex != null) {
             logger.trace("The {} index is not null", GeoIpDownloader.DATABASES_INDEX);
             if (clusterState.getRoutingTable().index(geoipIndex.getWriteIndex()).allPrimaryShardsActive() == false) {
-                throw new ElasticsearchException("not all primary shards of [" + DATABASES_INDEX + "] index are active");
+                logger.debug(
+                    "Not updating geoip database because not all primary shards of the [" + DATABASES_INDEX + "] index are active."
+                );
+                return;
             }
             var blockException = clusterState.blocks().indexBlockedException(ClusterBlockLevel.WRITE, geoipIndex.getWriteIndex().getName());
             if (blockException != null) {
-                throw blockException;
+                logger.debug(
+                    "Not updating geoip database because there is a write block on the " + geoipIndex.getWriteIndex().getName() + " index",
+                    blockException
+                );
+                return;
             }
         }
         if (eagerDownloadSupplier.get() || atLeastOneGeoipProcessorSupplier.get()) {