Merge branch 'elastic:main' into create-kql-plugin

.backportrc.json

@@ -1,10 +1,10 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "8.x", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "8.x", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
     "^v9.0.0$" : "main",
-    "^v8.16.0$" : "8.x",
+    "^v8.17.0$" : "8.x",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/pipelines/intake.template.yml

@@ -75,6 +75,7 @@ steps:
   - trigger: elasticsearch-dra-workflow
     label: Trigger DRA snapshot workflow
     async: true
+    branches: "main 8.* 7.17"
     build:
       branch: "$BUILDKITE_BRANCH"
       commit: "$BUILDKITE_COMMIT"

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -304,6 +304,22 @@ steps:
         env:
           BWC_VERSION: 8.16.0
 
+      - label: "{{matrix.image}} / 8.17.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.17.0
+
       - label: "{{matrix.image}} / 9.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -325,6 +325,25 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 8.17.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: 8.17.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
       - label: 9.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.0#bwcTest
         timeout_in_minutes: 300
@@ -410,7 +429,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -452,7 +471,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.15.3", "8.16.0", "9.0.0"]
+            BWC_VERSION: ["8.15.3", "8.16.0", "8.17.0", "9.0.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -554,7 +573,7 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n2-standard-8
       buildDirectory: /dev/shm/bk
-    if: build.branch == "main" || build.branch == "7.17"
+    if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
   - label: check-branch-consistency
     command: .ci/scripts/run-gradle.sh branchConsistency
     timeout_in_minutes: 15

.ci/bwcVersions

@@ -16,4 +16,5 @@ BWC_VERSION:
   - "8.14.3"
   - "8.15.3"
   - "8.16.0"
+  - "8.17.0"
   - "9.0.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,5 @@
 BWC_VERSION:
   - "8.15.3"
   - "8.16.0"
+  - "8.17.0"
   - "9.0.0"

branches.json

@@ -4,6 +4,9 @@
     {
       "branch": "main"
     },
+    {
+      "branch": "8.16"
+    },
     {
       "branch": "8.x"
     },

docs/changelog/114837.yaml

@@ -0,0 +1,5 @@
+pr: 114837
+summary: Add warning headers for ingest pipelines containing special characters
+area: Ingest Node
+type: bug
+issues: [ 104411 ]

docs/changelog/114869.yaml

@@ -0,0 +1,5 @@
+pr: 114869
+summary: Standardize error code when bulk body is invalid
+area: CRUD
+type: bug
+issues: []

docs/changelog/114888.yaml

@@ -0,0 +1,6 @@
+pr: 114888
+summary: Fix ST_CENTROID_AGG when no records are aggregated
+area: ES|QL
+type: bug
+issues:
+ - 106025

docs/changelog/114924.yaml

@@ -0,0 +1,5 @@
+pr: 114924
+summary: Reducing error-level stack trace logging for normal events in `GeoIpDownloader`
+area: Ingest Node
+type: bug
+issues: []

docs/plugins/mapper-annotated-text.asciidoc

@@ -167,8 +167,16 @@ duplicates removed. So:
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "text": {
         "type": "annotated_text",
@@ -215,8 +223,16 @@ are preserved.
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "text": { "type": "annotated_text", "store": true }
     }

docs/reference/cat/indices.asciidoc

@@ -6,8 +6,8 @@
 
 [IMPORTANT]
 ====
-cat APIs are only intended for human consumption using the command line or {kib} 
-console. They are _not_ intended for use by applications. For application 
+cat APIs are only intended for human consumption using the command line or {kib}
+console. They are _not_ intended for use by applications. For application
 consumption, use the <<indices-get-index,get index API>>.
 ====
 
@@ -50,6 +50,10 @@ indexing and search. As a result, all document counts include hidden
 To get an accurate count of {es} documents, use the <<cat-count,cat count>> or
 <<search-count,count>> APIs.
 
+Note that information such as document count, deleted document count and store size are not shown for
+indices restored from <<snapshots-source-only-repository,source-only snapshots>> since these indices
+do not contain the relevant data structures to retrieve this information from.
+
 
 [[cat-indices-api-path-params]]
 ==== {api-path-parms-title}

docs/reference/how-to/size-your-shards.asciidoc

@@ -208,6 +208,7 @@ index can be <<indices-delete-index,removed>>. You may then consider setting
 <<indices-add-alias,Create Alias>> against the destination index for the source 
 index's name to point to it for continuity. 
 
+See this https://www.youtube.com/watch?v=sHyNYnwbYro[fixing shard sizes video] for an example troubleshooting walkthrough.
 
 [discrete]
 [[shard-count-recommendation]]
@@ -571,6 +572,8 @@ PUT _cluster/settings
 }
 ----
 
+For more information, see <<troubleshooting-shards-capacity-issues,Troubleshooting shards capacity>>.
+
 [discrete]
 [[troubleshooting-max-docs-limit]]
 ==== Number of documents in the shard cannot exceed [2147483519]

docs/reference/ilm/error-handling.asciidoc

@@ -8,6 +8,9 @@ When this happens, {ilm-init} moves the index to an `ERROR` step.
 If {ilm-init} cannot resolve the error automatically, execution is halted  
 until you resolve the underlying issues with the policy, index, or cluster.
 
+See this https://www.youtube.com/watch?v=VCIqkji3IwY[{ilm-init} health video] 
+for example troubleshooting walkthrough.
+
 For example, you might have a `shrink-index` policy that shrinks an index to four shards once it
 is at least five days old: 
 
@@ -183,6 +186,8 @@ The rollover action then manages setting and updating the alias to
 
 Do not explicitly configure this same alias in the aliases section of an index template.
 
+See this https://www.youtube.com/watch?v=Ww5POq4zZtY[resolving `duplicate alias` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== index.lifecycle.rollover_alias [x] does not point to index [y]
 
@@ -191,20 +196,26 @@ Either the index is using the wrong alias or the alias does not exist.
 Check the `index.lifecycle.rollover_alias` <<indices-get-settings, index setting>>.
 To see what aliases are configured, use <<cat-alias, _cat/aliases>>.
 
+See this https://www.youtube.com/watch?v=NKSe67x7aw8[resolving `not point to index` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== Setting [index.lifecycle.rollover_alias] for index [y] is empty or not defined
 
 The `index.lifecycle.rollover_alias` setting must be configured for the rollover action to work.
 
 Update the index settings to set `index.lifecycle.rollover_alias`.
 
+See this https://www.youtube.com/watch?v=LRpMC2GS_FQ[resolving `empty or not defined` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== Alias [x] has more than one write index [y,z]
 
 Only one index can be designated as the write index for a particular alias.
 
 Use the <<indices-aliases, aliases>> API to set `is_write_index:false` for all but one index.
 
+See this https://www.youtube.com/watch?v=jCUvZCT5Hm4[resolving `more than one write index` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== index name [x] does not match pattern ^.*-\d+
 
@@ -214,6 +225,8 @@ For example, `my-index` does not match the pattern requirement.
 
 Append a numeric value to the index name, for example `my-index-000001`.
 
+See this https://www.youtube.com/watch?v=9sp1zF6iL00[resolving `does not match pattern` video] for an example troubleshooting walkthrough.
+
 [discrete]
 ==== CircuitBreakingException: [x] data too large, data for [y]
 
@@ -227,8 +240,7 @@ For more information, see <<circuit-breaker-errors>>.
 
 This indicates that the cluster is running out of disk space.
 This can happen when you don't have {ilm} set up to roll over from hot to warm nodes.
-
-Consider adding nodes, upgrading your hardware, or deleting unneeded indices.
+For more information, see <<fix-watermark-errors,Fix watermark errors>>.
 
 [discrete]
 ==== security_exception: action [<action-name>] is unauthorized for user [<user-name>] with roles [<role-name>], this action is granted by the index privileges [manage_follow_index,manage,all]

docs/reference/inference/service-elser.asciidoc

@@ -80,12 +80,13 @@ Must be a power of 2. Max allowed value is 32.
 [[inference-example-elser]]
 ==== ELSER service example
 
-The following example shows how to create an {infer} endpoint called
-`my-elser-model` to perform a `sparse_embedding` task type.
+The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type.
 Refer to the {ml-docs}/ml-nlp-elser.html[ELSER model documentation] for more info.
 
-The request below will automatically download the ELSER model if it isn't
-already downloaded and then deploy the model.
+NOTE: If you want to optimize your ELSER endpoint for ingest, set the number of threads to `1` (`"num_threads": 1`).
+If you want to optimize your ELSER endpoint for search, set the number of threads to greater than `1`.
+
+The request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model.
 
 [source,console]
 ------------------------------------------------------------
@@ -100,7 +101,6 @@ PUT _inference/sparse_embedding/my-elser-model
 ------------------------------------------------------------
 // TEST[skip:TBD]
 
-
 Example response:
 
 [source,console-result]
@@ -130,12 +130,12 @@ If using the Python client, you can set the `timeout` parameter to a higher valu
 [[inference-example-elser-adaptive-allocation]]
 ==== Setting adaptive allocation for the ELSER service
 
-The following example shows how to create an {infer} endpoint called
-`my-elser-model` to perform a `sparse_embedding` task type and configure
-adaptive allocations.
+NOTE: For more information on how to optimize your ELSER endpoints, refer to {ml-docs}/ml-nlp-elser.html#elser-recommendations[the ELSER recommendations] section in the model documentation.
+To learn more about model autoscaling, refer to the {ml-docs}/ml-nlp-auto-scale.html[trained model autoscaling] page.
+
+The following example shows how to create an {infer} endpoint called `my-elser-model` to perform a `sparse_embedding` task type and configure adaptive allocations.
 
-The request below will automatically download the ELSER model if it isn't
-already downloaded and then deploy the model.
+The request below will automatically download the ELSER model if it isn't already downloaded and then deploy the model.
 
 [source,console]
 ------------------------------------------------------------

docs/reference/mapping/fields/synthetic-source.asciidoc

@@ -2,7 +2,7 @@
 ==== Synthetic `_source`
 
 IMPORTANT: Synthetic `_source` is Generally Available only for TSDB indices
-(indices that have `index.mode` set to `time_series`). For other indices
+(indices that have `index.mode` set to `time_series`). For other indices,
 synthetic `_source` is in technical preview. Features in technical preview may
 be changed or removed in a future release. Elastic will work to fix
 any issues, but features in technical preview are not subject to the support SLA
@@ -11,15 +11,19 @@ of official GA features.
 Though very handy to have around, the source field takes up a significant amount
 of space on disk. Instead of storing source documents on disk exactly as you
 send them, Elasticsearch can reconstruct source content on the fly upon retrieval.
-Enable this by setting `mode: synthetic` in `_source`:
+Enable this by using the value `synthetic` for the index setting `index.mapping.source.mode`:
 
 [source,console,id=enable-synthetic-source-example]
 ----
 PUT idx
 {
-  "mappings": {
-    "_source": {
-      "mode": "synthetic"
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
     }
   }
 }
@@ -38,7 +42,7 @@ properties when used with synthetic `_source`.
 <<synthetic-source-fields-native-list, Most field types>> construct synthetic `_source` using existing data, most
 commonly <<doc-values,`doc_values`>> and <<stored-fields, stored fields>>. For these field types, no additional space
 is needed to store the contents of `_source` field. Due to the storage layout of <<doc-values,`doc_values`>>, the
-generated `_source` field undergoes <<synthetic-source-modifications, modifications>> compared to original document.
+generated `_source` field undergoes <<synthetic-source-modifications, modifications>> compared to the original document.
 
 For all other field types, the original value of the field is stored as is, in the same way as the `_source` field in
 non-synthetic mode. In this case there are no modifications and field data in `_source` is the same as in the original
@@ -227,10 +231,16 @@ For instance:
 ----
 PUT idx_keep
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": {
-      "mode": "synthetic"
-    },
     "properties": {
       "path": {
         "type": "object",