Merge branch 'main' into blockFactory

.buildkite/hooks/pre-command

@@ -74,3 +74,8 @@ if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then
   SNYK_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/migrated/snyk)
   export SNYK_TOKEN
 fi
+
+if [[ "$BUILDKITE_AGENT_META_DATA_PROVIDER" != *"k8s"* ]]; then
+  # Run in the background, while the job continues
+  nohup .buildkite/scripts/setup-monitoring.sh </dev/null >/dev/null 2>&1 &
+fi

.buildkite/scripts/setup-monitoring.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -euo pipefail
+
+ELASTIC_AGENT_URL=$(vault read -field=url secret/ci/elastic-elasticsearch/elastic-agent-token)
+ELASTIC_AGENT_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/elastic-agent-token)
+
+if [[ ! -d /opt/elastic-agent ]]; then
+  sudo mkdir /opt/elastic-agent
+  sudo chown -R buildkite-agent:buildkite-agent /opt/elastic-agent
+  cd /opt/elastic-agent
+
+  archive=elastic-agent-8.10.1-linux-x86_64.tar.gz
+  if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then
+    archive=elastic-agent-8.10.1-linux-arm64.tar.gz
+  fi
+
+  curl -L -O "https://artifacts.elastic.co/downloads/beats/elastic-agent/$archive"
+
+  tar xzf "$archive" --directory=. --strip-components=1
+fi
+
+cd /opt/elastic-agent
+sudo ./elastic-agent install -f --url="$ELASTIC_AGENT_URL" --enrollment-token="$ELASTIC_AGENT_TOKEN"

benchmarks/src/main/java/org/elasticsearch/benchmark/routing/allocation/AllocationBenchmark.java

@@ -152,7 +152,7 @@ public void setUp() throws Exception {
             .metadata(metadata)
             .routingTable(routingTable)
             .nodes(nb)
-            .compatibilityVersions(compatibilityVersions)
+            .nodeIdsToCompatibilityVersions(compatibilityVersions)
             .build();
     }
 

docs/changelog/99432.yaml

@@ -0,0 +1,5 @@
+pr: 99432
+summary: "ESQL: Enable arithmetics for durations and periods"
+area: ES|QL
+type: enhancement
+issues: [99293]

docs/changelog/99474.yaml

@@ -0,0 +1,5 @@
+pr: 99474
+summary: Add `java.net.NetPermission` to APM module's permissions
+area: Infra/Core
+type: bug
+issues: []

docs/changelog/99555.yaml

@@ -0,0 +1,5 @@
+pr: 99555
+summary: Use mappings version to retrieve system index mappings at creation time
+area: Infra/Core
+type: enhancement
+issues: []

docs/changelog/99588.yaml

@@ -0,0 +1,6 @@
+pr: 99588
+summary: Make ESQL more resilient to non-indexed fields
+area: ES|QL
+type: bug
+issues:
+ - 99506

docs/changelog/99627.yaml

@@ -0,0 +1,5 @@
+pr: 99627
+summary: Fix thread context in `getRepositoryData`
+area: Snapshot/Restore
+type: bug
+issues: []

docs/changelog/99641.yaml

@@ -0,0 +1,5 @@
+pr: 99641
+summary: Chunk the cluster allocation explain response
+area: Network
+type: enhancement
+issues: [97803]

docs/changelog/99655.yaml

@@ -0,0 +1,5 @@
+pr: 99655
+summary: "[Profiling] Allow to wait until resources created"
+area: Application
+type: enhancement
+issues: []

docs/changelog/99660.yaml

@@ -0,0 +1,5 @@
+pr: 99660
+summary: Close expired search contexts on SEARCH thread
+area: Search
+type: bug
+issues: []

docs/changelog/99677.yaml

@@ -0,0 +1,5 @@
+pr: 99677
+summary: Using 1 MB chunks for elser model storage
+area: Machine Learning
+type: bug
+issues: [ ]

docs/reference/docs/reindex.asciidoc

@@ -1142,6 +1142,7 @@ You cannot specify both `reindex.ssl.certificate_authorities` and
 
 `reindex.ssl.truststore.password`::
 The password to the truststore (`reindex.ssl.truststore.path`).
+deprecated:[7.17.0] Prefer `reindex.ssl.truststore.secure_password` instead.
 This setting cannot be used with `reindex.ssl.truststore.secure_password`.
 
 `reindex.ssl.truststore.secure_password` (<<secure-settings,Secure>>)::
@@ -1175,6 +1176,7 @@ You cannot specify both `reindex.ssl.key` and `reindex.ssl.keystore.path`.
 `reindex.ssl.key_passphrase`::
 Specifies the passphrase to decrypt the PEM encoded private key
 (`reindex.ssl.key`) if it is encrypted.
+deprecated:[7.17.0] Prefer `reindex.ssl.secure_key_passphrase` instead.
 Cannot be used with `reindex.ssl.secure_key_passphrase`.
 
 `reindex.ssl.secure_key_passphrase` (<<secure-settings,Secure>>)::
@@ -1194,17 +1196,19 @@ If the keystore path ends in ".p12", ".pfx" or "pkcs12", this setting defaults
 to `PKCS12`. Otherwise, it defaults to `jks`.
 
 `reindex.ssl.keystore.password`::
-The password to the keystore (`reindex.ssl.keystore.path`). This setting cannot be used
-with `reindex.ssl.keystore.secure_password`.
+The password to the keystore (`reindex.ssl.keystore.path`).
+deprecated:[7.17.0] Prefer `reindex.ssl.keystore.secure_password` instead.
+This setting cannot be used with `reindex.ssl.keystore.secure_password`.
 
 `reindex.ssl.keystore.secure_password` (<<secure-settings,Secure>>)::
 The password to the keystore (`reindex.ssl.keystore.path`).
 This setting cannot be used with `reindex.ssl.keystore.password`.
 
 `reindex.ssl.keystore.key_password`::
 The password for the key in the keystore (`reindex.ssl.keystore.path`).
-Defaults to the keystore password. This setting cannot be used with
-`reindex.ssl.keystore.secure_key_password`.
+Defaults to the keystore password.
+deprecated:[7.17.0] Prefer `reindex.ssl.keystore.secure_key_password` instead.
+This setting cannot be used with `reindex.ssl.keystore.secure_key_password`.
 
 `reindex.ssl.keystore.secure_key_password` (<<secure-settings,Secure>>)::
 The password for the key in the keystore (`reindex.ssl.keystore.path`).

docs/reference/esql/functions/date_format.asciidoc

@@ -7,5 +7,5 @@ is specified, the `yyyy-MM-dd'T'HH:mm:ss.SSSZ` format is used.
 ----
 FROM employees
 | KEEP first_name, last_name, hire_date
-| EVAL hired = DATE_FORMAT(hire_date, "YYYY-MM-dd")
+| EVAL hired = DATE_FORMAT("YYYY-MM-dd", hire_date)
 ----

docs/reference/esql/functions/types/date_extract.asciidoc

@@ -1,5 +1,5 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 arg1 | arg2 | result
-datetime | keyword | long
+keyword | datetime | long
 |===

docs/reference/ingest/processors/reroute.asciidoc

@@ -4,8 +4,6 @@
 <titleabbrev>Reroute</titleabbrev>
 ++++
 
-experimental::[]
-
 The `reroute` processor allows to route a document to another target index or data stream.
 It has two main modes:
 

docs/reference/settings/common-defs.asciidoc

@@ -57,20 +57,21 @@ end::ssl-key-pem[]
 
 tag::ssl-key-passphrase[]
 The passphrase that is used to decrypt the private key. Since the key might not
-be encrypted, this value is optional.
+be encrypted, this value is optional. deprecated:[7.17.0] Prefer `ssl.secure_key_passphrase` instead.
 +
 You cannot use this setting and `ssl.secure_key_passphrase` at the same time.
 end::ssl-key-passphrase[]
 
 tag::ssl-keystore-key-password[]
 The password for the key in the keystore. The default is the keystore password.
+deprecated:[7.17.0] Prefer `ssl.keystore.secure_key_password` instead.
 +
 You cannot use this setting and `ssl.keystore.secure_password` at the same time.
 //TBD: You cannot use this setting and `ssl.keystore.secure_key_password` at the same time.
 end::ssl-keystore-key-password[]
 
 tag::ssl-keystore-password[]
-The password for the keystore.
+The password for the keystore. deprecated:[7.17.0] Prefer `ssl.keystore.secure_password` instead.
 //TBD: You cannot use this setting and `ssl.keystore.secure_password` at the same time.
 end::ssl-keystore-password[]
 
@@ -122,7 +123,7 @@ or `SSLv3`. See <<fips-140-compliance>>.
 end::ssl-supported-protocols[]
 
 tag::ssl-truststore-password[]
-The password for the truststore.
+The password for the truststore. deprecated:[7.17.0] Prefer `ssl.truststore.secure_password` instead.
 +
 You cannot use this setting and `ssl.truststore.secure_password` at the same
 time.
@@ -160,7 +161,7 @@ Authority (CA); has a `hostname` or IP address that matches the names within
 the certificate.
 
 `certificate`::
-Validates the provided certificate and verifies that it's signed by a 
+Validates the provided certificate and verifies that it's signed by a
 trusted authority (CA), but doesn't check the certificate `hostname`.
 
 `none`::
@@ -173,4 +174,4 @@ resolve TLS errors.
 =====
 +
 Defaults to `full`.
-end::ssl-verification-mode-values[]
+end::ssl-verification-mode-values[]

docs/reference/settings/monitoring-settings.asciidoc

@@ -49,7 +49,7 @@ and {ls} is ignored.
 [[xpack-monitoring-collection-interval]]
 // tag::monitoring-collection-interval-tag[]
 `xpack.monitoring.collection.interval` {ess-icon}::
-deprecated:[6.3.0,"Use `xpack.monitoring.collection.enabled` set to `false` instead."] 
+deprecated:[6.3.0,"Use `xpack.monitoring.collection.enabled` set to `false` instead."]
 (<<cluster-update-settings,Dynamic>>) Setting to `-1` to disable data collection
 is no longer supported beginning with 7.0.0.
 +
@@ -60,7 +60,7 @@ option in `kibana.yml` to the same value.
 
 `xpack.monitoring.elasticsearch.collection.enabled`::
 (<<cluster-update-settings,Dynamic>>)  deprecated:[7.16.0] Controls whether statistics about your
-{es} cluster should be collected. Defaults to `true`. This is different from 
+{es} cluster should be collected. Defaults to `true`. This is different from
 `xpack.monitoring.collection.enabled`, which allows you to enable or disable all
 monitoring collection. However, this setting simply disables the collection of
 {es} data while still allowing other data (e.g., {kib}, {ls}, Beats, or APM
@@ -285,18 +285,18 @@ For example: `["elasticsearch_version_mismatch","xpack_license_expiration"]`.
 You can configure the following TLS/SSL settings.
 
 +{ssl-prefix}.ssl.supported_protocols+::
-(<<static-cluster-setting,Static>>)  deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>)  deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-supported-protocols]
 
 ifdef::verifies[]
 +{ssl-prefix}.ssl.verification_mode+::
-(<<static-cluster-setting,Static>>)  deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>)  deprecated:[7.16.0]
 Controls the verification of certificates.
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-verification-mode-values]
 endif::verifies[]
 
 +{ssl-prefix}.ssl.cipher_suites+::
-(<<static-cluster-setting,Static>>)  deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>)  deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-cipher-suites-values]
 
 [#{ssl-context}-tls-ssl-key-trusted-certificate-settings]
@@ -318,19 +318,19 @@ When using PEM encoded files, use the following settings:
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-pem]
 
 +{ssl-prefix}.ssl.key_passphrase+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-key-passphrase]
 
 +{ssl-prefix}.ssl.secure_key_passphrase+::
-(<<secure-settings,Secure>>) deprecated:[7.16.0] 
+(<<secure-settings,Secure>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-secure-key-passphrase]
 
 +{ssl-prefix}.ssl.certificate+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate]
 
 +{ssl-prefix}.ssl.certificate_authorities+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-certificate-authorities]
 
 ===== Java keystore files
@@ -339,35 +339,35 @@ When using Java keystore files (JKS), which contain the private key, certificate
 and certificates that should be trusted, use the following settings:
 
 +{ssl-prefix}.ssl.keystore.path+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path]
 
 +{ssl-prefix}.ssl.keystore.password+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>)
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password]
 
 +{ssl-prefix}.ssl.keystore.secure_password+::
-(<<secure-settings,Secure>>) deprecated:[7.16.0] 
+(<<secure-settings,Secure>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password]
 
 +{ssl-prefix}.ssl.keystore.key_password+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>)
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password]
 
 +{ssl-prefix}.ssl.keystore.secure_key_password+::
-(<<secure-settings,Secure>>) deprecated:[7.16.0] 
+(<<secure-settings,Secure>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password]
 
 +{ssl-prefix}.ssl.truststore.path+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path]
 
 +{ssl-prefix}.ssl.truststore.password+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>)
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password]
 
 +{ssl-prefix}.ssl.truststore.secure_password+::
-(<<secure-settings,Secure>>) deprecated:[7.16.0] 
+(<<secure-settings,Secure>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password]
 
 [#{ssl-context}-pkcs12-files]
@@ -379,43 +379,43 @@ that contain the private key, certificate and certificates that should be truste
 PKCS#12 files are configured in the same way as Java keystore files:
 
 +{ssl-prefix}.ssl.keystore.path+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path]
 
 +{ssl-prefix}.ssl.keystore.type+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12]
 
 +{ssl-prefix}.ssl.keystore.password+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password]
 
 +{ssl-prefix}.ssl.keystore.secure_password+::
-(<<secure-settings,Secure>>) deprecated:[7.16.0] 
+(<<secure-settings,Secure>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password]
 
 +{ssl-prefix}.ssl.keystore.key_password+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>)
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-key-password]
 
 +{ssl-prefix}.ssl.keystore.secure_key_password+::
-(<<secure-settings,Secure>>) deprecated:[7.16.0] 
+(<<secure-settings,Secure>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-key-password]
 
 +{ssl-prefix}.ssl.truststore.path+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-path]
 
 +{ssl-prefix}.ssl.truststore.type+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>) deprecated:[7.16.0]
 Set this to `PKCS12` to indicate that the truststore is a PKCS#12 file.
 //TBD:Should this use the ssl-truststore-type definition and default values?
 
 +{ssl-prefix}.ssl.truststore.password+::
-(<<static-cluster-setting,Static>>) deprecated:[7.16.0] 
+(<<static-cluster-setting,Static>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-password]
 
 +{ssl-prefix}.ssl.truststore.secure_password+::
-(<<secure-settings,Secure>>) deprecated:[7.16.0] 
+(<<secure-settings,Secure>>) deprecated:[7.16.0]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-truststore-secure-password]
 

docs/reference/settings/security-settings.asciidoc

@@ -2414,6 +2414,12 @@ include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-path]
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-type-pkcs12]
 // end::jwt-ssl-keystore-type-tag[]
 
+// tag::jwt-ssl-keystore-password-tag[]
+`ssl.keystore.password` {ess-icon}::
+(<<static-cluster-setting,Static>>)
+include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-password]
+// end::jwt-ssl-keystore-password-tag[]
+
 `ssl.keystore.secure_password`::
 (<<secure-settings,Secure>>)
 include::{es-repo-dir}/settings/common-defs.asciidoc[tag=ssl-keystore-secure-password]