Merge branch 'feature/semantic-text' into carlosdelest/semantic-text-…

…copy-to-support-inference

# Conflicts:
#	server/src/test/java/org/elasticsearch/index/mapper/FieldTypeLookupTests.java
#	x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilter.java
#	x-pack/plugin/inference/src/test/java/org/elasticsearch/cluster/metadata/SemanticTextClusterMetadataTests.java
#	x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/action/filter/ShardBulkInferenceActionFilterTests.java
#	x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/10_semantic_text_inference.yml

.ci/bwcVersions

@@ -1,120 +1,34 @@
 BWC_VERSION:
-  - "7.0.0"
   - "7.0.1"
-  - "7.1.0"
   - "7.1.1"
-  - "7.2.0"
   - "7.2.1"
-  - "7.3.0"
-  - "7.3.1"
   - "7.3.2"
-  - "7.4.0"
-  - "7.4.1"
   - "7.4.2"
-  - "7.5.0"
-  - "7.5.1"
   - "7.5.2"
-  - "7.6.0"
-  - "7.6.1"
   - "7.6.2"
-  - "7.7.0"
   - "7.7.1"
-  - "7.8.0"
   - "7.8.1"
-  - "7.9.0"
-  - "7.9.1"
-  - "7.9.2"
   - "7.9.3"
-  - "7.10.0"
-  - "7.10.1"
   - "7.10.2"
-  - "7.11.0"
-  - "7.11.1"
   - "7.11.2"
-  - "7.12.0"
   - "7.12.1"
-  - "7.13.0"
-  - "7.13.1"
-  - "7.13.2"
-  - "7.13.3"
   - "7.13.4"
-  - "7.14.0"
-  - "7.14.1"
   - "7.14.2"
-  - "7.15.0"
-  - "7.15.1"
   - "7.15.2"
-  - "7.16.0"
-  - "7.16.1"
-  - "7.16.2"
   - "7.16.3"
-  - "7.17.0"
-  - "7.17.1"
-  - "7.17.2"
-  - "7.17.3"
-  - "7.17.4"
-  - "7.17.5"
-  - "7.17.6"
-  - "7.17.7"
-  - "7.17.8"
-  - "7.17.9"
-  - "7.17.10"
-  - "7.17.11"
-  - "7.17.12"
-  - "7.17.13"
-  - "7.17.14"
-  - "7.17.15"
-  - "7.17.16"
-  - "7.17.17"
-  - "7.17.18"
-  - "7.17.19"
   - "7.17.20"
-  - "8.0.0"
   - "8.0.1"
-  - "8.1.0"
-  - "8.1.1"
-  - "8.1.2"
   - "8.1.3"
-  - "8.2.0"
-  - "8.2.1"
-  - "8.2.2"
   - "8.2.3"
-  - "8.3.0"
-  - "8.3.1"
-  - "8.3.2"
   - "8.3.3"
-  - "8.4.0"
-  - "8.4.1"
-  - "8.4.2"
   - "8.4.3"
-  - "8.5.0"
-  - "8.5.1"
-  - "8.5.2"
   - "8.5.3"
-  - "8.6.0"
-  - "8.6.1"
   - "8.6.2"
-  - "8.7.0"
   - "8.7.1"
-  - "8.8.0"
-  - "8.8.1"
   - "8.8.2"
-  - "8.9.0"
-  - "8.9.1"
   - "8.9.2"
-  - "8.10.0"
-  - "8.10.1"
-  - "8.10.2"
-  - "8.10.3"
   - "8.10.4"
-  - "8.11.0"
-  - "8.11.1"
-  - "8.11.2"
-  - "8.11.3"
   - "8.11.4"
-  - "8.12.0"
-  - "8.12.1"
   - "8.12.2"
-  - "8.13.0"
   - "8.13.1"
   - "8.14.0"

build.gradle

@@ -75,6 +75,11 @@ class ListExpansion {
   String variable
 }
 
+// Filters out intermediate patch releases to reduce the load of CI testing
+def filterIntermediatePatches = { List<Version> versions ->
+  versions.groupBy {"${it.major}.${it.minor}"}.values().collect {it.max()}
+}
+
 tasks.register("updateCIBwcVersions") {
   def writeVersions = { File file, List<Version> versions ->
     file.text = ""
@@ -119,29 +124,29 @@ tasks.register("updateCIBwcVersions") {
   }
 
   doLast {
-    writeVersions(file(".ci/bwcVersions"), BuildParams.bwcVersions.allIndexCompatible)
-    writeVersions(file(".ci/snapshotBwcVersions"), BuildParams.bwcVersions.unreleasedIndexCompatible)
+    writeVersions(file(".ci/bwcVersions"), filterIntermediatePatches(BuildParams.bwcVersions.allIndexCompatible))
+    writeVersions(file(".ci/snapshotBwcVersions"), filterIntermediatePatches(BuildParams.bwcVersions.unreleasedIndexCompatible))
     expandBwcList(
       ".buildkite/pipelines/intake.yml",
       ".buildkite/pipelines/intake.template.yml",
-      BuildParams.bwcVersions.unreleasedIndexCompatible
+      filterIntermediatePatches(BuildParams.bwcVersions.unreleasedIndexCompatible)
     )
     writeBuildkitePipeline(
       ".buildkite/pipelines/periodic.yml",
       ".buildkite/pipelines/periodic.template.yml",
       [
-        new ListExpansion(versions: BuildParams.bwcVersions.unreleasedIndexCompatible, variable: "BWC_LIST"),
+        new ListExpansion(versions: filterIntermediatePatches(BuildParams.bwcVersions.unreleasedIndexCompatible), variable: "BWC_LIST"),
       ],
       [
-        new StepExpansion(templatePath: ".buildkite/pipelines/periodic.bwc.template.yml", versions: BuildParams.bwcVersions.allIndexCompatible, variable: "BWC_STEPS"),
+        new StepExpansion(templatePath: ".buildkite/pipelines/periodic.bwc.template.yml", versions: filterIntermediatePatches(BuildParams.bwcVersions.allIndexCompatible), variable: "BWC_STEPS"),
       ]
     )
 
     expandBwcSteps(
       ".buildkite/pipelines/periodic-packaging.yml",
       ".buildkite/pipelines/periodic-packaging.template.yml",
       ".buildkite/pipelines/periodic-packaging.bwc.template.yml",
-      BuildParams.bwcVersions.allIndexCompatible
+      filterIntermediatePatches(BuildParams.bwcVersions.allIndexCompatible)
     )
   }
 }
@@ -170,7 +175,7 @@ tasks.register("verifyVersions") {
           .collect { Version.fromString(it) }
       )
     }
-    verifyCiYaml(file(".ci/bwcVersions"), BuildParams.bwcVersions.allIndexCompatible)
+    verifyCiYaml(file(".ci/bwcVersions"), filterIntermediatePatches(BuildParams.bwcVersions.allIndexCompatible))
     verifyCiYaml(file(".ci/snapshotBwcVersions"), BuildParams.bwcVersions.unreleasedIndexCompatible)
 
     // Make sure backport bot config file is up to date

docs/changelog/105682.yaml

@@ -0,0 +1,20 @@
+pr: 105682
+summary: Introduce global retention in data stream lifecycle.
+area: Data streams
+type: feature
+issues:
+  - 106169
+highlight:
+  title: Add global retention in data stream lifecycle
+  body: |-
+    Data stream lifecycle now supports configuring retention on a cluster level, namely global retention. Global retention 
+    allows us to configure two different retentions:
+    
+    - `default_retention` is applied to all data streams managed by the data stream lifecycle that do not have retention
+    defined on the data stream level.
+    - `max_retention` is applied to all data streams managed by the data stream lifecycle and it allows any data stream 
+    data to be deleted after the `max_retention` has passed.
+
+    Furthermore, we introduce the term `effective_retention` which is the retention applied at a certain moment to a data
+    stream considering all the available retention configurations.
+  notable: true

docs/changelog/105745.yaml

@@ -0,0 +1,6 @@
+pr: 105745
+summary: Fix `noop_update_total` is not being updated when using the `_bulk`
+area: CRUD
+type: bug
+issues:
+  - 105742

docs/changelog/106377.yaml

@@ -0,0 +1,5 @@
+pr: 106377
+summary: Add transport version for search load autoscaling
+area: Search
+type: enhancement
+issues: []

docs/changelog/106531.yaml

@@ -0,0 +1,5 @@
+pr: 106531
+summary: Get and Query API Key with profile uid
+area: Security
+type: feature
+issues: []

docs/changelog/106636.yaml

@@ -0,0 +1,5 @@
+pr: 106636
+summary: "ESQL: Add OPTIONS clause to FROM command"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/106638.yaml

@@ -0,0 +1,5 @@
+pr: 106638
+summary: Allow users to get status of own async search tasks
+area: Authorization
+type: enhancement
+issues: []

docs/changelog/106793.yaml

@@ -0,0 +1,7 @@
+pr: 106793
+summary: Fail checkpoint on missing clusters
+area: Transform
+type: bug
+issues:
+ - 104533
+ - 106790

docs/changelog/106862.yaml

@@ -0,0 +1,5 @@
+pr: 106862
+summary: Extend support of `allowedFields` to `getMatchingFieldNames` and `getAllFields`
+area: "Mapping"
+type: bug
+issues: []

docs/reference/data-streams/data-stream-apis.asciidoc

@@ -25,7 +25,13 @@ preview:[]
 preview:[]
 * <<data-streams-explain-lifecycle,Explain data stream lifecycle>>
 preview:[]
-* <<data-streams-get-lifecycle-stats, Get data stream lifecycle stats>>
+* <<data-streams-get-lifecycle-stats,Get data stream lifecycle stats>>
+preview:[]
+* <<data-streams-put-global-retention,Update the global retention for data stream lifecycle managed data streams>>
+preview:[]
+* <<data-streams-get-global-retention,Get the global retention for data stream lifecycle managed data streams>>
+preview:[]
+* <<data-streams-delete-global-retention,Delete the global retention for data stream lifecycle managed data streams>>
 preview:[]
 
 The following API is available for <<tsds,time series data streams>>:
@@ -59,4 +65,10 @@ include::{es-repo-dir}/data-streams/lifecycle/apis/explain-lifecycle.asciidoc[]
 
 include::{es-repo-dir}/data-streams/lifecycle/apis/get-lifecycle-stats.asciidoc[]
 
+include::{es-repo-dir}/data-streams/lifecycle/apis/put-global-retention.asciidoc[]
+
+include::{es-repo-dir}/data-streams/lifecycle/apis/get-global-retention.asciidoc[]
+
+include::{es-repo-dir}/data-streams/lifecycle/apis/delete-global-retention.asciidoc[]
+
 include::{es-repo-dir}/indices/downsample-data-stream.asciidoc[]

docs/reference/data-streams/lifecycle/apis/delete-global-retention.asciidoc

@@ -0,0 +1,121 @@
+[[data-streams-delete-global-retention]]
+=== Delete the global retention of data streams
+++++
+<titleabbrev>Delete Data Stream Global Retention</titleabbrev>
+++++
+
+preview::[]
+
+Deletes the global retention configuration that applies on every data stream managed by <<data-stream-lifecycle,data stream lifecycle>>.
+
+[[delete-global-retention-api-prereqs]]
+==== {api-prereq-title}
+
+** If the {es} {security-features} are enabled, you must have the `manage_data_stream_global_retention` <<privileges-list-cluster,cluster privilege>> to use this API.
+
+[[data-streams-delete-global-retention-request]]
+==== {api-request-title}
+
+`DELETE _data_stream/_global_retention`
+
+[[data-streams-delete-global-retention-desc]]
+==== {api-description-title}
+
+Deletes the global retention configuration that is applied on data streams managed by data stream lifecycle.
+
+[role="child_attributes"]
+[[delete-global-retention-api-query-parms]]
+==== {api-query-parms-title}
+
+`dry_run`::
+(Boolean) Signals that the request should determine the effect of the removal of the existing without updating
+the global retention. The default value is `false`, which means the removal will happen.
+
+[[delete-global-retention-api-response-body]]
+==== {api-response-body-title}
+
+`acknowledged`::
+(boolean)
+True, if the global retention has been removed. False, if it fails or if it was a dry run.
+
+`dry_run`::
+(boolean)
+True, if this was a dry run, false otherwise.
+
+`affected_data_streams`::
+(array of objects)
+Contains information about the data streams affected by the change.
++
+.Properties of objects in `affected_data_streams`
+[%collapsible%open]
+====
+`name`::
+(string)
+Name of the data stream.
+`previous_effective_retention`::
+(string)
+The retention that was effective before the change of this request. `infinite` if there was no retention applicable.
+`new_effective_retention`::
+(string)
+The retention that is or would be effective after this request. `infinite` if there is no retention applicable.
+====
+
+[[data-streams-delete-global-retention-example]]
+==== {api-examples-title}
+
+////
+
+[source,console]
+--------------------------------------------------
+PUT _data_stream/_global_retention
+{
+  "default_retention": "7d",
+  "max_retention": "90d"
+}
+
+PUT /_index_template/template
+{
+  "index_patterns": ["my-data-stream*"],
+  "template": {
+    "lifecycle": {}
+  },
+  "data_stream": { }
+}
+
+PUT /_data_stream/my-data-stream
+----
+// TESTSETUP
+////
+
+////
+[source,console]
+----
+DELETE /_data_stream/my-data-stream*
+DELETE /_index_template/template
+DELETE /_data_stream/_global_retention
+----
+// TEARDOWN
+////
+
+Let's update the global retention:
+[source,console]
+--------------------------------------------------
+DELETE _data_stream/_global_retention
+--------------------------------------------------
+
+The response will look like the following:
+
+[source,console-result]
+--------------------------------------------------
+{
+  "acknowledged": true,
+  "dry_run": false,
+  "affected_data_streams": [
+    {
+      "name": "my-data-stream",
+      "previous_effective_retention": "7d",
+      "new_effective_retention": "infinite"
+    }
+  ]
+}
+--------------------------------------------------