Merge remote-tracking branch 'origin/main' into ignore-dynamic-beyond…

…-limit

.buildkite/hooks/pre-command

@@ -69,3 +69,8 @@ if [[ "${USE_DRA_CREDENTIALS:-}" == "true" ]]; then
   DRA_VAULT_ADDR=https://secrets.elastic.co:8200
   export DRA_VAULT_ADDR
 fi
+
+if [[ "${USE_SNYK_CREDENTIALS:-}" == "true" ]]; then
+  SNYK_TOKEN=$(vault read -field=token secret/ci/elastic-elasticsearch/migrated/snyk)
+  export SNYK_TOKEN
+fi

.buildkite/pipelines/ecs-dynamic-template-tests.yml

@@ -0,0 +1,9 @@
+steps:
+  - label: ecs-dynamic-templates-tests
+    command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.configure_test_clusters_with_one_processor=true :x-pack:plugin:stack:javaRestTest
+    timeout_in_minutes: 420
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      diskSizeGb: 350
+      machineType: custom-32-98304

.buildkite/pipelines/intake.yml

@@ -40,7 +40,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.14", "8.9.2", "8.10.0", "8.11.0"]
+            BWC_VERSION: ["7.17.14", "8.9.3", "8.10.0", "8.11.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1584,6 +1584,22 @@ steps:
         env:
           BWC_VERSION: 8.9.2
 
+      - label: "{{matrix.image}} / 8.9.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.9.3
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.9.3
+
       - label: "{{matrix.image}} / 8.10.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh --build-cache -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/ -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.10.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.template.yml

@@ -103,3 +103,13 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       diskSizeGb: 350
       machineType: custom-32-98304
+  - label: Upload Snyk Dependency Graph
+    command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH
+    env:
+      USE_SNYK_CREDENTIALS: "true"
+    timeout_in_minutes: 20
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n2-standard-8
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/periodic.yml

@@ -972,6 +972,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.9.2
+      - label: 8.9.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.9.3#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.9.3
       - label: 8.10.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.10.0#bwcTest
         timeout_in_minutes: 300
@@ -1094,3 +1104,13 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       diskSizeGb: 350
       machineType: custom-32-98304
+  - label: Upload Snyk Dependency Graph
+    command: .ci/scripts/run-gradle.sh uploadSnykDependencyGraph -PsnykTargetReference=$BUILDKITE_BRANCH
+    env:
+      USE_SNYK_CREDENTIALS: "true"
+    timeout_in_minutes: 20
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: n2-standard-8
+      buildDirectory: /dev/shm/bk

.buildkite/scripts/periodic.trigger.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+exit 0
+
 echo "steps:"
 
 source .buildkite/scripts/branches.sh

.ci/bwcVersions

@@ -96,5 +96,6 @@ BWC_VERSION:
   - "8.9.0"
   - "8.9.1"
   - "8.9.2"
+  - "8.9.3"
   - "8.10.0"
   - "8.11.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
   - "7.17.14"
-  - "8.9.2"
+  - "8.9.3"
   - "8.10.0"
   - "8.11.0"

TESTING.asciidoc

@@ -453,6 +453,11 @@ You can run a group of YAML test by using wildcards:
   --tests "org.elasticsearch.test.rest.ClientYamlTestSuiteIT.test {yaml=index/*/*}"
 ---------------------------------------------------------------------------
 
+Note that if the selected test via the `--tests` filter is not a valid test, i.e., the YAML test
+runner is not able to parse and load it, you might get an error message indicating that the test
+was not found. In such cases, running the whole suite without using the `--tests` could show more
+specific error messages about why the test runner is not able to parse or load a certain test.
+
 The YAML REST tests support all the options provided by the randomized runner, plus the following:
 
 * `tests.rest.blacklist`: comma separated globs that identify tests that are

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -44,6 +44,7 @@
 import org.openjdk.jmh.annotations.Warmup;
 
 import java.time.Duration;
+import java.util.Arrays;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -133,9 +134,7 @@ private static FieldAttribute intField() {
 
     private static Layout layout(FieldAttribute... fields) {
         Layout.Builder layout = new Layout.Builder();
-        for (FieldAttribute field : fields) {
-            layout.appendChannel(field.id());
-        }
+        layout.append(Arrays.asList(fields));
         return layout.build();
     }
 

catalog-info.yaml

@@ -98,45 +98,45 @@ spec:
         build_pull_requests: false
         publish_commit_status: false
         trigger_mode: none
----
-# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
-apiVersion: backstage.io/v1alpha1
-kind: Resource
-metadata:
-  name: buildkite-pipeline-elasticsearch-periodic-trigger
-  description: Triggers periodic pipelines for all required branches
-  links:
-    - title: Pipeline
-      url: https://buildkite.com/elastic/elasticsearch-periodic-trigger
-spec:
-  type: buildkite-pipeline
-  system: buildkite
-  owner: group:elasticsearch-team
-  implementation:
-    apiVersion: buildkite.elastic.dev/v1
-    kind: Pipeline
-    metadata:
-      description: ":elasticsearch: Triggers periodic pipelines for all required branches"
-      name: elasticsearch / periodic / trigger
-    spec:
-      repository: elastic/elasticsearch
-      pipeline_file: .buildkite/scripts/periodic.trigger.sh
-      branch_configuration: main
-      teams:
-        elasticsearch-team: {}
-        ml-core: {}
-        everyone:
-          access_level: BUILD_AND_READ
-      provider_settings:
-        build_branches: false
-        build_pull_requests: false
-        publish_commit_status: false
-        trigger_mode: none
-      schedules:
-        Periodically on main:
-          branch: main
-          cronline: "0 0,8,16 * * * America/New_York"
-          message: "Triggers pipelines 3x daily"
+# ---
+# # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+# apiVersion: backstage.io/v1alpha1
+# kind: Resource
+# metadata:
+#   name: buildkite-pipeline-elasticsearch-periodic-trigger
+#   description: Triggers periodic pipelines for all required branches
+#   links:
+#     - title: Pipeline
+#       url: https://buildkite.com/elastic/elasticsearch-periodic-trigger
+# spec:
+#   type: buildkite-pipeline
+#   system: buildkite
+#   owner: group:elasticsearch-team
+#   implementation:
+#     apiVersion: buildkite.elastic.dev/v1
+#     kind: Pipeline
+#     metadata:
+#       description: ":elasticsearch: Triggers periodic pipelines for all required branches"
+#       name: elasticsearch / periodic / trigger
+#     spec:
+#       repository: elastic/elasticsearch
+#       pipeline_file: .buildkite/scripts/periodic.trigger.sh
+#       branch_configuration: main
+#       teams:
+#         elasticsearch-team: {}
+#         ml-core: {}
+#         everyone:
+#           access_level: BUILD_AND_READ
+#       provider_settings:
+#         build_branches: false
+#         build_pull_requests: false
+#         publish_commit_status: false
+#         trigger_mode: none
+#       schedules:
+#         Periodically on main:
+#           branch: main
+#           cronline: "0 0,8,16 * * * America/New_York"
+#           message: "Triggers pipelines 3x daily"
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1
@@ -269,3 +269,37 @@ spec:
           branch: lucene_snapshot
           cronline: "0 9,12,15,18 * * * America/New_York"
           message: "Runs tests against lucene_snapshot branch several times per day"
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elasticsearch-ecs-dynamic-template-tests
+  description: Runs ECS dynamic template tests against main branch
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elasticsearch-ecs-dynamic-template-tests
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: group:elasticsearch-team
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      description: ":elasticsearch: ECS dynamic template tests against main branch"
+      name: elasticsearch / ecs-dynamic-templates / tests
+    spec:
+      repository: elastic/elasticsearch
+      pipeline_file: .buildkite/pipelines/ecs-dynamic-template-tests.yml
+      provider_settings:
+        trigger_mode: none
+      teams:
+        elasticsearch-team: {}
+        ml-core: {}
+        everyone:
+          access_level: READ_ONLY
+      schedules:
+        Daily:
+          branch: main
+          cronline: "0 12 * * * America/New_York"

docs/changelog/98518.yaml

@@ -0,0 +1,6 @@
+pr: 98518
+summary: Add `index.look_back_time` setting for tsdb data streams
+area: TSDB
+type: enhancement
+issues:
+ - 98463

docs/changelog/98915.yaml

@@ -0,0 +1,5 @@
+pr: 98915
+summary: Avoid risk of OOM in datafeeds when memory is constrained
+area: Machine Learning
+type: bug
+issues: [89769]

docs/changelog/98974.yaml

@@ -0,0 +1,5 @@
+pr: 98974
+summary: "ESQL: RIGHT function"
+area: ES|QL
+type: feature
+issues: []

docs/changelog/99231.yaml

@@ -0,0 +1,5 @@
+pr: 99231
+summary: Add manage permission for fleet managed threat intel indices
+area: Authorization
+type: enhancement
+issues: []

docs/changelog/99281.yaml

@@ -0,0 +1,5 @@
+pr: 99281
+summary: Fix PIT when resolving with deleted indices
+area: Search
+type: bug
+issues: []

docs/changelog/99286.yaml

@@ -0,0 +1,5 @@
+pr: 99286
+summary: "ESQL: Log execution time consistently"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/99300.yaml

@@ -0,0 +1,5 @@
+pr: 99300
+summary: Change `GetFromTranslog` to indices action
+area: CRUD
+type: bug
+issues: []

docs/changelog/99310.yaml

@@ -0,0 +1,6 @@
+pr: 99310
+summary: "ESQL: \"params\" correctly parses the values including an optional \"type\""
+area: ES|QL
+type: bug
+issues:
+ - 99294

docs/reference/data-streams/set-up-tsds.asciidoc

@@ -177,6 +177,7 @@ Optionally, the index settings component template for a TSDS can include:
 
 * Your lifecycle policy in the `index.lifecycle.name` index setting.
 * The <<tsds-look-ahead-time,`index.look_ahead_time`>> index setting.
+* The <<tsds-look-back-time,`index.look_back_time`>> index setting.
 * Other index settings, such as <<index-codec,`index.codec`>>, for your TSDS's
 backing indices.
 

docs/reference/data-streams/tsds-index-settings.asciidoc

@@ -33,6 +33,15 @@ days). Only indices with an `index.mode` of `time_series` support this setting.
 For more information, refer to <<tsds-look-ahead-time>>. Additionally this setting
 can not be less than `time_series.poll_interval` cluster setting.
 
+[[index-look-back-time]]
+`index.look_back_time`::
+(<<_static_index_settings,Static>>, <<time-units,time units>>)
+Interval used to calculate the `index.time_series.start_time` for a TSDS's first
+backing index when a tsdb data stream is created. Defaults to `2h` (2 hours).
+Accepts `1m` (one minute) to `7d` (seven days). Only indices with an `index.mode`
+of `time_series` support this setting. For more information,
+refer to <<tsds-look-back-time>>.
+
 [[index-routing-path]] `index.routing_path`::
 (<<_static_index_settings,Static>>, string or array of strings) Plain `keyword`
 fields used to route documents in a TSDS to index shards. Supports wildcards

docs/reference/data-streams/tsds.asciidoc

@@ -253,6 +253,22 @@ value borders the `index.time_series.start_time` for the new write index. This
 ensures the `@timestamp` ranges for neighboring backing indices always border
 but never overlap.
 
+[discrete]
+[[tsds-look-back-time]]
+==== Look-back time
+
+Use the <<index-look-back-time,`index.look_back_time`>> index setting to
+configure how far in the past you can add documents to an index. When you
+create a data stream for a TSDS, {es} calculates the index's
+`index.time_series.start_time` value as:
+
+`now - index.look_back_time`
+
+This setting is only used when a data stream gets created and controls
+the `index.time_series.start_time` index setting of the first backing index.
+Configuring this index setting can be useful to accept documents with `@timestamp`
+field values that are older than 2 hours (the `index.look_back_time` default).
+
 [discrete]
 [[tsds-accepted-time-range]]
 ==== Accepted time range for adding data