From dcd7fb7d694251772c7b635d2227bc174f2248d3 Mon Sep 17 00:00:00 2001
From: "Joey F. Poon" <joey@kaijudev.com>
Date: Wed, 20 Nov 2024 22:50:53 +0900
Subject: [PATCH 01/14] Add `.security-workflow-insights` perms for
 kibana_system (#116485)

Adds auto_configure, read, write, and create_index permissions for the
new .security-workflows-insights-* index. This index is created and
used internally by Kibana.
---
 .../KibanaOwnedReservedRoleDescriptors.java   | 16 ++++++++++++++
 .../authz/store/ReservedRolesStoreTests.java  | 22 +++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java
index 259e66f633bac..cc589b53eaa1a 100644
--- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java
+++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java
@@ -484,6 +484,22 @@ static RoleDescriptor kibanaSystem(String name) {
                 // Endpoint heartbeat. Kibana reads from these to determine metering/billing for
                 // endpoints.
                 RoleDescriptor.IndicesPrivileges.builder().indices(".logs-endpoint.heartbeat-*").privileges("read", "create_index").build(),
+                // Security Solution workflows insights. Kibana creates, manages, and uses these
+                // to provide users with insights on potential configuration improvements
+                RoleDescriptor.IndicesPrivileges.builder()
+                    .indices(".edr-workflow-insights-*")
+                    .privileges(
+                        "create_index",
+                        "auto_configure",
+                        "manage",
+                        "read",
+                        "write",
+                        "delete",
+                        TransportUpdateSettingsAction.TYPE.name(),
+                        TransportPutMappingAction.TYPE.name(),
+                        RolloverAction.NAME
+                    )
+                    .build(),
                 // For connectors telemetry. Will be removed once we switched to connectors API
                 RoleDescriptor.IndicesPrivileges.builder().indices(".elastic-connectors*").privileges("read").build() },
             null,
diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
index 9818a890d465f..17579fd6368ce 100644
--- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
+++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java
@@ -1106,6 +1106,28 @@ public void testKibanaSystemRole() {
             assertThat(kibanaRole.indices().allowedIndicesMatcher(RolloverAction.NAME).test(indexAbstraction), is(true));
         });
 
+        // index for Security Solution workflow insights
+        Arrays.asList(".edr-workflow-insights-" + randomAlphaOfLength(randomIntBetween(0, 13))).forEach((index) -> {
+            final IndexAbstraction indexAbstraction = mockIndexAbstraction(index);
+            assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:foo").test(indexAbstraction), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher("indices:bar").test(indexAbstraction), is(false));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportDeleteIndexAction.TYPE.name()).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(GetIndexAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportCreateIndexAction.TYPE.name()).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportIndexAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportDeleteAction.NAME).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportSearchAction.TYPE.name()).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportMultiSearchAction.TYPE.name()).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportGetAction.TYPE.name()).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(READ_CROSS_CLUSTER_NAME).test(indexAbstraction), is(false));
+            assertThat(
+                kibanaRole.indices().allowedIndicesMatcher(TransportUpdateSettingsAction.TYPE.name()).test(indexAbstraction),
+                is(true)
+            );
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(TransportPutMappingAction.TYPE.name()).test(indexAbstraction), is(true));
+            assertThat(kibanaRole.indices().allowedIndicesMatcher(RolloverAction.NAME).test(indexAbstraction), is(true));
+        });
+
         // Data telemetry reads mappings, metadata and stats of indices
         Arrays.asList(randomAlphaOfLengthBetween(8, 24), "packetbeat-*").forEach((index) -> {
             logger.info("index name [{}]", index);

From 0e641793cbd228d6fffac03b2d6b3367c7c99a88 Mon Sep 17 00:00:00 2001
From: Pat Whelan <pat.whelan@elastic.co>
Date: Wed, 20 Nov 2024 09:29:47 -0500
Subject: [PATCH 02/14] [ML] Randomly generate uuids (#116662)

randomAlpha is generating UTF16 strings, which return as UTF8 and fail
to match.  Generating random UTF8 strings can generate control
characters, which the REST handler throws an error on.  So we're
generating UUIDs.

Fix #113430
---
 muted-tests.yml                                               | 3 ---
 .../org/elasticsearch/xpack/inference/InferenceCrudIT.java    | 4 ++--
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index 463715c5afd68..cc568144698ee 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -112,9 +112,6 @@ tests:
 - class: org.elasticsearch.xpack.shutdown.NodeShutdownIT
   method: testStalledShardMigrationProperlyDetected
   issue: https://github.com/elastic/elasticsearch/issues/115697
-- class: org.elasticsearch.xpack.inference.InferenceCrudIT
-  method: testSupportedStream
-  issue: https://github.com/elastic/elasticsearch/issues/113430
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_start_stop/Verify start transform reuses destination index}
   issue: https://github.com/elastic/elasticsearch/issues/115808
diff --git a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
index 591db6db8495a..78e064b42bbb2 100644
--- a/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
+++ b/x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceCrudIT.java
@@ -432,7 +432,7 @@ public void testUnsupportedStream() throws Exception {
         assertEquals(TaskType.SPARSE_EMBEDDING.toString(), singleModel.get("task_type"));
 
         try {
-            var events = streamInferOnMockService(modelId, TaskType.SPARSE_EMBEDDING, List.of(randomAlphaOfLength(10)));
+            var events = streamInferOnMockService(modelId, TaskType.SPARSE_EMBEDDING, List.of(randomUUID()));
             assertThat(events.size(), equalTo(2));
             events.forEach(event -> {
                 switch (event.name()) {
@@ -457,7 +457,7 @@ public void testSupportedStream() throws Exception {
         assertEquals(modelId, singleModel.get("inference_id"));
         assertEquals(TaskType.COMPLETION.toString(), singleModel.get("task_type"));
 
-        var input = IntStream.range(1, 2 + randomInt(8)).mapToObj(i -> randomAlphaOfLength(10)).toList();
+        var input = IntStream.range(1, 2 + randomInt(8)).mapToObj(i -> randomUUID()).toList();
         try {
             var events = streamInferOnMockService(modelId, TaskType.COMPLETION, input);
 

From c3f73d0319b312d2e44df89da0fcb32d774e5954 Mon Sep 17 00:00:00 2001
From: Mark Tozzi <mark.tozzi@gmail.com>
Date: Wed, 20 Nov 2024 09:31:01 -0500
Subject: [PATCH 03/14] Esql Enable Date Nanos (#117080)

This enables date nanos support as tech preview. Basic operations, like reading values, binary comparisons, and functions that don't care about type should work, but some functions are not yet supported. Most notably, Bucket is not yet supported, although Date_Trunc is and can be used for grouping. See the docs for the full list of limitations.

relates to #109352
---
 docs/changelog/117080.yaml                    |  5 ++
 docs/reference/esql/esql-limitations.asciidoc |  4 +-
 .../functions/kibana/definition/case.json     | 42 +++++++++
 .../functions/kibana/definition/coalesce.json | 18 ++++
 .../functions/kibana/definition/count.json    |  2 +-
 .../kibana/definition/count_distinct.json     | 66 ++++++++++++++
 .../kibana/definition/date_trunc.json         | 36 ++++++++
 .../functions/kibana/definition/equals.json   | 18 ++++
 .../kibana/definition/greater_than.json       | 18 ++++
 .../definition/greater_than_or_equal.json     | 18 ++++
 .../functions/kibana/definition/greatest.json | 18 ++++
 .../functions/kibana/definition/least.json    | 18 ++++
 .../kibana/definition/less_than.json          | 18 ++++
 .../kibana/definition/less_than_or_equal.json | 18 ++++
 .../functions/kibana/definition/match.json    |  2 +-
 .../esql/functions/kibana/definition/max.json | 12 +++
 .../esql/functions/kibana/definition/min.json | 12 +++
 .../functions/kibana/definition/mv_count.json | 12 +++
 .../kibana/definition/mv_dedupe.json          | 12 +++
 .../functions/kibana/definition/mv_first.json | 12 +++
 .../functions/kibana/definition/mv_last.json  | 12 +++
 .../functions/kibana/definition/mv_max.json   | 12 +++
 .../functions/kibana/definition/mv_min.json   | 12 +++
 .../functions/kibana/definition/mv_slice.json | 24 +++++
 .../functions/kibana/definition/mv_sort.json  | 18 ++++
 .../kibana/definition/not_equals.json         | 18 ++++
 .../functions/kibana/definition/qstr.json     |  2 +-
 .../kibana/definition/to_date_nanos.json      | 87 ++++++++++++++++++-
 .../kibana/definition/to_datetime.json        | 12 +++
 .../functions/kibana/definition/to_long.json  | 12 +++
 .../kibana/definition/to_string.json          | 12 +++
 .../functions/kibana/definition/values.json   | 12 +++
 .../esql/functions/kibana/docs/match.md       | 10 +--
 .../esql/functions/kibana/docs/qstr.md        | 10 +--
 .../esql/functions/types/case.asciidoc        |  2 +
 .../esql/functions/types/coalesce.asciidoc    |  1 +
 .../functions/types/count_distinct.asciidoc   |  4 +
 .../esql/functions/types/date_trunc.asciidoc  |  2 +
 .../esql/functions/types/equals.asciidoc      |  1 +
 .../functions/types/greater_than.asciidoc     |  1 +
 .../types/greater_than_or_equal.asciidoc      |  1 +
 .../esql/functions/types/greatest.asciidoc    |  1 +
 .../esql/functions/types/least.asciidoc       |  1 +
 .../esql/functions/types/less_than.asciidoc   |  1 +
 .../types/less_than_or_equal.asciidoc         |  1 +
 .../esql/functions/types/max.asciidoc         |  1 +
 .../esql/functions/types/min.asciidoc         |  1 +
 .../esql/functions/types/mv_count.asciidoc    |  1 +
 .../esql/functions/types/mv_dedupe.asciidoc   |  1 +
 .../esql/functions/types/mv_first.asciidoc    |  1 +
 .../esql/functions/types/mv_last.asciidoc     |  1 +
 .../esql/functions/types/mv_max.asciidoc      |  1 +
 .../esql/functions/types/mv_min.asciidoc      |  1 +
 .../esql/functions/types/mv_slice.asciidoc    |  1 +
 .../esql/functions/types/mv_sort.asciidoc     |  1 +
 .../esql/functions/types/not_equals.asciidoc  |  1 +
 .../functions/types/to_date_nanos.asciidoc    |  8 +-
 .../esql/functions/types/to_datetime.asciidoc |  1 +
 .../esql/functions/types/to_long.asciidoc     |  1 +
 .../esql/functions/types/to_string.asciidoc   |  1 +
 .../esql/functions/types/values.asciidoc      |  1 +
 .../esql/core/plugin/EsqlCorePlugin.java      |  1 -
 .../xpack/esql/core/type/DataType.java        |  1 -
 .../xpack/esql/action/EsqlCapabilities.java   | 12 +--
 .../function/EsqlFunctionRegistry.java        |  2 +
 .../expression/function/aggregate/Max.java    |  4 +-
 .../expression/function/aggregate/Min.java    |  4 +-
 .../expression/function/aggregate/Values.java |  7 +-
 .../function/scalar/date/DateTrunc.java       |  4 +-
 .../operator/comparison/LessThan.java         |  4 +-
 .../function/MultiRowTestCaseSupplier.java    | 36 ++++++++
 .../aggregate/CountDistinctTests.java         |  1 +
 .../function/aggregate/MaxTests.java          |  9 ++
 .../function/aggregate/MinTests.java          |  9 ++
 .../function/aggregate/ValuesTests.java       |  1 +
 .../AbstractMultivalueFunctionTestCase.java   |  4 -
 .../operator/comparison/LessThanTests.java    |  4 +-
 77 files changed, 716 insertions(+), 40 deletions(-)
 create mode 100644 docs/changelog/117080.yaml

diff --git a/docs/changelog/117080.yaml b/docs/changelog/117080.yaml
new file mode 100644
index 0000000000000..5909f966e0fa2
--- /dev/null
+++ b/docs/changelog/117080.yaml
@@ -0,0 +1,5 @@
+pr: 117080
+summary: Esql Enable Date Nanos (tech preview)
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/reference/esql/esql-limitations.asciidoc b/docs/reference/esql/esql-limitations.asciidoc
index 1772e956bd9e2..c2849e4889f98 100644
--- a/docs/reference/esql/esql-limitations.asciidoc
+++ b/docs/reference/esql/esql-limitations.asciidoc
@@ -25,6 +25,9 @@ include::processing-commands/limit.asciidoc[tag=limitation]
 * `alias`
 * `boolean`
 * `date`
+* `date_nanos` (Tech Preview)
+** The following functions don't yet support date nanos: `bucket`, `date_format`, `date_parse`, `date_diff`, `date_extract`
+** You can use `to_datetime` to cast to millisecond dates to use unsupported functions
 * `double` (`float`, `half_float`, `scaled_float` are represented as `double`)
 * `ip`
 * `keyword` family including `keyword`, `constant_keyword`, and `wildcard`
@@ -50,7 +53,6 @@ include::processing-commands/limit.asciidoc[tag=limitation]
 ** `position`
 ** `aggregate_metric_double`
 * Date/time
-** `date_nanos`
 ** `date_range`
 * Other types
 ** `binary`
diff --git a/docs/reference/esql/functions/kibana/definition/case.json b/docs/reference/esql/functions/kibana/definition/case.json
index bf498f690551c..51693d9d30660 100644
--- a/docs/reference/esql/functions/kibana/definition/case.json
+++ b/docs/reference/esql/functions/kibana/definition/case.json
@@ -172,6 +172,48 @@
       "variadic" : true,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "condition",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : "A condition."
+        },
+        {
+          "name" : "trueValue",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "The value that's returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "date_nanos"
+    },
+    {
+      "params" : [
+        {
+          "name" : "condition",
+          "type" : "boolean",
+          "optional" : false,
+          "description" : "A condition."
+        },
+        {
+          "name" : "trueValue",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "The value that's returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches."
+        },
+        {
+          "name" : "elseValue",
+          "type" : "date_nanos",
+          "optional" : true,
+          "description" : "The value that's returned when no condition evaluates to `true`."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/coalesce.json b/docs/reference/esql/functions/kibana/definition/coalesce.json
index 7f49195190951..c929323397c9b 100644
--- a/docs/reference/esql/functions/kibana/definition/coalesce.json
+++ b/docs/reference/esql/functions/kibana/definition/coalesce.json
@@ -88,6 +88,24 @@
       "variadic" : true,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "first",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Expression to evaluate."
+        },
+        {
+          "name" : "rest",
+          "type" : "date_nanos",
+          "optional" : true,
+          "description" : "Other expression to evaluate."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/count.json b/docs/reference/esql/functions/kibana/definition/count.json
index 88d4ba3d3e339..329a18c4d9d01 100644
--- a/docs/reference/esql/functions/kibana/definition/count.json
+++ b/docs/reference/esql/functions/kibana/definition/count.json
@@ -151,7 +151,7 @@
   ],
   "examples" : [
     "FROM employees\n| STATS COUNT(height)",
-    "FROM employees \n| STATS count = COUNT(*) BY languages \n| SORT languages DESC",
+    "FROM employees\n| STATS count = COUNT(*) BY languages\n| SORT languages DESC",
     "ROW words=\"foo;bar;baz;qux;quux;foo\"\n| STATS word_count = COUNT(SPLIT(words, \";\"))",
     "ROW n=1\n| WHERE n < 0\n| STATS COUNT(n)",
     "ROW n=1\n| STATS COUNT(n > 0 OR NULL), COUNT(n < 0 OR NULL)"
diff --git a/docs/reference/esql/functions/kibana/definition/count_distinct.json b/docs/reference/esql/functions/kibana/definition/count_distinct.json
index 3addd08df60df..54b99ee84ce2d 100644
--- a/docs/reference/esql/functions/kibana/definition/count_distinct.json
+++ b/docs/reference/esql/functions/kibana/definition/count_distinct.json
@@ -136,6 +136,72 @@
       "variadic" : false,
       "returnType" : "long"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "integer",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Column or literal for which to count the number of distinct values."
+        },
+        {
+          "name" : "precision",
+          "type" : "unsigned_long",
+          "optional" : true,
+          "description" : "Precision threshold. Refer to <<esql-agg-count-distinct-approximate>>. The maximum supported value is 40000. Thresholds above this number will have the same effect as a threshold of 40000. The default value is 3000."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/date_trunc.json b/docs/reference/esql/functions/kibana/definition/date_trunc.json
index 871994407233b..cdda984a0ce7e 100644
--- a/docs/reference/esql/functions/kibana/definition/date_trunc.json
+++ b/docs/reference/esql/functions/kibana/definition/date_trunc.json
@@ -22,6 +22,24 @@
       "variadic" : false,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "interval",
+          "type" : "date_period",
+          "optional" : false,
+          "description" : "Interval; expressed using the timespan literal syntax."
+        },
+        {
+          "name" : "date",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Date expression"
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
@@ -39,6 +57,24 @@
       ],
       "variadic" : false,
       "returnType" : "date"
+    },
+    {
+      "params" : [
+        {
+          "name" : "interval",
+          "type" : "time_duration",
+          "optional" : false,
+          "description" : "Interval; expressed using the timespan literal syntax."
+        },
+        {
+          "name" : "date",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Date expression"
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
     }
   ],
   "examples" : [
diff --git a/docs/reference/esql/functions/kibana/definition/equals.json b/docs/reference/esql/functions/kibana/definition/equals.json
index 59df59eaccc4e..885d949f4b20f 100644
--- a/docs/reference/esql/functions/kibana/definition/equals.json
+++ b/docs/reference/esql/functions/kibana/definition/equals.json
@@ -77,6 +77,24 @@
       "variadic" : false,
       "returnType" : "boolean"
     },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/greater_than.json b/docs/reference/esql/functions/kibana/definition/greater_than.json
index 7354112551e2c..cf6e30a0a4547 100644
--- a/docs/reference/esql/functions/kibana/definition/greater_than.json
+++ b/docs/reference/esql/functions/kibana/definition/greater_than.json
@@ -23,6 +23,24 @@
       "variadic" : false,
       "returnType" : "boolean"
     },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json b/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json
index 832eed417ef4a..2535c68af6acf 100644
--- a/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json
+++ b/docs/reference/esql/functions/kibana/definition/greater_than_or_equal.json
@@ -23,6 +23,24 @@
       "variadic" : false,
       "returnType" : "boolean"
     },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/greatest.json b/docs/reference/esql/functions/kibana/definition/greatest.json
index eebb4fad1eb1d..077100317dfca 100644
--- a/docs/reference/esql/functions/kibana/definition/greatest.json
+++ b/docs/reference/esql/functions/kibana/definition/greatest.json
@@ -53,6 +53,24 @@
       "variadic" : true,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "first",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "First of the columns to evaluate."
+        },
+        {
+          "name" : "rest",
+          "type" : "date_nanos",
+          "optional" : true,
+          "description" : "The rest of the columns to evaluate."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/least.json b/docs/reference/esql/functions/kibana/definition/least.json
index 02fa58f92eaef..18ec65c60f475 100644
--- a/docs/reference/esql/functions/kibana/definition/least.json
+++ b/docs/reference/esql/functions/kibana/definition/least.json
@@ -52,6 +52,24 @@
       "variadic" : true,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "first",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "First of the columns to evaluate."
+        },
+        {
+          "name" : "rest",
+          "type" : "date_nanos",
+          "optional" : true,
+          "description" : "The rest of the columns to evaluate."
+        }
+      ],
+      "variadic" : true,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/less_than.json b/docs/reference/esql/functions/kibana/definition/less_than.json
index 66578d73b8e9c..a73754d200d46 100644
--- a/docs/reference/esql/functions/kibana/definition/less_than.json
+++ b/docs/reference/esql/functions/kibana/definition/less_than.json
@@ -23,6 +23,24 @@
       "variadic" : false,
       "returnType" : "boolean"
     },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json b/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json
index 5ffd4567cdb07..7af477db32a34 100644
--- a/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json
+++ b/docs/reference/esql/functions/kibana/definition/less_than_or_equal.json
@@ -23,6 +23,24 @@
       "variadic" : false,
       "returnType" : "boolean"
     },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/match.json b/docs/reference/esql/functions/kibana/definition/match.json
index 8a355360a790f..4a5b05a3f501b 100644
--- a/docs/reference/esql/functions/kibana/definition/match.json
+++ b/docs/reference/esql/functions/kibana/definition/match.json
@@ -78,7 +78,7 @@
     }
   ],
   "examples" : [
-    "from books \n| where match(author, \"Faulkner\")\n| keep book_no, author \n| sort book_no \n| limit 5;"
+    "FROM books \n| WHERE MATCH(author, \"Faulkner\")\n| KEEP book_no, author \n| SORT book_no \n| LIMIT 5;"
   ],
   "preview" : true,
   "snapshot_only" : false
diff --git a/docs/reference/esql/functions/kibana/definition/max.json b/docs/reference/esql/functions/kibana/definition/max.json
index 45fd26571b091..7f3d2215ee099 100644
--- a/docs/reference/esql/functions/kibana/definition/max.json
+++ b/docs/reference/esql/functions/kibana/definition/max.json
@@ -28,6 +28,18 @@
       "variadic" : false,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/min.json b/docs/reference/esql/functions/kibana/definition/min.json
index ae71fba049dbe..74e3fd8208f1b 100644
--- a/docs/reference/esql/functions/kibana/definition/min.json
+++ b/docs/reference/esql/functions/kibana/definition/min.json
@@ -28,6 +28,18 @@
       "variadic" : false,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/mv_count.json b/docs/reference/esql/functions/kibana/definition/mv_count.json
index 4767b35ec7cac..90ace2525f710 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_count.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_count.json
@@ -52,6 +52,18 @@
       "variadic" : false,
       "returnType" : "integer"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "integer"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/mv_dedupe.json b/docs/reference/esql/functions/kibana/definition/mv_dedupe.json
index bfca58bc3e140..ce2c96dbc1757 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_dedupe.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_dedupe.json
@@ -53,6 +53,18 @@
       "variadic" : false,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/mv_first.json b/docs/reference/esql/functions/kibana/definition/mv_first.json
index a2b6358023e4b..552f568c9b171 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_first.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_first.json
@@ -52,6 +52,18 @@
       "variadic" : false,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/mv_last.json b/docs/reference/esql/functions/kibana/definition/mv_last.json
index b6dc268af5305..78d7b348a6042 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_last.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_last.json
@@ -52,6 +52,18 @@
       "variadic" : false,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/mv_max.json b/docs/reference/esql/functions/kibana/definition/mv_max.json
index 27d2b010dc02c..a1e55c58cff70 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_max.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_max.json
@@ -28,6 +28,18 @@
       "variadic" : false,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/mv_min.json b/docs/reference/esql/functions/kibana/definition/mv_min.json
index 410e97335687f..7998ca4eda94e 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_min.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_min.json
@@ -28,6 +28,18 @@
       "variadic" : false,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Multivalue expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/mv_slice.json b/docs/reference/esql/functions/kibana/definition/mv_slice.json
index dbbfe0ffb5a78..df4d48145fac6 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_slice.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_slice.json
@@ -100,6 +100,30 @@
       "variadic" : false,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Multivalue expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "start",
+          "type" : "integer",
+          "optional" : false,
+          "description" : "Start position. If `null`, the function returns `null`. The start argument can be negative. An index of -1 is used to specify the last value in the list."
+        },
+        {
+          "name" : "end",
+          "type" : "integer",
+          "optional" : true,
+          "description" : "End position(included). Optional; if omitted, the position at `start` is returned. The end argument can be negative. An index of -1 is used to specify the last value in the list."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/mv_sort.json b/docs/reference/esql/functions/kibana/definition/mv_sort.json
index 4cb255fb0afcb..072c05743af33 100644
--- a/docs/reference/esql/functions/kibana/definition/mv_sort.json
+++ b/docs/reference/esql/functions/kibana/definition/mv_sort.json
@@ -40,6 +40,24 @@
       "variadic" : false,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Multivalue expression. If `null`, the function returns `null`."
+        },
+        {
+          "name" : "order",
+          "type" : "keyword",
+          "optional" : true,
+          "description" : "Sort order. The valid options are ASC and DESC, the default is ASC."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/not_equals.json b/docs/reference/esql/functions/kibana/definition/not_equals.json
index 69389d4c8d077..24f31115cbc37 100644
--- a/docs/reference/esql/functions/kibana/definition/not_equals.json
+++ b/docs/reference/esql/functions/kibana/definition/not_equals.json
@@ -77,6 +77,24 @@
       "variadic" : false,
       "returnType" : "boolean"
     },
+    {
+      "params" : [
+        {
+          "name" : "lhs",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "An expression."
+        },
+        {
+          "name" : "rhs",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "An expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "boolean"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/qstr.json b/docs/reference/esql/functions/kibana/definition/qstr.json
index 9823c3cff8923..76473349a3414 100644
--- a/docs/reference/esql/functions/kibana/definition/qstr.json
+++ b/docs/reference/esql/functions/kibana/definition/qstr.json
@@ -30,7 +30,7 @@
     }
   ],
   "examples" : [
-    "from books \n| where qstr(\"author: Faulkner\")\n| keep book_no, author \n| sort book_no \n| limit 5;"
+    "FROM books \n| WHERE QSTR(\"author: Faulkner\")\n| KEEP book_no, author \n| SORT book_no \n| LIMIT 5;"
   ],
   "preview" : true,
   "snapshot_only" : false
diff --git a/docs/reference/esql/functions/kibana/definition/to_date_nanos.json b/docs/reference/esql/functions/kibana/definition/to_date_nanos.json
index 07ffe84444f02..d9409bceb8e6f 100644
--- a/docs/reference/esql/functions/kibana/definition/to_date_nanos.json
+++ b/docs/reference/esql/functions/kibana/definition/to_date_nanos.json
@@ -4,7 +4,92 @@
   "name" : "to_date_nanos",
   "description" : "Converts an input to a nanosecond-resolution date value (aka date_nanos).",
   "note" : "The range for date nanos is 1970-01-01T00:00:00.000000000Z to 2262-04-11T23:47:16.854775807Z.  Additionally, integers cannot be converted into date nanos, as the range of integer nanoseconds only covers about 2 seconds after epoch.",
-  "signatures" : [ ],
+  "signatures" : [
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date",
+          "optional" : false,
+          "description" : "Input value. The input can be a single- or multi-valued column or an expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Input value. The input can be a single- or multi-valued column or an expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "double",
+          "optional" : false,
+          "description" : "Input value. The input can be a single- or multi-valued column or an expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "keyword",
+          "optional" : false,
+          "description" : "Input value. The input can be a single- or multi-valued column or an expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "long",
+          "optional" : false,
+          "description" : "Input value. The input can be a single- or multi-valued column or an expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "text",
+          "optional" : false,
+          "description" : "Input value. The input can be a single- or multi-valued column or an expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "unsigned_long",
+          "optional" : false,
+          "description" : "Input value. The input can be a single- or multi-valued column or an expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    }
+  ],
   "preview" : true,
   "snapshot_only" : false
 }
diff --git a/docs/reference/esql/functions/kibana/definition/to_datetime.json b/docs/reference/esql/functions/kibana/definition/to_datetime.json
index 072aa66aac669..8f9ecbd139d32 100644
--- a/docs/reference/esql/functions/kibana/definition/to_datetime.json
+++ b/docs/reference/esql/functions/kibana/definition/to_datetime.json
@@ -17,6 +17,18 @@
       "variadic" : false,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Input value. The input can be a single- or multi-valued column or an expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/to_long.json b/docs/reference/esql/functions/kibana/definition/to_long.json
index afd6de001bbc6..eb1ce7220c3f9 100644
--- a/docs/reference/esql/functions/kibana/definition/to_long.json
+++ b/docs/reference/esql/functions/kibana/definition/to_long.json
@@ -52,6 +52,18 @@
       "variadic" : false,
       "returnType" : "long"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Input value. The input can be a single- or multi-valued column or an expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "long"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/to_string.json b/docs/reference/esql/functions/kibana/definition/to_string.json
index 33e95d5bed1c2..1c86e81b31136 100644
--- a/docs/reference/esql/functions/kibana/definition/to_string.json
+++ b/docs/reference/esql/functions/kibana/definition/to_string.json
@@ -52,6 +52,18 @@
       "variadic" : false,
       "returnType" : "keyword"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : "Input value. The input can be a single- or multi-valued column or an expression."
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "keyword"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/definition/values.json b/docs/reference/esql/functions/kibana/definition/values.json
index ae69febd4f755..95ac402bb242a 100644
--- a/docs/reference/esql/functions/kibana/definition/values.json
+++ b/docs/reference/esql/functions/kibana/definition/values.json
@@ -28,6 +28,18 @@
       "variadic" : false,
       "returnType" : "date"
     },
+    {
+      "params" : [
+        {
+          "name" : "field",
+          "type" : "date_nanos",
+          "optional" : false,
+          "description" : ""
+        }
+      ],
+      "variadic" : false,
+      "returnType" : "date_nanos"
+    },
     {
       "params" : [
         {
diff --git a/docs/reference/esql/functions/kibana/docs/match.md b/docs/reference/esql/functions/kibana/docs/match.md
index 3c06662982bbf..b866637b41b85 100644
--- a/docs/reference/esql/functions/kibana/docs/match.md
+++ b/docs/reference/esql/functions/kibana/docs/match.md
@@ -6,9 +6,9 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ
 Performs a match query on the specified field. Returns true if the provided query matches the row.
 
 ```
-from books 
-| where match(author, "Faulkner")
-| keep book_no, author 
-| sort book_no 
-| limit 5;
+FROM books 
+| WHERE MATCH(author, "Faulkner")
+| KEEP book_no, author 
+| SORT book_no 
+| LIMIT 5;
 ```
diff --git a/docs/reference/esql/functions/kibana/docs/qstr.md b/docs/reference/esql/functions/kibana/docs/qstr.md
index 37b5777623185..9b5dc3f9a22eb 100644
--- a/docs/reference/esql/functions/kibana/docs/qstr.md
+++ b/docs/reference/esql/functions/kibana/docs/qstr.md
@@ -6,9 +6,9 @@ This is generated by ESQL's AbstractFunctionTestCase. Do no edit it. See ../READ
 Performs a query string query. Returns true if the provided query string matches the row.
 
 ```
-from books 
-| where qstr("author: Faulkner")
-| keep book_no, author 
-| sort book_no 
-| limit 5;
+FROM books 
+| WHERE QSTR("author: Faulkner")
+| KEEP book_no, author 
+| SORT book_no 
+| LIMIT 5;
 ```
diff --git a/docs/reference/esql/functions/types/case.asciidoc b/docs/reference/esql/functions/types/case.asciidoc
index c6fb6a091e9d0..9e6915a37fc14 100644
--- a/docs/reference/esql/functions/types/case.asciidoc
+++ b/docs/reference/esql/functions/types/case.asciidoc
@@ -13,6 +13,8 @@ boolean | cartesian_shape | cartesian_shape | cartesian_shape
 boolean | cartesian_shape | | cartesian_shape
 boolean | date | date | date
 boolean | date | | date
+boolean | date_nanos | date_nanos | date_nanos
+boolean | date_nanos | | date_nanos
 boolean | double | double | double
 boolean | double | | double
 boolean | geo_point | geo_point | geo_point
diff --git a/docs/reference/esql/functions/types/coalesce.asciidoc b/docs/reference/esql/functions/types/coalesce.asciidoc
index 23a249494e0a2..b6479dc7ff86a 100644
--- a/docs/reference/esql/functions/types/coalesce.asciidoc
+++ b/docs/reference/esql/functions/types/coalesce.asciidoc
@@ -10,6 +10,7 @@ boolean | | boolean
 cartesian_point | cartesian_point | cartesian_point
 cartesian_shape | cartesian_shape | cartesian_shape
 date | date | date
+date_nanos | date_nanos | date_nanos
 geo_point | geo_point | geo_point
 geo_shape | geo_shape | geo_shape
 integer | integer | integer
diff --git a/docs/reference/esql/functions/types/count_distinct.asciidoc b/docs/reference/esql/functions/types/count_distinct.asciidoc
index c365c8814573c..f5758a8914d20 100644
--- a/docs/reference/esql/functions/types/count_distinct.asciidoc
+++ b/docs/reference/esql/functions/types/count_distinct.asciidoc
@@ -13,6 +13,10 @@ date | integer | long
 date | long | long
 date | unsigned_long | long
 date | | long
+date_nanos | integer | long
+date_nanos | long | long
+date_nanos | unsigned_long | long
+date_nanos | | long
 double | integer | long
 double | long | long
 double | unsigned_long | long
diff --git a/docs/reference/esql/functions/types/date_trunc.asciidoc b/docs/reference/esql/functions/types/date_trunc.asciidoc
index aa7dee99c6c44..c610f9119e51c 100644
--- a/docs/reference/esql/functions/types/date_trunc.asciidoc
+++ b/docs/reference/esql/functions/types/date_trunc.asciidoc
@@ -6,5 +6,7 @@
 |===
 interval | date | result
 date_period | date | date
+date_period | date_nanos | date_nanos
 time_duration | date | date
+time_duration | date_nanos | date_nanos
 |===
diff --git a/docs/reference/esql/functions/types/equals.asciidoc b/docs/reference/esql/functions/types/equals.asciidoc
index ad0e46ef4b8da..8d48b7ebf084a 100644
--- a/docs/reference/esql/functions/types/equals.asciidoc
+++ b/docs/reference/esql/functions/types/equals.asciidoc
@@ -9,6 +9,7 @@ boolean | boolean | boolean
 cartesian_point | cartesian_point | boolean
 cartesian_shape | cartesian_shape | boolean
 date | date | boolean
+date_nanos | date_nanos | boolean
 double | double | boolean
 double | integer | boolean
 double | long | boolean
diff --git a/docs/reference/esql/functions/types/greater_than.asciidoc b/docs/reference/esql/functions/types/greater_than.asciidoc
index c506328126a94..8000fd34c8507 100644
--- a/docs/reference/esql/functions/types/greater_than.asciidoc
+++ b/docs/reference/esql/functions/types/greater_than.asciidoc
@@ -6,6 +6,7 @@
 |===
 lhs | rhs | result
 date | date | boolean
+date_nanos | date_nanos | boolean
 double | double | boolean
 double | integer | boolean
 double | long | boolean
diff --git a/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc b/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc
index c506328126a94..8000fd34c8507 100644
--- a/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc
+++ b/docs/reference/esql/functions/types/greater_than_or_equal.asciidoc
@@ -6,6 +6,7 @@
 |===
 lhs | rhs | result
 date | date | boolean
+date_nanos | date_nanos | boolean
 double | double | boolean
 double | integer | boolean
 double | long | boolean
diff --git a/docs/reference/esql/functions/types/greatest.asciidoc b/docs/reference/esql/functions/types/greatest.asciidoc
index 7df77a6991315..0bc11b569d426 100644
--- a/docs/reference/esql/functions/types/greatest.asciidoc
+++ b/docs/reference/esql/functions/types/greatest.asciidoc
@@ -8,6 +8,7 @@ first | rest | result
 boolean | boolean | boolean
 boolean | | boolean
 date | date | date
+date_nanos | date_nanos | date_nanos
 double | double | double
 integer | integer | integer
 integer | | integer
diff --git a/docs/reference/esql/functions/types/least.asciidoc b/docs/reference/esql/functions/types/least.asciidoc
index 7df77a6991315..0bc11b569d426 100644
--- a/docs/reference/esql/functions/types/least.asciidoc
+++ b/docs/reference/esql/functions/types/least.asciidoc
@@ -8,6 +8,7 @@ first | rest | result
 boolean | boolean | boolean
 boolean | | boolean
 date | date | date
+date_nanos | date_nanos | date_nanos
 double | double | double
 integer | integer | integer
 integer | | integer
diff --git a/docs/reference/esql/functions/types/less_than.asciidoc b/docs/reference/esql/functions/types/less_than.asciidoc
index c506328126a94..8000fd34c8507 100644
--- a/docs/reference/esql/functions/types/less_than.asciidoc
+++ b/docs/reference/esql/functions/types/less_than.asciidoc
@@ -6,6 +6,7 @@
 |===
 lhs | rhs | result
 date | date | boolean
+date_nanos | date_nanos | boolean
 double | double | boolean
 double | integer | boolean
 double | long | boolean
diff --git a/docs/reference/esql/functions/types/less_than_or_equal.asciidoc b/docs/reference/esql/functions/types/less_than_or_equal.asciidoc
index c506328126a94..8000fd34c8507 100644
--- a/docs/reference/esql/functions/types/less_than_or_equal.asciidoc
+++ b/docs/reference/esql/functions/types/less_than_or_equal.asciidoc
@@ -6,6 +6,7 @@
 |===
 lhs | rhs | result
 date | date | boolean
+date_nanos | date_nanos | boolean
 double | double | boolean
 double | integer | boolean
 double | long | boolean
diff --git a/docs/reference/esql/functions/types/max.asciidoc b/docs/reference/esql/functions/types/max.asciidoc
index 564fb8dc3bfb0..adf457dac31b8 100644
--- a/docs/reference/esql/functions/types/max.asciidoc
+++ b/docs/reference/esql/functions/types/max.asciidoc
@@ -7,6 +7,7 @@
 field | result
 boolean | boolean
 date | date
+date_nanos | date_nanos
 double | double
 integer | integer
 ip | ip
diff --git a/docs/reference/esql/functions/types/min.asciidoc b/docs/reference/esql/functions/types/min.asciidoc
index 564fb8dc3bfb0..adf457dac31b8 100644
--- a/docs/reference/esql/functions/types/min.asciidoc
+++ b/docs/reference/esql/functions/types/min.asciidoc
@@ -7,6 +7,7 @@
 field | result
 boolean | boolean
 date | date
+date_nanos | date_nanos
 double | double
 integer | integer
 ip | ip
diff --git a/docs/reference/esql/functions/types/mv_count.asciidoc b/docs/reference/esql/functions/types/mv_count.asciidoc
index 260c531731f04..c58c4eda44396 100644
--- a/docs/reference/esql/functions/types/mv_count.asciidoc
+++ b/docs/reference/esql/functions/types/mv_count.asciidoc
@@ -9,6 +9,7 @@ boolean | integer
 cartesian_point | integer
 cartesian_shape | integer
 date | integer
+date_nanos | integer
 double | integer
 geo_point | integer
 geo_shape | integer
diff --git a/docs/reference/esql/functions/types/mv_dedupe.asciidoc b/docs/reference/esql/functions/types/mv_dedupe.asciidoc
index 976de79bb0910..1524ec86cd5ec 100644
--- a/docs/reference/esql/functions/types/mv_dedupe.asciidoc
+++ b/docs/reference/esql/functions/types/mv_dedupe.asciidoc
@@ -9,6 +9,7 @@ boolean | boolean
 cartesian_point | cartesian_point
 cartesian_shape | cartesian_shape
 date | date
+date_nanos | date_nanos
 double | double
 geo_point | geo_point
 geo_shape | geo_shape
diff --git a/docs/reference/esql/functions/types/mv_first.asciidoc b/docs/reference/esql/functions/types/mv_first.asciidoc
index 47736e76d1db4..e68af2f992b43 100644
--- a/docs/reference/esql/functions/types/mv_first.asciidoc
+++ b/docs/reference/esql/functions/types/mv_first.asciidoc
@@ -9,6 +9,7 @@ boolean | boolean
 cartesian_point | cartesian_point
 cartesian_shape | cartesian_shape
 date | date
+date_nanos | date_nanos
 double | double
 geo_point | geo_point
 geo_shape | geo_shape
diff --git a/docs/reference/esql/functions/types/mv_last.asciidoc b/docs/reference/esql/functions/types/mv_last.asciidoc
index 47736e76d1db4..e68af2f992b43 100644
--- a/docs/reference/esql/functions/types/mv_last.asciidoc
+++ b/docs/reference/esql/functions/types/mv_last.asciidoc
@@ -9,6 +9,7 @@ boolean | boolean
 cartesian_point | cartesian_point
 cartesian_shape | cartesian_shape
 date | date
+date_nanos | date_nanos
 double | double
 geo_point | geo_point
 geo_shape | geo_shape
diff --git a/docs/reference/esql/functions/types/mv_max.asciidoc b/docs/reference/esql/functions/types/mv_max.asciidoc
index d4e014554c86c..ffba14489b97c 100644
--- a/docs/reference/esql/functions/types/mv_max.asciidoc
+++ b/docs/reference/esql/functions/types/mv_max.asciidoc
@@ -7,6 +7,7 @@
 field | result
 boolean | boolean
 date | date
+date_nanos | date_nanos
 double | double
 integer | integer
 ip | ip
diff --git a/docs/reference/esql/functions/types/mv_min.asciidoc b/docs/reference/esql/functions/types/mv_min.asciidoc
index d4e014554c86c..ffba14489b97c 100644
--- a/docs/reference/esql/functions/types/mv_min.asciidoc
+++ b/docs/reference/esql/functions/types/mv_min.asciidoc
@@ -7,6 +7,7 @@
 field | result
 boolean | boolean
 date | date
+date_nanos | date_nanos
 double | double
 integer | integer
 ip | ip
diff --git a/docs/reference/esql/functions/types/mv_slice.asciidoc b/docs/reference/esql/functions/types/mv_slice.asciidoc
index 60c1f6315a599..75f45e333ee0c 100644
--- a/docs/reference/esql/functions/types/mv_slice.asciidoc
+++ b/docs/reference/esql/functions/types/mv_slice.asciidoc
@@ -9,6 +9,7 @@ boolean | integer | integer | boolean
 cartesian_point | integer | integer | cartesian_point
 cartesian_shape | integer | integer | cartesian_shape
 date | integer | integer | date
+date_nanos | integer | integer | date_nanos
 double | integer | integer | double
 geo_point | integer | integer | geo_point
 geo_shape | integer | integer | geo_shape
diff --git a/docs/reference/esql/functions/types/mv_sort.asciidoc b/docs/reference/esql/functions/types/mv_sort.asciidoc
index c21ea5983945e..83d3e45c7be02 100644
--- a/docs/reference/esql/functions/types/mv_sort.asciidoc
+++ b/docs/reference/esql/functions/types/mv_sort.asciidoc
@@ -7,6 +7,7 @@
 field | order | result
 boolean | keyword | boolean
 date | keyword | date
+date_nanos | keyword | date_nanos
 double | keyword | double
 integer | keyword | integer
 ip | keyword | ip
diff --git a/docs/reference/esql/functions/types/not_equals.asciidoc b/docs/reference/esql/functions/types/not_equals.asciidoc
index ad0e46ef4b8da..8d48b7ebf084a 100644
--- a/docs/reference/esql/functions/types/not_equals.asciidoc
+++ b/docs/reference/esql/functions/types/not_equals.asciidoc
@@ -9,6 +9,7 @@ boolean | boolean | boolean
 cartesian_point | cartesian_point | boolean
 cartesian_shape | cartesian_shape | boolean
 date | date | boolean
+date_nanos | date_nanos | boolean
 double | double | boolean
 double | integer | boolean
 double | long | boolean
diff --git a/docs/reference/esql/functions/types/to_date_nanos.asciidoc b/docs/reference/esql/functions/types/to_date_nanos.asciidoc
index 1f50b65f25a77..dec6833d14b08 100644
--- a/docs/reference/esql/functions/types/to_date_nanos.asciidoc
+++ b/docs/reference/esql/functions/types/to_date_nanos.asciidoc
@@ -5,5 +5,11 @@
 [%header.monospaced.styled,format=dsv,separator=|]
 |===
 field | result
-date_nanos
+date | date_nanos
+date_nanos | date_nanos
+double | date_nanos
+keyword | date_nanos
+long | date_nanos
+text | date_nanos
+unsigned_long | date_nanos
 |===
diff --git a/docs/reference/esql/functions/types/to_datetime.asciidoc b/docs/reference/esql/functions/types/to_datetime.asciidoc
index 80c986efca794..118ed6c09c11c 100644
--- a/docs/reference/esql/functions/types/to_datetime.asciidoc
+++ b/docs/reference/esql/functions/types/to_datetime.asciidoc
@@ -6,6 +6,7 @@
 |===
 field | result
 date | date
+date_nanos | date
 double | date
 integer | date
 keyword | date
diff --git a/docs/reference/esql/functions/types/to_long.asciidoc b/docs/reference/esql/functions/types/to_long.asciidoc
index a07990cb1cfbf..1009543c1bbde 100644
--- a/docs/reference/esql/functions/types/to_long.asciidoc
+++ b/docs/reference/esql/functions/types/to_long.asciidoc
@@ -9,6 +9,7 @@ boolean | long
 counter_integer | long
 counter_long | long
 date | long
+date_nanos | long
 double | long
 integer | long
 keyword | long
diff --git a/docs/reference/esql/functions/types/to_string.asciidoc b/docs/reference/esql/functions/types/to_string.asciidoc
index 26a5b31a2a589..9d4188214b3d8 100644
--- a/docs/reference/esql/functions/types/to_string.asciidoc
+++ b/docs/reference/esql/functions/types/to_string.asciidoc
@@ -9,6 +9,7 @@ boolean | keyword
 cartesian_point | keyword
 cartesian_shape | keyword
 date | keyword
+date_nanos | keyword
 double | keyword
 geo_point | keyword
 geo_shape | keyword
diff --git a/docs/reference/esql/functions/types/values.asciidoc b/docs/reference/esql/functions/types/values.asciidoc
index 564fb8dc3bfb0..adf457dac31b8 100644
--- a/docs/reference/esql/functions/types/values.asciidoc
+++ b/docs/reference/esql/functions/types/values.asciidoc
@@ -7,6 +7,7 @@
 field | result
 boolean | boolean
 date | date
+date_nanos | date_nanos
 double | double
 integer | integer
 ip | ip
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java
index d84a471815a9a..61b480968e974 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java
@@ -12,7 +12,6 @@
 import org.elasticsearch.plugins.Plugin;
 
 public class EsqlCorePlugin extends Plugin implements ExtensiblePlugin {
-    public static final FeatureFlag DATE_NANOS_FEATURE_FLAG = new FeatureFlag("esql_date_nanos");
 
     public static final FeatureFlag SEMANTIC_TEXT_FEATURE_FLAG = new FeatureFlag("esql_semantic_text");
 }
diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
index e980b1509813e..1c65dd386667f 100644
--- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
+++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java
@@ -209,7 +209,6 @@ public enum DataType {
      * check that sending them to a function produces a sane error message.
      */
     public static final Map<DataType, FeatureFlag> UNDER_CONSTRUCTION = Map.ofEntries(
-        Map.entry(DATE_NANOS, EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG),
         Map.entry(SEMANTIC_TEXT, EsqlCorePlugin.SEMANTIC_TEXT_FEATURE_FLAG)
     );
 
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
index 2a62216072e9c..ab3a7e3e7d0b8 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
@@ -322,32 +322,32 @@ public enum Cap {
         /**
          * Support for nanosecond dates as a data type
          */
-        DATE_NANOS_TYPE(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG),
+        DATE_NANOS_TYPE(),
 
         /**
          * Support for to_date_nanos function
          */
-        TO_DATE_NANOS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG),
+        TO_DATE_NANOS(),
 
         /**
          * Support for date nanos type in binary comparisons
          */
-        DATE_NANOS_BINARY_COMPARISON(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG),
+        DATE_NANOS_BINARY_COMPARISON(),
 
         /**
          * Support Least and Greatest functions on Date Nanos type
          */
-        LEAST_GREATEST_FOR_DATENANOS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG),
+        LEAST_GREATEST_FOR_DATENANOS(),
 
         /**
          * Support for date_trunc function on date nanos type
          */
-        DATE_TRUNC_DATE_NANOS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG),
+        DATE_TRUNC_DATE_NANOS(),
 
         /**
          * support aggregations on date nanos
          */
-        DATE_NANOS_AGGREGATIONS(EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG),
+        DATE_NANOS_AGGREGATIONS(),
 
         /**
          * Support for datetime in least and greatest functions
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
index ca02441d2e1ad..eafb1fdbcbdcb 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java
@@ -501,7 +501,9 @@ private static DataType getTargetType(String[] names) {
                 types.add(type);
             }
         }
+
         return types.stream()
+            .filter(DATA_TYPE_CASTING_PRIORITY::containsKey)
             .min((dt1, dt2) -> DATA_TYPE_CASTING_PRIORITY.get(dt1).compareTo(DATA_TYPE_CASTING_PRIORITY.get(dt2)))
             .orElse(UNSUPPORTED);
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java
index ac2d4ff3cbc43..2165c3c7ad1a0 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Max.java
@@ -55,7 +55,7 @@ public class Max extends AggregateFunction implements ToAggregator, SurrogateExp
     );
 
     @FunctionInfo(
-        returnType = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "long", "version" },
+        returnType = { "boolean", "double", "integer", "long", "date", "date_nanos", "ip", "keyword", "long", "version" },
         description = "The maximum value of a field.",
         isAggregation = true,
         examples = {
@@ -72,7 +72,7 @@ public Max(
         Source source,
         @Param(
             name = "field",
-            type = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" }
+            type = { "boolean", "double", "integer", "long", "date", "date_nanos", "ip", "keyword", "text", "long", "version" }
         ) Expression field
     ) {
         this(source, field, Literal.TRUE);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java
index a5fc8196847b7..7d67868dd4134 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Min.java
@@ -55,7 +55,7 @@ public class Min extends AggregateFunction implements ToAggregator, SurrogateExp
     );
 
     @FunctionInfo(
-        returnType = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "long", "version" },
+        returnType = { "boolean", "double", "integer", "long", "date", "date_nanos", "ip", "keyword", "long", "version" },
         description = "The minimum value of a field.",
         isAggregation = true,
         examples = {
@@ -72,7 +72,7 @@ public Min(
         Source source,
         @Param(
             name = "field",
-            type = { "boolean", "double", "integer", "long", "date", "ip", "keyword", "text", "long", "version" }
+            type = { "boolean", "double", "integer", "long", "date", "date_nanos", "ip", "keyword", "text", "long", "version" }
         ) Expression field
     ) {
         this(source, field, Literal.TRUE);
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java
index 111eab051719b..e7df990b20422 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java
@@ -52,7 +52,7 @@ public class Values extends AggregateFunction implements ToAggregator {
     );
 
     @FunctionInfo(
-        returnType = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "version" },
+        returnType = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "version" },
         preview = true,
         description = "Returns all values in a group as a multivalued field. The order of the returned values isn't guaranteed. "
             + "If you need the values returned in order use <<esql-mv_sort>>.",
@@ -70,7 +70,10 @@ public class Values extends AggregateFunction implements ToAggregator {
     )
     public Values(
         Source source,
-        @Param(name = "field", type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "version" }) Expression v
+        @Param(
+            name = "field",
+            type = { "boolean", "date", "date_nanos", "double", "integer", "ip", "keyword", "long", "text", "version" }
+        ) Expression v
     ) {
         this(source, v, Literal.TRUE);
     }
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java
index 6e38d72500840..a35b67d7ac3fd 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateTrunc.java
@@ -62,7 +62,7 @@ public interface DateTruncFactoryProvider {
     protected static final ZoneId DEFAULT_TZ = ZoneOffset.UTC;
 
     @FunctionInfo(
-        returnType = "date",
+        returnType = { "date", "date_nanos" },
         description = "Rounds down a date to the closest interval.",
         examples = {
             @Example(file = "date", tag = "docsDateTrunc"),
@@ -83,7 +83,7 @@ public DateTrunc(
             type = { "date_period", "time_duration" },
             description = "Interval; expressed using the timespan literal syntax."
         ) Expression interval,
-        @Param(name = "date", type = { "date" }, description = "Date expression") Expression field
+        @Param(name = "date", type = { "date", "date_nanos" }, description = "Date expression") Expression field
     ) {
         super(source, List.of(interval, field));
         this.interval = interval;
diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java
index 56ade3982e0d8..3ae7bd93092ef 100644
--- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java
+++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThan.java
@@ -53,12 +53,12 @@ public LessThan(
         Source source,
         @Param(
             name = "lhs",
-            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
+            type = { "boolean", "date_nanos", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
             description = "An expression."
         ) Expression left,
         @Param(
             name = "rhs",
-            type = { "boolean", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
+            type = { "boolean", "date_nanos", "date", "double", "integer", "ip", "keyword", "long", "text", "unsigned_long", "version" },
             description = "An expression."
         ) Expression right
     ) {
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java
index 7fe67707a7976..775ca45bfa124 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/MultiRowTestCaseSupplier.java
@@ -262,6 +262,42 @@ public static List<TypedDataSupplier> dateCases(int minRows, int maxRows) {
         return cases;
     }
 
+    /**
+     *
+     * Generate cases for {@link DataType#DATE_NANOS}.
+     *
+     */
+    public static List<TypedDataSupplier> dateNanosCases(int minRows, int maxRows) {
+        List<TypedDataSupplier> cases = new ArrayList<>();
+        addSuppliers(cases, minRows, maxRows, "<1970-01-01T00:00:00.000000000Z>", DataType.DATE_NANOS, () -> 0L);
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "<date nanos>",
+            DataType.DATE_NANOS,
+            () -> ESTestCase.randomLongBetween(0, 10 * (long) 10e11)
+        );
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "<far future date nanos>",
+            DataType.DATE_NANOS,
+            () -> ESTestCase.randomLongBetween(10 * (long) 10e11, Long.MAX_VALUE)
+        );
+        addSuppliers(
+            cases,
+            minRows,
+            maxRows,
+            "<nanos near the end of time>",
+            DataType.DATE_NANOS,
+            () -> ESTestCase.randomLongBetween(Long.MAX_VALUE / 100 * 99, Long.MAX_VALUE)
+        );
+
+        return cases;
+    }
+
     public static List<TypedDataSupplier> booleanCases(int minRows, int maxRows) {
         List<TypedDataSupplier> cases = new ArrayList<>();
 
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java
index 5e23083d7c810..fff2d824fc710 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountDistinctTests.java
@@ -52,6 +52,7 @@ public static Iterable<Object[]> parameters() {
             MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true),
             MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true),
             MultiRowTestCaseSupplier.dateCases(1, 1000),
+            MultiRowTestCaseSupplier.dateNanosCases(1, 1000),
             MultiRowTestCaseSupplier.booleanCases(1, 1000),
             MultiRowTestCaseSupplier.ipCases(1, 1000),
             MultiRowTestCaseSupplier.versionCases(1, 1000),
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java
index 9756804a1ec0f..7d4b46f2a902a 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MaxTests.java
@@ -90,6 +90,15 @@ public static Iterable<Object[]> parameters() {
                         equalTo(200L)
                     )
                 ),
+                new TestCaseSupplier(
+                    List.of(DataType.DATE_NANOS),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(200L), DataType.DATE_NANOS, "field")),
+                        "Max[field=Attribute[channel=0]]",
+                        DataType.DATE_NANOS,
+                        equalTo(200L)
+                    )
+                ),
                 new TestCaseSupplier(
                     List.of(DataType.BOOLEAN),
                     () -> new TestCaseSupplier.TestCase(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java
index 171181496c889..58ef8d86017a8 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/MinTests.java
@@ -90,6 +90,15 @@ public static Iterable<Object[]> parameters() {
                         equalTo(200L)
                     )
                 ),
+                new TestCaseSupplier(
+                    List.of(DataType.DATE_NANOS),
+                    () -> new TestCaseSupplier.TestCase(
+                        List.of(TestCaseSupplier.TypedData.multiRow(List.of(200L), DataType.DATE_NANOS, "field")),
+                        "Min[field=Attribute[channel=0]]",
+                        DataType.DATE_NANOS,
+                        equalTo(200L)
+                    )
+                ),
                 new TestCaseSupplier(
                     List.of(DataType.BOOLEAN),
                     () -> new TestCaseSupplier.TestCase(
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java
index 55320543d0ec3..29faceee7497e 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesTests.java
@@ -45,6 +45,7 @@ public static Iterable<Object[]> parameters() {
             MultiRowTestCaseSupplier.longCases(1, 1000, Long.MIN_VALUE, Long.MAX_VALUE, true),
             MultiRowTestCaseSupplier.doubleCases(1, 1000, -Double.MAX_VALUE, Double.MAX_VALUE, true),
             MultiRowTestCaseSupplier.dateCases(1, 1000),
+            MultiRowTestCaseSupplier.dateNanosCases(1, 1000),
             MultiRowTestCaseSupplier.booleanCases(1, 1000),
             MultiRowTestCaseSupplier.ipCases(1, 1000),
             MultiRowTestCaseSupplier.versionCases(1, 1000),
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java
index 65f5653f27e1a..11894cf5b847b 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/AbstractMultivalueFunctionTestCase.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.geometry.Geometry;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.core.expression.Expression;
-import org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.util.NumericUtils;
@@ -397,9 +396,6 @@ protected static void dateNanos(
         DataType expectedDataType,
         BiFunction<Integer, LongStream, Matcher<Object>> matcher
     ) {
-        if (EsqlCorePlugin.DATE_NANOS_FEATURE_FLAG.isEnabled() == false) {
-            return;
-        }
         cases.add(
             new TestCaseSupplier(
                 name + "(epoch nanos)",
diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java
index e8f9f26a76f43..0d114b4964920 100644
--- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java
+++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/LessThanTests.java
@@ -113,8 +113,8 @@ public static Iterable<Object[]> parameters() {
                 "rhs",
                 (l, r) -> ((Number) l).longValue() < ((Number) r).longValue(),
                 DataType.BOOLEAN,
-                TestCaseSupplier.dateCases(),
-                TestCaseSupplier.dateCases(),
+                TestCaseSupplier.dateNanosCases(),
+                TestCaseSupplier.dateNanosCases(),
                 List.of(),
                 false
             )

From dea1e7dfa7ae598879a7d04233b9665d55c32497 Mon Sep 17 00:00:00 2001
From: Jan Kuipers <148754765+jan-elastic@users.noreply.github.com>
Date: Wed, 20 Nov 2024 15:34:19 +0100
Subject: [PATCH 04/14] Propagate scoring function through random sampler
 (#116957)

* Propagate scoring function through random sampler.

* Update docs/changelog/116957.yaml

* Correct score mode in random sampler weight

* Fix random sampling with scores and p=1.0

* Unit test with scores

* YAML test

* Add capability
---
 docs/changelog/116957.yaml                    |  5 ++
 modules/aggregations/build.gradle             |  2 +-
 .../test/aggregations/random_sampler.yml      | 60 +++++++++++++++++++
 .../action/search/SearchCapabilities.java     |  3 +
 .../search/aggregations/AggregatorBase.java   |  2 +-
 .../random/RandomSamplerAggregator.java       | 53 ++++++++++++----
 .../RandomSamplerAggregatorFactory.java       | 42 +------------
 .../random/RandomSamplerAggregatorTests.java  | 38 ++++++++++++
 8 files changed, 150 insertions(+), 55 deletions(-)
 create mode 100644 docs/changelog/116957.yaml

diff --git a/docs/changelog/116957.yaml b/docs/changelog/116957.yaml
new file mode 100644
index 0000000000000..1020190de180d
--- /dev/null
+++ b/docs/changelog/116957.yaml
@@ -0,0 +1,5 @@
+pr: 116957
+summary: Propagate scoring function through random sampler
+area: Machine Learning
+type: bug
+issues: [ 110134 ]
diff --git a/modules/aggregations/build.gradle b/modules/aggregations/build.gradle
index 5df0a890af753..2835180904620 100644
--- a/modules/aggregations/build.gradle
+++ b/modules/aggregations/build.gradle
@@ -20,7 +20,7 @@ esplugin {
 
 restResources {
   restApi {
-    include '_common', 'indices', 'cluster', 'index', 'search', 'nodes', 'bulk', 'scripts_painless_execute', 'put_script'
+    include 'capabilities', '_common', 'indices', 'cluster', 'index', 'search', 'nodes', 'bulk', 'scripts_painless_execute', 'put_script'
   }
   restTests {
     // Pulls in all aggregation tests from core AND the forwards v7's core for forwards compatibility
diff --git a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/random_sampler.yml b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/random_sampler.yml
index 5b2c2dc379cb9..4d8efe2a6f9d8 100644
--- a/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/random_sampler.yml
+++ b/modules/aggregations/src/yamlRestTest/resources/rest-api-spec/test/aggregations/random_sampler.yml
@@ -142,6 +142,66 @@ setup:
           }
   - match: { aggregations.sampled.mean.value: 1.0 }
 ---
+"Test random_sampler aggregation with scored subagg":
+  - requires:
+      capabilities:
+        - method: POST
+          path: /_search
+          capabilities: [ random_sampler_with_scored_subaggs ]
+      test_runner_features: capabilities
+      reason: "Support for random sampler with scored subaggs capability required"
+  - do:
+      search:
+        index: data
+        size: 0
+        body: >
+          {
+            "query": {
+              "function_score": {
+                "random_score": {}
+              }
+            },
+            "aggs": {
+              "sampled": {
+                "random_sampler": {
+                  "probability": 0.5
+                },
+                "aggs": {
+                  "top": {
+                    "top_hits": {}
+                  }
+                }
+              }
+            }
+          }
+  - is_true: aggregations.sampled.top.hits
+  - do:
+      search:
+        index: data
+        size: 0
+        body: >
+          {
+            "query": {
+              "function_score": {
+                "random_score": {}
+              }
+            },
+            "aggs": {
+              "sampled": {
+                "random_sampler": {
+                  "probability": 1.0
+                },
+                "aggs": {
+                  "top": {
+                    "top_hits": {}
+                  }
+                }
+              }
+            }
+          }
+  - match: { aggregations.sampled.top.hits.total.value: 6 }
+  - is_true: aggregations.sampled.top.hits.hits.0._score
+---
 "Test random_sampler aggregation with poor settings":
   - requires:
       cluster_features: ["gte_v8.2.0"]
diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
index 7b57481ad5716..241f30b367782 100644
--- a/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
+++ b/server/src/main/java/org/elasticsearch/rest/action/search/SearchCapabilities.java
@@ -41,6 +41,8 @@ private SearchCapabilities() {}
     /** Support multi-dense-vector script field access. */
     private static final String MULTI_DENSE_VECTOR_SCRIPT_ACCESS = "multi_dense_vector_script_access";
 
+    private static final String RANDOM_SAMPLER_WITH_SCORED_SUBAGGS = "random_sampler_with_scored_subaggs";
+
     public static final Set<String> CAPABILITIES;
     static {
         HashSet<String> capabilities = new HashSet<>();
@@ -50,6 +52,7 @@ private SearchCapabilities() {}
         capabilities.add(DENSE_VECTOR_DOCVALUE_FIELDS);
         capabilities.add(TRANSFORM_RANK_RRF_TO_RETRIEVER);
         capabilities.add(NESTED_RETRIEVER_INNER_HITS_SUPPORT);
+        capabilities.add(RANDOM_SAMPLER_WITH_SCORED_SUBAGGS);
         if (MultiDenseVectorFieldMapper.FEATURE_FLAG.isEnabled()) {
             capabilities.add(MULTI_DENSE_VECTOR_FIELD_MAPPER);
             capabilities.add(MULTI_DENSE_VECTOR_SCRIPT_ACCESS);
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
index bf9116207b375..1ea7769b33384 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregatorBase.java
@@ -40,7 +40,7 @@ public abstract class AggregatorBase extends Aggregator {
 
     protected final String name;
     protected final Aggregator parent;
-    private final AggregationContext context;
+    protected final AggregationContext context;
     private final Map<String, Object> metadata;
 
     protected final Aggregator[] subAggregators;
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregator.java
index 921cbb96385ad..699b8c6b5d500 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregator.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregator.java
@@ -9,12 +9,15 @@
 
 package org.elasticsearch.search.aggregations.bucket.sampler.random;
 
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.CollectionTerminatedException;
 import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Scorer;
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.util.Bits;
-import org.elasticsearch.common.CheckedSupplier;
 import org.elasticsearch.common.util.LongArray;
 import org.elasticsearch.search.aggregations.AggregationExecutionContext;
 import org.elasticsearch.search.aggregations.Aggregator;
@@ -22,6 +25,7 @@
 import org.elasticsearch.search.aggregations.CardinalityUpperBound;
 import org.elasticsearch.search.aggregations.InternalAggregation;
 import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
 import org.elasticsearch.search.aggregations.bucket.BucketsAggregator;
 import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregator;
 import org.elasticsearch.search.aggregations.support.AggregationContext;
@@ -34,14 +38,13 @@ public class RandomSamplerAggregator extends BucketsAggregator implements Single
     private final int seed;
     private final Integer shardSeed;
     private final double probability;
-    private final CheckedSupplier<Weight, IOException> weightSupplier;
+    private Weight weight;
 
     RandomSamplerAggregator(
         String name,
         int seed,
         Integer shardSeed,
         double probability,
-        CheckedSupplier<Weight, IOException> weightSupplier,
         AggregatorFactories factories,
         AggregationContext context,
         Aggregator parent,
@@ -56,10 +59,33 @@ public class RandomSamplerAggregator extends BucketsAggregator implements Single
                 RandomSamplerAggregationBuilder.NAME + " aggregation [" + name + "] must have sub aggregations configured"
             );
         }
-        this.weightSupplier = weightSupplier;
         this.shardSeed = shardSeed;
     }
 
+    /**
+     * This creates the query weight which will be used in the aggregator.
+     *
+     * This weight is a boolean query between {@link RandomSamplingQuery} and the configured top level query of the search. This allows
+     * the aggregation to iterate the documents directly, thus sampling in the background instead of the foreground.
+     * @return weight to be used, is cached for additional usages
+     * @throws IOException when building the weight or queries fails;
+     */
+    private Weight getWeight() throws IOException {
+        if (weight == null) {
+            ScoreMode scoreMode = scoreMode();
+            BooleanQuery.Builder fullQuery = new BooleanQuery.Builder().add(
+                context.query(),
+                scoreMode.needsScores() ? BooleanClause.Occur.MUST : BooleanClause.Occur.FILTER
+            );
+            if (probability < 1.0) {
+                Query sampleQuery = new RandomSamplingQuery(probability, seed, shardSeed == null ? context.shardRandomSeed() : shardSeed);
+                fullQuery.add(sampleQuery, BooleanClause.Occur.FILTER);
+            }
+            weight = context.searcher().createWeight(context.searcher().rewrite(fullQuery.build()), scoreMode, 1f);
+        }
+        return weight;
+    }
+
     @Override
     public InternalAggregation[] buildAggregations(LongArray owningBucketOrds) throws IOException {
         return buildAggregationsForSingleBucket(
@@ -101,22 +127,26 @@ protected LeafBucketCollector getLeafCollector(AggregationExecutionContext aggCt
         if (sub.isNoop()) {
             return LeafBucketCollector.NO_OP_COLLECTOR;
         }
+
+        Scorer scorer = getWeight().scorer(aggCtx.getLeafReaderContext());
+        // This means there are no docs to iterate, possibly due to the fields not existing
+        if (scorer == null) {
+            return LeafBucketCollector.NO_OP_COLLECTOR;
+        }
+        sub.setScorer(scorer);
+
         // No sampling is being done, collect all docs
+        // TODO know when sampling would be much slower and skip sampling: https://github.com/elastic/elasticsearch/issues/84353
         if (probability >= 1.0) {
             grow(1);
-            return new LeafBucketCollector() {
+            return new LeafBucketCollectorBase(sub, null) {
                 @Override
                 public void collect(int doc, long owningBucketOrd) throws IOException {
                     collectExistingBucket(sub, doc, 0);
                 }
             };
         }
-        // TODO know when sampling would be much slower and skip sampling: https://github.com/elastic/elasticsearch/issues/84353
-        Scorer scorer = weightSupplier.get().scorer(aggCtx.getLeafReaderContext());
-        // This means there are no docs to iterate, possibly due to the fields not existing
-        if (scorer == null) {
-            return LeafBucketCollector.NO_OP_COLLECTOR;
-        }
+
         final DocIdSetIterator docIt = scorer.iterator();
         final Bits liveDocs = aggCtx.getLeafReaderContext().reader().getLiveDocs();
         try {
@@ -136,5 +166,4 @@ public void collect(int doc, long owningBucketOrd) throws IOException {
         // Since we have done our own collection, there is nothing for the leaf collector to do
         return LeafBucketCollector.NO_OP_COLLECTOR;
     }
-
 }
diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregatorFactory.java
index 67c958046dac7..50921501896d3 100644
--- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregatorFactory.java
+++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregatorFactory.java
@@ -9,10 +9,6 @@
 
 package org.elasticsearch.search.aggregations.bucket.sampler.random;
 
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.ScoreMode;
-import org.apache.lucene.search.Weight;
 import org.elasticsearch.search.aggregations.Aggregator;
 import org.elasticsearch.search.aggregations.AggregatorFactories;
 import org.elasticsearch.search.aggregations.AggregatorFactory;
@@ -30,7 +26,6 @@ public class RandomSamplerAggregatorFactory extends AggregatorFactory {
     private final Integer shardSeed;
     private final double probability;
     private final SamplingContext samplingContext;
-    private Weight weight;
 
     RandomSamplerAggregatorFactory(
         String name,
@@ -57,41 +52,6 @@ public Optional<SamplingContext> getSamplingContext() {
     @Override
     public Aggregator createInternal(Aggregator parent, CardinalityUpperBound cardinality, Map<String, Object> metadata)
         throws IOException {
-        return new RandomSamplerAggregator(
-            name,
-            seed,
-            shardSeed,
-            probability,
-            this::getWeight,
-            factories,
-            context,
-            parent,
-            cardinality,
-            metadata
-        );
+        return new RandomSamplerAggregator(name, seed, shardSeed, probability, factories, context, parent, cardinality, metadata);
     }
-
-    /**
-     * This creates the query weight which will be used in the aggregator.
-     *
-     * This weight is a boolean query between {@link RandomSamplingQuery} and the configured top level query of the search. This allows
-     * the aggregation to iterate the documents directly, thus sampling in the background instead of the foreground.
-     * @return weight to be used, is cached for additional usages
-     * @throws IOException when building the weight or queries fails;
-     */
-    private Weight getWeight() throws IOException {
-        if (weight == null) {
-            RandomSamplingQuery query = new RandomSamplingQuery(
-                probability,
-                seed,
-                shardSeed == null ? context.shardRandomSeed() : shardSeed
-            );
-            BooleanQuery booleanQuery = new BooleanQuery.Builder().add(query, BooleanClause.Occur.FILTER)
-                .add(context.query(), BooleanClause.Occur.FILTER)
-                .build();
-            weight = context.searcher().createWeight(context.searcher().rewrite(booleanQuery), ScoreMode.COMPLETE_NO_SCORES, 1f);
-        }
-        return weight;
-    }
-
 }
diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregatorTests.java
index f75f9f474c8e8..2f51a5a09a8ac 100644
--- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregatorTests.java
+++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/sampler/random/RandomSamplerAggregatorTests.java
@@ -11,22 +11,29 @@
 
 import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.tests.index.RandomIndexWriter;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.AggregationBuilders;
 import org.elasticsearch.search.aggregations.AggregatorTestCase;
 import org.elasticsearch.search.aggregations.bucket.filter.Filter;
 import org.elasticsearch.search.aggregations.metrics.Avg;
 import org.elasticsearch.search.aggregations.metrics.Max;
 import org.elasticsearch.search.aggregations.metrics.Min;
+import org.elasticsearch.search.aggregations.metrics.TopHits;
 import org.hamcrest.Description;
 import org.hamcrest.Matcher;
 import org.hamcrest.TypeSafeMatcher;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.DoubleStream;
@@ -37,6 +44,8 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.lessThan;
 import static org.hamcrest.Matchers.lessThanOrEqualTo;
 import static org.hamcrest.Matchers.not;
 import static org.hamcrest.Matchers.notANumber;
@@ -76,6 +85,35 @@ public void testAggregationSampling() throws IOException {
         assertThat(avgAvg, closeTo(1.5, 0.5));
     }
 
+    public void testAggregationSampling_withScores() throws IOException {
+        long[] counts = new long[5];
+        AtomicInteger integer = new AtomicInteger();
+        do {
+            testCase(RandomSamplerAggregatorTests::writeTestDocs, (InternalRandomSampler result) -> {
+                counts[integer.get()] = result.getDocCount();
+                if (result.getDocCount() > 0) {
+                    TopHits agg = result.getAggregations().get("top");
+                    List<SearchHit> hits = Arrays.asList(agg.getHits().getHits());
+                    assertThat(Strings.toString(result), hits, hasSize(1));
+                    assertThat(Strings.toString(result), hits.get(0).getScore(), allOf(greaterThan(0.0f), lessThan(1.0f)));
+                }
+            },
+                new AggTestConfig(
+                    new RandomSamplerAggregationBuilder("my_agg").subAggregation(AggregationBuilders.topHits("top").size(1))
+                        .setProbability(0.25),
+                    longField(NUMERIC_FIELD_NAME)
+                ).withQuery(
+                    new BooleanQuery.Builder().add(
+                        new TermQuery(new Term(KEYWORD_FIELD_NAME, KEYWORD_FIELD_VALUE)),
+                        BooleanClause.Occur.SHOULD
+                    ).build()
+                )
+            );
+        } while (integer.incrementAndGet() < 5);
+        long avgCount = LongStream.of(counts).sum() / integer.get();
+        assertThat(avgCount, allOf(greaterThanOrEqualTo(20L), lessThanOrEqualTo(70L)));
+    }
+
     public void testAggregationSamplingNestedAggsScaled() throws IOException {
         // in case 0 docs get sampled, which can rarely happen
         // in case the test index has many segments.

From 20166ae425a7d4bb715f6c1f3c157f4037e9479c Mon Sep 17 00:00:00 2001
From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com>
Date: Wed, 20 Nov 2024 16:40:23 +0200
Subject: [PATCH 05/14] Unmute `Failed to snapshot indices with synthetic
 source` (#117117)

---
 muted-tests.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index cc568144698ee..20cea26eb4be7 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -239,9 +239,6 @@ tests:
 - class: org.elasticsearch.upgrades.QueryBuilderBWCIT
   method: testQueryBuilderBWC {cluster=UPGRADED}
   issue: https://github.com/elastic/elasticsearch/issues/116990
-- class: org.elasticsearch.xpack.test.rest.XPackRestIT
-  method: test {p0=snapshot/10_basic/Failed to snapshot indices with synthetic source}
-  issue: https://github.com/elastic/elasticsearch/issues/117082
 - class: org.elasticsearch.discovery.ClusterDisruptionIT
   method: testAckedIndexing
   issue: https://github.com/elastic/elasticsearch/issues/117024

From d5bc38958617516511b68de8995e57e824c0da95 Mon Sep 17 00:00:00 2001
From: Nik Everett <nik9000@gmail.com>
Date: Wed, 20 Nov 2024 09:51:23 -0500
Subject: [PATCH 06/14] ESQL: Fix invariant test in TOP(bytes) (#117049)

Fixes a self-test in the code for `TOP(bytes)`, specifically around
the merging used for grouping by ordinals.
---
 .../data/sort/BytesRefBucketedSort.java       |  2 +-
 .../data/sort/BucketedSortTestCase.java       | 46 +++++++++++++++++++
 2 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/BytesRefBucketedSort.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/BytesRefBucketedSort.java
index 9198de53b1e04..6dca94b9bc79a 100644
--- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/BytesRefBucketedSort.java
+++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/sort/BytesRefBucketedSort.java
@@ -147,7 +147,7 @@ public void merge(int bucket, BytesRefBucketedSort other, int otherBucket) {
             // The value was never collected.
             return;
         }
-        other.checkInvariant(bucket);
+        other.checkInvariant(otherBucket);
         long otherStart = other.startIndex(otherBucket, otherRootIndex);
         long otherEnd = other.common.endIndex(otherRootIndex);
         // TODO: This can be improved for heapified buckets by making use of the heap structures
diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java
index f857f50b2d30f..339c2bba2a734 100644
--- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java
+++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/data/sort/BucketedSortTestCase.java
@@ -363,6 +363,52 @@ public final void testMergeEmptyToEmpty() {
         }
     }
 
+    public final void testMergeOtherBigger() {
+        try (T sort = build(SortOrder.ASC, 3)) {
+            var values = threeSortedValues();
+
+            collect(sort, values.get(0), 0);
+            collect(sort, values.get(1), 0);
+            collect(sort, values.get(2), 0);
+
+            try (T other = build(SortOrder.ASC, 3)) {
+                collect(other, values.get(0), 0);
+                collect(other, values.get(1), 1);
+                collect(other, values.get(2), 2);
+
+                merge(sort, 0, other, 0);
+                merge(sort, 0, other, 1);
+                merge(sort, 0, other, 2);
+            }
+
+            assertBlock(sort, 0, List.of(values.get(0), values.get(0), values.get(1)));
+        }
+    }
+
+    public final void testMergeThisBigger() {
+        try (T sort = build(SortOrder.ASC, 3)) {
+            var values = threeSortedValues();
+
+            collect(sort, values.get(0), 0);
+            collect(sort, values.get(1), 1);
+            collect(sort, values.get(2), 2);
+
+            try (T other = build(SortOrder.ASC, 3)) {
+                collect(other, values.get(0), 0);
+                collect(other, values.get(1), 0);
+                collect(other, values.get(2), 0);
+
+                merge(sort, 0, other, 0);
+                merge(sort, 1, other, 0);
+                merge(sort, 2, other, 0);
+            }
+
+            assertBlock(sort, 0, List.of(values.get(0), values.get(0), values.get(1)));
+            assertBlock(sort, 1, List.of(values.get(0), values.get(1), values.get(1)));
+            assertBlock(sort, 2, values);
+        }
+    }
+
     protected void assertBlock(T sort, int groupId, List<V> values) {
         var blockFactory = TestBlockFactory.getNonBreakingInstance();
 

From 537bba6787ee43fc691e8bb2a586a51f46090bae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20FOUCRET?= <aurelien.foucret@gmail.com>
Date: Wed, 20 Nov 2024 16:20:44 +0100
Subject: [PATCH 07/14] Disable KqlQueryBuilderTests on non-snapshots build.
 (#117025)

---
 muted-tests.yml                                           | 2 --
 x-pack/plugin/kql/build.gradle                            | 8 --------
 .../xpack/kql/query/KqlQueryBuilderTests.java             | 6 ++++++
 3 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index 20cea26eb4be7..a7d8995253dd8 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -180,8 +180,6 @@ tests:
 - class: org.elasticsearch.xpack.downsample.ILMDownsampleDisruptionIT
   method: testILMDownsampleRollingRestart
   issue: https://github.com/elastic/elasticsearch/issues/114233
-- class: org.elasticsearch.xpack.kql.query.KqlQueryBuilderTests
-  issue: https://github.com/elastic/elasticsearch/issues/116487
 - class: org.elasticsearch.reservedstate.service.FileSettingsServiceTests
   method: testInvalidJSON
   issue: https://github.com/elastic/elasticsearch/issues/116521
diff --git a/x-pack/plugin/kql/build.gradle b/x-pack/plugin/kql/build.gradle
index 054011a458fe0..79f2c91114bd9 100644
--- a/x-pack/plugin/kql/build.gradle
+++ b/x-pack/plugin/kql/build.gradle
@@ -1,5 +1,3 @@
-import org.elasticsearch.gradle.internal.info.BuildParams
-
 import static org.elasticsearch.gradle.util.PlatformUtils.normalize
 
 apply plugin: 'elasticsearch.internal-es-plugin'
@@ -28,12 +26,6 @@ dependencies {
 
 tasks.named('yamlRestTest').configure {
   usesDefaultDistribution()
-
-  /****************************************************************
-  *  Enable QA/rest integration tests for snapshot builds only   *
-  *  TODO: Enable for all builds upon this feature release       *
-    ****************************************************************/
-  enabled = buildParams.isSnapshotBuild()
 }
 
 /**********************************
diff --git a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java
index 2bc23c7d457dd..7323f7d6d1a4e 100644
--- a/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java
+++ b/x-pack/plugin/kql/src/test/java/org/elasticsearch/xpack/kql/query/KqlQueryBuilderTests.java
@@ -8,6 +8,7 @@
 package org.elasticsearch.xpack.kql.query;
 
 import org.apache.lucene.search.Query;
+import org.elasticsearch.Build;
 import org.elasticsearch.core.Strings;
 import org.elasticsearch.index.query.MultiMatchQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
@@ -21,6 +22,7 @@
 import org.elasticsearch.test.AbstractQueryTestCase;
 import org.elasticsearch.xpack.kql.KqlPlugin;
 import org.hamcrest.Matchers;
+import org.junit.BeforeClass;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -34,6 +36,10 @@
 import static org.hamcrest.Matchers.nullValue;
 
 public class KqlQueryBuilderTests extends AbstractQueryTestCase<KqlQueryBuilder> {
+    @BeforeClass
+    protected static void ensureSnapshotBuild() {
+        assumeTrue("requires snapshot builds", Build.current().isSnapshot());
+    }
 
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {

From 21c4431d450e67096d8c8dc3f44f36e49b8ab0d9 Mon Sep 17 00:00:00 2001
From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com>
Date: Wed, 20 Nov 2024 17:23:02 +0200
Subject: [PATCH 08/14] Unmute org.elasticsearch.upgrades.IndexingIT (#117155)

---
 muted-tests.yml | 36 +-----------------------------------
 1 file changed, 1 insertion(+), 35 deletions(-)

diff --git a/muted-tests.yml b/muted-tests.yml
index a7d8995253dd8..2216873a3a265 100644
--- a/muted-tests.yml
+++ b/muted-tests.yml
@@ -246,41 +246,7 @@ tests:
 - class: org.elasticsearch.xpack.apmdata.APMYamlTestSuiteIT
   method: test {yaml=/10_apm/Test template reinstallation}
   issue: https://github.com/elastic/elasticsearch/issues/116445
-- class: org.elasticsearch.upgrades.DownsampleIT
-  method: testRollupIndex {upgradedNodes=3}
-  issue: https://github.com/elastic/elasticsearch/issues/117122
-- class: org.elasticsearch.upgrades.DownsampleIT
-  method: testRollupIndex {upgradedNodes=1}
-  issue: https://github.com/elastic/elasticsearch/issues/117123
-- class: org.elasticsearch.upgrades.DownsampleIT
-  method: testRollupIndex {upgradedNodes=2}
-  issue: https://github.com/elastic/elasticsearch/issues/117124
-- class: org.elasticsearch.upgrades.IndexingIT
-  method: testAutoIdWithOpTypeCreate {upgradedNodes=1}
-  issue: https://github.com/elastic/elasticsearch/issues/117125
-- class: org.elasticsearch.upgrades.IndexingIT
-  method: testTsdb {upgradedNodes=1}
-  issue: https://github.com/elastic/elasticsearch/issues/117126
-- class: org.elasticsearch.upgrades.IndexingIT
-  method: testIndexing {upgradedNodes=1}
-  issue: https://github.com/elastic/elasticsearch/issues/117127
-- class: org.elasticsearch.upgrades.IndexingIT
-  method: testSyntheticSource {upgradedNodes=1}
-  issue: https://github.com/elastic/elasticsearch/issues/117128
-- class: org.elasticsearch.upgrades.IndexingIT
-  method: testIndexing {upgradedNodes=3}
-  issue: https://github.com/elastic/elasticsearch/issues/117135
-- class: org.elasticsearch.upgrades.IndexingIT
-  method: testTsdb {upgradedNodes=3}
-  issue: https://github.com/elastic/elasticsearch/issues/117136
-- class: org.elasticsearch.upgrades.IndexingIT
-  method: testIndexing {upgradedNodes=2}
-  issue: https://github.com/elastic/elasticsearch/issues/117137
-- class: org.elasticsearch.upgrades.IndexingIT
-  method: testTsdb {upgradedNodes=2}
-  issue: https://github.com/elastic/elasticsearch/issues/117138
-- class: org.elasticsearch.upgrades.IndexingIT
-  issue: https://github.com/elastic/elasticsearch/issues/117140
+
 
 # Examples:
 #

From e5209f9b22b2dd4ea9504e26a3d2173141b494a1 Mon Sep 17 00:00:00 2001
From: Tim Grein <tim.grein@elastic.co>
Date: Wed, 20 Nov 2024 16:50:07 +0100
Subject: [PATCH 09/14] [Inference API] Expand RateLimiter docs (#117156)

---
 .../xpack/inference/common/RateLimiter.java         | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/RateLimiter.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/RateLimiter.java
index bbc5082d45004..b74e473155aec 100644
--- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/RateLimiter.java
+++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/common/RateLimiter.java
@@ -28,6 +28,14 @@
  *
  * By setting the accumulated tokens limit to a value greater than zero, it effectively allows bursts of traffic. If the accumulated
  * tokens limit is set to zero, it will force the acquiring thread to wait on each call.
+ *
+ * Example:
+ * Time unit: Second
+ * Tokens to produce per time unit: 10
+ * Limit for tokens in bucket: 100
+ *
+ * Tokens in bucket after n seconds (n second -> tokens in bucket):
+ * 1 sec -> 10 tokens, 2 sec -> 20 tokens, ... , 10 sec -> 100 tokens (bucket full), ... 200 sec -> 100 tokens (no increase in tokens)
  */
 public class RateLimiter {
 
@@ -76,6 +84,7 @@ public final synchronized void setRate(double newAccumulatedTokensLimit, double
             throw new IllegalArgumentException(Strings.format("Tokens per time unit must be less than or equal to %s", Double.MAX_VALUE));
         }
 
+        // If the new token limit is smaller than what we've accumulated already we need to drop tokens to meet the new token limit
         accumulatedTokens = Math.min(accumulatedTokens, newAccumulatedTokensLimit);
 
         accumulatedTokensLimit = newAccumulatedTokensLimit;
@@ -88,7 +97,8 @@ public final synchronized void setRate(double newAccumulatedTokensLimit, double
     }
 
     /**
-     * Causes the thread to wait until the tokens are available
+     * Causes the thread to wait until the tokens are available.
+     * This reserves token in advance leading to a reduction of accumulated tokens.
      * @param tokens the number of items of work that should be throttled, typically you'd pass a value of 1 here
      * @throws InterruptedException _
      */
@@ -130,6 +140,7 @@ private static void validateTokenRequest(int tokens) {
 
     /**
      * Returns the amount of time to wait for the tokens to become available.
+     * This reserves tokens in advance leading to a reduction of accumulated tokens.
      * @param tokens the number of items of work that should be throttled, typically you'd pass a value of 1 here. Must be greater than 0.
      * @return the amount of time to wait
      */

From c2c0901bbac39334c61e9677aa3ee4fb921ef549 Mon Sep 17 00:00:00 2001
From: Simon Cooper <simon.cooper@elastic.co>
Date: Wed, 20 Nov 2024 15:57:14 +0000
Subject: [PATCH 10/14] Remove desired node historical features (#116951)

---
 .../upgrades/DesiredNodesUpgradeIT.java       | 96 +------------------
 .../TransportUpdateDesiredNodesAction.java    | 17 ----
 .../UpdateDesiredNodesRequest.java            |  7 --
 .../cluster/metadata/DesiredNode.java         |  6 --
 .../cluster/metadata/MetadataFeatures.java    | 12 ---
 .../cluster/metadata/DesiredNodeTests.java    | 32 -------
 ...toricalFeaturesMetadataExtractorTests.java |  3 +-
 7 files changed, 5 insertions(+), 168 deletions(-)

diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java
index eb01d67432fe3..d9adec47ff483 100644
--- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java
+++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/DesiredNodesUpgradeIT.java
@@ -14,7 +14,6 @@
 import org.elasticsearch.Build;
 import org.elasticsearch.action.admin.cluster.desirednodes.UpdateDesiredNodesRequest;
 import org.elasticsearch.client.Request;
-import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.cluster.metadata.DesiredNode;
 import org.elasticsearch.cluster.metadata.DesiredNodeWithStatus;
 import org.elasticsearch.common.Strings;
@@ -42,22 +41,7 @@ public DesiredNodesUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
         desiredNodesVersion = upgradedNodes + 1;
     }
 
-    private enum ProcessorsPrecision {
-        DOUBLE,
-        FLOAT
-    }
-
     public void testUpgradeDesiredNodes() throws Exception {
-        if (oldClusterHasFeature(DesiredNode.DOUBLE_PROCESSORS_SUPPORTED)) {
-            assertUpgradedNodesCanReadDesiredNodes();
-        } else if (oldClusterHasFeature(DesiredNode.RANGE_FLOAT_PROCESSORS_SUPPORTED)) {
-            assertDesiredNodesUpdatedWithRoundedUpFloatsAreIdempotent();
-        } else {
-            assertDesiredNodesWithFloatProcessorsAreRejectedInOlderVersions();
-        }
-    }
-
-    private void assertUpgradedNodesCanReadDesiredNodes() throws Exception {
         if (isMixedCluster() || isUpgradedCluster()) {
             final Map<String, Object> desiredNodes = getLatestDesiredNodes();
             final String historyId = extractValue(desiredNodes, "history_id");
@@ -66,60 +50,10 @@ private void assertUpgradedNodesCanReadDesiredNodes() throws Exception {
             assertThat(version, is(equalTo(desiredNodesVersion - 1)));
         }
 
-        addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(desiredNodesVersion, ProcessorsPrecision.DOUBLE);
+        addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(desiredNodesVersion);
         assertAllDesiredNodesAreActualized();
     }
 
-    private void assertDesiredNodesUpdatedWithRoundedUpFloatsAreIdempotent() throws Exception {
-        // We define the same set of desired nodes to ensure that they are equal across all
-        // the test runs, otherwise we cannot guarantee an idempotent update in this test
-        final var desiredNodes = getNodeNames().stream()
-            .map(
-                nodeName -> new DesiredNode(
-                    Settings.builder().put(NODE_NAME_SETTING.getKey(), nodeName).build(),
-                    1238.49922909,
-                    ByteSizeValue.ofGb(32),
-                    ByteSizeValue.ofGb(128),
-                    clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version()
-                )
-            )
-            .toList();
-
-        if (isMixedCluster()) {
-            updateDesiredNodes(desiredNodes, desiredNodesVersion - 1);
-        }
-        for (int i = 0; i < 2; i++) {
-            updateDesiredNodes(desiredNodes, desiredNodesVersion);
-        }
-
-        final Map<String, Object> latestDesiredNodes = getLatestDesiredNodes();
-        final int latestDesiredNodesVersion = extractValue(latestDesiredNodes, "version");
-        assertThat(latestDesiredNodesVersion, is(equalTo(desiredNodesVersion)));
-
-        if (isUpgradedCluster()) {
-            assertAllDesiredNodesAreActualized();
-        }
-    }
-
-    private void assertDesiredNodesWithFloatProcessorsAreRejectedInOlderVersions() throws Exception {
-        if (isOldCluster()) {
-            addClusterNodesToDesiredNodesWithIntegerProcessors(1);
-        } else if (isMixedCluster()) {
-            // Processor ranges or float processors are forbidden during upgrades: 8.2 -> 8.3 clusters
-            final var responseException = expectThrows(
-                ResponseException.class,
-                () -> addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(desiredNodesVersion, ProcessorsPrecision.FLOAT)
-            );
-            final var statusCode = responseException.getResponse().getStatusLine().getStatusCode();
-            assertThat(statusCode, is(equalTo(400)));
-        } else {
-            assertAllDesiredNodesAreActualized();
-            addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(4, ProcessorsPrecision.FLOAT);
-        }
-
-        getLatestDesiredNodes();
-    }
-
     private Map<String, Object> getLatestDesiredNodes() throws IOException {
         final var getDesiredNodesRequest = new Request("GET", "/_internal/desired_nodes/_latest");
         final var response = client().performRequest(getDesiredNodesRequest);
@@ -140,15 +74,14 @@ private void assertAllDesiredNodesAreActualized() throws Exception {
         }
     }
 
-    private void addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(int version, ProcessorsPrecision processorsPrecision)
-        throws Exception {
+    private void addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(int version) throws Exception {
         final List<DesiredNode> nodes;
         if (randomBoolean()) {
             nodes = getNodeNames().stream()
                 .map(
                     nodeName -> new DesiredNode(
                         Settings.builder().put(NODE_NAME_SETTING.getKey(), nodeName).build(),
-                        processorsPrecision == ProcessorsPrecision.DOUBLE ? randomDoubleProcessorCount() : 0.5f,
+                        randomDoubleProcessorCount(),
                         ByteSizeValue.ofGb(randomIntBetween(10, 24)),
                         ByteSizeValue.ofGb(randomIntBetween(128, 256)),
                         clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version()
@@ -157,9 +90,7 @@ private void addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(int ve
                 .toList();
         } else {
             nodes = getNodeNames().stream().map(nodeName -> {
-                double minProcessors = processorsPrecision == ProcessorsPrecision.DOUBLE
-                    ? randomDoubleProcessorCount()
-                    : randomFloatProcessorCount();
+                double minProcessors = randomDoubleProcessorCount();
                 return new DesiredNode(
                     Settings.builder().put(NODE_NAME_SETTING.getKey(), nodeName).build(),
                     new DesiredNode.ProcessorsRange(minProcessors, minProcessors + randomIntBetween(10, 20)),
@@ -172,21 +103,6 @@ private void addClusterNodesToDesiredNodesWithProcessorsOrProcessorRanges(int ve
         updateDesiredNodes(nodes, version);
     }
 
-    private void addClusterNodesToDesiredNodesWithIntegerProcessors(int version) throws Exception {
-        final var nodes = getNodeNames().stream()
-            .map(
-                nodeName -> new DesiredNode(
-                    Settings.builder().put(NODE_NAME_SETTING.getKey(), nodeName).build(),
-                    randomIntBetween(1, 24),
-                    ByteSizeValue.ofGb(randomIntBetween(10, 24)),
-                    ByteSizeValue.ofGb(randomIntBetween(128, 256)),
-                    clusterHasFeature(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED) ? null : Build.current().version()
-                )
-            )
-            .toList();
-        updateDesiredNodes(nodes, version);
-    }
-
     private void updateDesiredNodes(List<DesiredNode> nodes, int version) throws IOException {
         final var request = new Request("PUT", "/_internal/desired_nodes/upgrade_test/" + version);
         try (var builder = JsonXContent.contentBuilder()) {
@@ -223,10 +139,6 @@ private double randomDoubleProcessorCount() {
         return randomDoubleBetween(0.5, 512.1234, true);
     }
 
-    private float randomFloatProcessorCount() {
-        return randomIntBetween(1, 512) + randomFloat();
-    }
-
     @SuppressWarnings("unchecked")
     private static <T> T extractValue(Map<String, Object> map, String path) {
         return (T) XContentMapValues.extractValue(path, map);
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java
index bed2815f5a895..a0948af88e2f5 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/TransportUpdateDesiredNodesAction.java
@@ -20,7 +20,6 @@
 import org.elasticsearch.cluster.block.ClusterBlockException;
 import org.elasticsearch.cluster.block.ClusterBlockLevel;
 import org.elasticsearch.cluster.desirednodes.VersionConflictException;
-import org.elasticsearch.cluster.metadata.DesiredNode;
 import org.elasticsearch.cluster.metadata.DesiredNodes;
 import org.elasticsearch.cluster.metadata.DesiredNodesMetadata;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
@@ -99,22 +98,6 @@ protected void masterOperation(
         );
     }
 
-    @Override
-    protected void doExecute(Task task, UpdateDesiredNodesRequest request, ActionListener<UpdateDesiredNodesResponse> listener) {
-        if (request.clusterHasRequiredFeatures(nf -> featureService.clusterHasFeature(clusterService.state(), nf)) == false) {
-            listener.onFailure(
-                new IllegalArgumentException(
-                    "Unable to use processor ranges, floating-point (with greater precision) processors "
-                        + "in mixed-clusters with nodes that do not support feature "
-                        + DesiredNode.RANGE_FLOAT_PROCESSORS_SUPPORTED.id()
-                )
-            );
-            return;
-        }
-
-        super.doExecute(task, request, listener);
-    }
-
     static ClusterState replaceDesiredNodes(ClusterState clusterState, DesiredNodes newDesiredNodes) {
         return clusterState.copyAndUpdateMetadata(
             metadata -> metadata.putCustom(DesiredNodesMetadata.TYPE, new DesiredNodesMetadata(newDesiredNodes))
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequest.java
index a94401fdd66f3..21b714b105b59 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequest.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/desirednodes/UpdateDesiredNodesRequest.java
@@ -18,7 +18,6 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.core.TimeValue;
-import org.elasticsearch.features.NodeFeature;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.XContentParser;
@@ -26,7 +25,6 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
-import java.util.function.Predicate;
 
 public class UpdateDesiredNodesRequest extends AcknowledgedRequest<UpdateDesiredNodesRequest> {
     private static final TransportVersion DRY_RUN_VERSION = TransportVersions.V_8_4_0;
@@ -117,11 +115,6 @@ public boolean isDryRun() {
         return dryRun;
     }
 
-    public boolean clusterHasRequiredFeatures(Predicate<NodeFeature> clusterHasFeature) {
-        return clusterHasFeature.test(DesiredNode.RANGE_FLOAT_PROCESSORS_SUPPORTED)
-            || nodes.stream().allMatch(n -> n.clusterHasRequiredFeatures(clusterHasFeature));
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java
index fb8559b19d81d..de3343c1944c1 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DesiredNode.java
@@ -37,7 +37,6 @@
 import java.util.Objects;
 import java.util.Set;
 import java.util.TreeSet;
-import java.util.function.Predicate;
 import java.util.regex.Pattern;
 
 import static java.lang.String.format;
@@ -48,7 +47,6 @@
 public final class DesiredNode implements Writeable, ToXContentObject, Comparable<DesiredNode> {
 
     public static final NodeFeature RANGE_FLOAT_PROCESSORS_SUPPORTED = new NodeFeature("desired_node.range_float_processors");
-    public static final NodeFeature DOUBLE_PROCESSORS_SUPPORTED = new NodeFeature("desired_node.double_processors");
     public static final NodeFeature DESIRED_NODE_VERSION_DEPRECATED = new NodeFeature("desired_node.version_deprecated");
 
     public static final TransportVersion RANGE_FLOAT_PROCESSORS_SUPPORT_TRANSPORT_VERSION = TransportVersions.V_8_3_0;
@@ -348,10 +346,6 @@ public Set<DiscoveryNodeRole> getRoles() {
         return roles;
     }
 
-    public boolean clusterHasRequiredFeatures(Predicate<NodeFeature> clusterHasFeature) {
-        return (processorsRange == null && processors.hasDecimals() == false) || clusterHasFeature.test(RANGE_FLOAT_PROCESSORS_SUPPORTED);
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataFeatures.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataFeatures.java
index 89f776a7ada0f..49bd38330e3af 100644
--- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataFeatures.java
+++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataFeatures.java
@@ -9,24 +9,12 @@
 
 package org.elasticsearch.cluster.metadata;
 
-import org.elasticsearch.Version;
 import org.elasticsearch.features.FeatureSpecification;
 import org.elasticsearch.features.NodeFeature;
 
-import java.util.Map;
 import java.util.Set;
 
 public class MetadataFeatures implements FeatureSpecification {
-    @Override
-    public Map<NodeFeature, Version> getHistoricalFeatures() {
-        return Map.of(
-            DesiredNode.RANGE_FLOAT_PROCESSORS_SUPPORTED,
-            Version.V_8_3_0,
-            DesiredNode.DOUBLE_PROCESSORS_SUPPORTED,
-            Version.V_8_5_0
-        );
-    }
-
     @Override
     public Set<NodeFeature> getFeatures() {
         return Set.of(DesiredNode.DESIRED_NODE_VERSION_DEPRECATED);
diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeTests.java
index 41651d52ceb9f..0e4b8271ceac7 100644
--- a/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeTests.java
+++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DesiredNodeTests.java
@@ -185,38 +185,6 @@ public void testNodeCPUsRoundUp() {
         }
     }
 
-    public void testDesiredNodeHasRangeFloatProcessors() {
-        final var settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), randomAlphaOfLength(10)).build();
-
-        {
-            final var desiredNode = new DesiredNode(
-                settings,
-                new DesiredNode.ProcessorsRange(0.4, 1.2),
-                ByteSizeValue.ofGb(1),
-                ByteSizeValue.ofGb(1)
-            );
-            assertThat(desiredNode.clusterHasRequiredFeatures(DesiredNode.RANGE_FLOAT_PROCESSORS_SUPPORTED::equals), is(true));
-            assertThat(desiredNode.clusterHasRequiredFeatures(nf -> false), is(false));
-        }
-
-        {
-            final var desiredNode = new DesiredNode(
-                settings,
-                randomIntBetween(0, 10) + randomDoubleBetween(0.00001, 0.99999, true),
-                ByteSizeValue.ofGb(1),
-                ByteSizeValue.ofGb(1)
-            );
-            assertThat(desiredNode.clusterHasRequiredFeatures(DesiredNode.RANGE_FLOAT_PROCESSORS_SUPPORTED::equals), is(true));
-            assertThat(desiredNode.clusterHasRequiredFeatures(nf -> false), is(false));
-        }
-
-        {
-            final var desiredNode = new DesiredNode(settings, 2.0f, ByteSizeValue.ofGb(1), ByteSizeValue.ofGb(1));
-            assertThat(desiredNode.clusterHasRequiredFeatures(DesiredNode.RANGE_FLOAT_PROCESSORS_SUPPORTED::equals), is(true));
-            assertThat(desiredNode.clusterHasRequiredFeatures(nf -> false), is(true));
-        }
-    }
-
     public void testEqualsOrProcessorsCloseTo() {
         final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), randomAlphaOfLength(10)).build();
         final double maxDelta = 1E-3;
diff --git a/test/metadata-extractor/src/test/java/org/elasticsearch/extractor/features/HistoricalFeaturesMetadataExtractorTests.java b/test/metadata-extractor/src/test/java/org/elasticsearch/extractor/features/HistoricalFeaturesMetadataExtractorTests.java
index e230982073699..d810f17ae552e 100644
--- a/test/metadata-extractor/src/test/java/org/elasticsearch/extractor/features/HistoricalFeaturesMetadataExtractorTests.java
+++ b/test/metadata-extractor/src/test/java/org/elasticsearch/extractor/features/HistoricalFeaturesMetadataExtractorTests.java
@@ -27,7 +27,6 @@
 import java.util.Set;
 
 import static org.elasticsearch.xcontent.XContentParserConfiguration.EMPTY;
-import static org.hamcrest.Matchers.anEmptyMap;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.hasEntry;
@@ -48,7 +47,7 @@ public void testExtractHistoricalMetadata() throws IOException {
             nodeFeatureVersionMap.putAll(historical);
             featureNamesSet.addAll(names);
         });
-        assertThat(nodeFeatureVersionMap, not(anEmptyMap()));
+        // assertThat(nodeFeatureVersionMap, not(anEmptyMap()));
         assertThat(featureNamesSet, not(empty()));
         assertThat(featureNamesSet, hasItem("test_features_enabled"));
 

From 9477bd691aeee7518a7ec7557c391e3bca5c5cad Mon Sep 17 00:00:00 2001
From: Stanislav Malyshev <smalyshev@users.noreply.github.com>
Date: Wed, 20 Nov 2024 09:15:12 -0700
Subject: [PATCH 11/14] Fix long metric deserialize & add - auto-resize needs
 to be set manually (#117105)

* Fix long metric deserialize & add - auto-resize needs to be set manually
---
 docs/changelog/117105.yaml                    |  6 +++++
 .../admin/cluster/stats/LongMetric.java       |  1 +
 .../stats/CCSTelemetrySnapshotTests.java      | 24 ++++++++++++++++++-
 3 files changed, 30 insertions(+), 1 deletion(-)
 create mode 100644 docs/changelog/117105.yaml

diff --git a/docs/changelog/117105.yaml b/docs/changelog/117105.yaml
new file mode 100644
index 0000000000000..de56c4d521a62
--- /dev/null
+++ b/docs/changelog/117105.yaml
@@ -0,0 +1,6 @@
+pr: 117105
+summary: Fix long metric deserialize & add - auto-resize needs to be set manually
+area: CCS
+type: bug
+issues:
+ - 116914
diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java
index 737e83d4b30a1..07d9c11ae4c07 100644
--- a/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java
+++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/stats/LongMetric.java
@@ -74,6 +74,7 @@ public static LongMetricValue fromStream(StreamInput in) throws IOException {
             try {
                 // TODO: not sure what is the good value for minBarForHighestToLowestValueRatio here?
                 Histogram dh = Histogram.decodeFromCompressedByteBuffer(bb, 1);
+                dh.setAutoResize(true);
                 return new LongMetricValue(dh);
             } catch (DataFormatException e) {
                 throw new IOException(e);
diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java
index e9188d9cb8f0d..a72630c327ea2 100644
--- a/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java
+++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/stats/CCSTelemetrySnapshotTests.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.action.admin.cluster.stats.CCSTelemetrySnapshot.PerClusterCCSTelemetry;
 import org.elasticsearch.action.admin.cluster.stats.LongMetric.LongMetricValue;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.io.stream.BytesStreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Tuple;
@@ -32,9 +33,13 @@
 public class CCSTelemetrySnapshotTests extends AbstractWireSerializingTestCase<CCSTelemetrySnapshot> {
 
     private LongMetricValue randomLongMetricValue() {
+        return randomLongMetricValueBetween(0, 1_000_000);
+    }
+
+    private LongMetricValue randomLongMetricValueBetween(int low, int high) {
         LongMetric v = new LongMetric();
         for (int i = 0; i < randomIntBetween(5, 10); i++) {
-            v.record(randomIntBetween(0, 1_000_000));
+            v.record(randomIntBetween(low, high));
         }
         return v.getValue();
     }
@@ -330,4 +335,21 @@ private String readJSONFromResource(String fileName) throws IOException {
             return new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
         }
     }
+
+    public void testRanges() throws IOException {
+        var value1 = randomLongMetricValueBetween(1_000_000, 10_000_000);
+        var count1 = value1.count();
+        var max1 = value1.max();
+        var output = new BytesStreamOutput();
+        value1.writeTo(output);
+        var value1Read = LongMetricValue.fromStream(output.bytes().streamInput());
+        var value2 = randomLongMetricValueBetween(0, 100);
+        var count2 = value2.count();
+        output = new BytesStreamOutput();
+        value2.writeTo(output);
+        var value2Read = LongMetricValue.fromStream(output.bytes().streamInput());
+        value2Read.add(value1Read);
+        assertThat(value2Read.count(), equalTo(count1 + count2));
+        assertThat(value2Read.max(), equalTo(max1));
+    }
 }

From ccdc5627788123860fbfe9b812e0f6db0c27e0da Mon Sep 17 00:00:00 2001
From: Sam Xiao <sam.xiao@elastic.co>
Date: Wed, 20 Nov 2024 11:30:52 -0500
Subject: [PATCH 12/14] Azure Fixture: Add additional batch delete path
 (#116985)

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
---
 .../src/main/java/fixture/azure/AzureHttpHandler.java     | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
index 92ce04b6bea5b..bbcfe1f75dc06 100644
--- a/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
+++ b/test/fixtures/azure-fixture/src/main/java/fixture/azure/AzureHttpHandler.java
@@ -355,6 +355,14 @@ public void handle(final HttpExchange exchange) throws IOException {
                                 throw new IllegalStateException("Got multiple deletes in a single request?");
                             }
                             toDelete = blobName;
+                        } else if (Regex.simpleMatch("DELETE /" + account + "/" + container + "/*", line)) {
+                            // possible alternative DELETE url, depending on which method is used in the batch client
+                            String path = RestUtils.decodeComponent(line.split("(\\s|\\?)")[1]);
+                            String blobName = path.split(account)[1];
+                            if (toDelete != null) {
+                                throw new IllegalStateException("Got multiple deletes in a single request?");
+                            }
+                            toDelete = blobName;
                         }
                     }
                     response.append("--").append(responseBoundary).append("--\r\n0\r\n");

From 312f8315d35a640ed43673de75dad61b65481391 Mon Sep 17 00:00:00 2001
From: Nikolaj Volgushev <n1v0lg@users.noreply.github.com>
Date: Wed, 20 Nov 2024 17:43:01 +0100
Subject: [PATCH 13/14] Longer RCS suite timeout due to slow keystore (#117157)

Rather than muting the suite and losing signal, bump the suite timeout to account for very slow keystore operations.

We should follow this up with performance improvements around keystore setup in tests.

Closes: https://github.com/elastic/elasticsearch/issues/116883
---
 .../RemoteClusterSecurityReloadCredentialsRestIT.java           | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityReloadCredentialsRestIT.java b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityReloadCredentialsRestIT.java
index 42982e6183613..fb941e9e815cf 100644
--- a/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityReloadCredentialsRestIT.java
+++ b/x-pack/plugin/security/qa/multi-cluster/src/javaRestTest/java/org/elasticsearch/xpack/remotecluster/RemoteClusterSecurityReloadCredentialsRestIT.java
@@ -42,7 +42,7 @@
 import static org.hamcrest.Matchers.nullValue;
 
 // account for slow stored secure settings updates (involves removing and re-creating the keystore)
-@TimeoutSuite(millis = 10 * TimeUnits.MINUTE)
+@TimeoutSuite(millis = 20 * TimeUnits.MINUTE)
 public class RemoteClusterSecurityReloadCredentialsRestIT extends AbstractRemoteClusterSecurityTestCase {
 
     private static final MutableSettingsProvider keystoreSettings = new MutableSettingsProvider();

From fe7818af04b564d222faa1c8ff166d01e59a9671 Mon Sep 17 00:00:00 2001
From: Nhat Nguyen <nhat.nguyen@elastic.co>
Date: Wed, 20 Nov 2024 10:27:44 -0800
Subject: [PATCH 14/14] Deprecate _source.mode in mappings (#117172)

Re-introduce #116689
---
 .../compat/RestCompatTestTransformTask.java   |  2 +-
 docs/changelog/116689.yaml                    | 10 ++
 .../LogsIndexModeFullClusterRestartIT.java    | 16 +---
 .../LogsIndexModeRollingUpgradeIT.java        | 20 +---
 rest-api-spec/build.gradle                    |  6 ++
 .../rest-api-spec/test/logsdb/10_settings.yml |  5 -
 .../test/logsdb/20_source_mapping.yml         | 15 ++-
 .../rest-api-spec/test/tsdb/20_mapping.yml    | 11 ---
 .../index/IndexSettingProvider.java           |  3 +-
 .../elasticsearch/index/IndexVersions.java    |  1 +
 .../index/mapper/SourceFieldMapper.java       | 92 ++++++++++++-------
 .../elasticsearch/node/NodeConstruction.java  |  2 +-
 .../mapper/DynamicFieldsBuilderTests.java     |  2 +-
 .../index/mapper/SourceFieldMapperTests.java  |  4 +-
 .../query/SearchExecutionContextTests.java    |  2 +-
 .../test/rest/ESRestTestCase.java             | 35 ++++++-
 .../test/rest/yaml/section/DoSection.java     |  3 +
 .../xpack/ccr/FollowIndexIT.java              |  8 +-
 .../esql/qa/rest/FieldExtractorTestCase.java  |  7 +-
 .../xpack/logsdb/LogsDBPlugin.java            |  9 +-
 .../SyntheticSourceIndexSettingsProvider.java |  8 +-
 ...heticSourceIndexSettingsProviderTests.java |  6 +-
 .../test/40_source_mode_setting.yml           | 29 +-----
 23 files changed, 162 insertions(+), 134 deletions(-)
 create mode 100644 docs/changelog/116689.yaml

diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java
index ef93dafa913cd..ba242a8e23861 100644
--- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java
+++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/test/rest/compat/compat/RestCompatTestTransformTask.java
@@ -137,7 +137,7 @@ public void skipTest(String fullTestName, String reason) {
         // However, the folder can be arbitrarily nest so, a == a1/a2/a3, and the test name can include forward slashes, so c == c1/c2/c3
         // So we also need to support a1/a2/a3/b/c1/c2/c3
 
-        String[] testParts = fullTestName.split("/");
+        String[] testParts = fullTestName.split("/", 3);
         if (testParts.length < 3) {
             throw new IllegalArgumentException(
                 "To skip tests, all 3 parts [folder/file/test name] must be defined. found [" + fullTestName + "]"
diff --git a/docs/changelog/116689.yaml b/docs/changelog/116689.yaml
new file mode 100644
index 0000000000000..0b1d1646868aa
--- /dev/null
+++ b/docs/changelog/116689.yaml
@@ -0,0 +1,10 @@
+pr: 116689
+summary: Deprecate `_source.mode` in mappings
+area: Mapping
+type: deprecation
+issues: []
+deprecation:
+  title: Deprecate `_source.mode` in mappings
+  area: Mapping
+  details: Configuring `_source.mode` in mappings is deprecated and will be removed in future versions. Use `index.mapping.source.mode` index setting instead.
+  impact: Use `index.mapping.source.mode` index setting instead
diff --git a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java
index 3459a29e98649..9866d94dccc3c 100644
--- a/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java
+++ b/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeFullClusterRestartIT.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.FormatNames;
-import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.hamcrest.Matcher;
@@ -30,9 +29,6 @@
 import java.util.Map;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.test.MapMatcher.assertMap;
-import static org.elasticsearch.test.MapMatcher.matchesMap;
-
 public class LogsIndexModeFullClusterRestartIT extends ParameterizedFullClusterRestartTestCase {
 
     @ClassRule
@@ -169,22 +165,16 @@ public void testLogsIndexing() throws IOException {
             assertOK(bulkIndexResponse);
             assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false));
 
-            assertIndexMappingsAndSettings(0, Matchers.nullValue(), matchesMap().extraOk());
-            assertIndexMappingsAndSettings(
-                1,
-                Matchers.equalTo("logsdb"),
-                matchesMap().extraOk().entry("_source", Map.of("mode", "synthetic"))
-            );
+            assertIndexSettings(0, Matchers.nullValue());
+            assertIndexSettings(1, Matchers.equalTo("logsdb"));
         }
     }
 
-    private void assertIndexMappingsAndSettings(int backingIndex, final Matcher<Object> indexModeMatcher, final MapMatcher mappingsMatcher)
-        throws IOException {
+    private void assertIndexSettings(int backingIndex, final Matcher<Object> indexModeMatcher) throws IOException {
         assertThat(
             getSettings(client(), getWriteBackingIndex(client(), "logs-apache-production", backingIndex)).get("index.mode"),
             indexModeMatcher
         );
-        assertMap(getIndexMappingAsMap(getWriteBackingIndex(client(), "logs-apache-production", backingIndex)), mappingsMatcher);
     }
 
     private static Request createDataStream(final String dataStreamName) {
diff --git a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java
index 8c369ebc9950d..1eb7cbd3f70c2 100644
--- a/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java
+++ b/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/LogsIndexModeRollingUpgradeIT.java
@@ -17,7 +17,6 @@
 import org.elasticsearch.common.network.InetAddresses;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.time.FormatNames;
-import org.elasticsearch.test.MapMatcher;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.hamcrest.Matcher;
@@ -30,9 +29,6 @@
 import java.util.Map;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.test.MapMatcher.assertMap;
-import static org.elasticsearch.test.MapMatcher.matchesMap;
-
 public class LogsIndexModeRollingUpgradeIT extends AbstractRollingUpgradeTestCase {
 
     @ClassRule()
@@ -160,14 +156,10 @@ public void testLogsIndexing() throws IOException {
             assertOK(bulkIndexResponse);
             assertThat(entityAsMap(bulkIndexResponse).get("errors"), Matchers.is(false));
 
-            assertIndexMappingsAndSettings(0, Matchers.nullValue(), matchesMap().extraOk());
-            assertIndexMappingsAndSettings(1, Matchers.nullValue(), matchesMap().extraOk());
-            assertIndexMappingsAndSettings(2, Matchers.nullValue(), matchesMap().extraOk());
-            assertIndexMappingsAndSettings(
-                3,
-                Matchers.equalTo("logsdb"),
-                matchesMap().extraOk().entry("_source", Map.of("mode", "synthetic"))
-            );
+            assertIndexSettings(0, Matchers.nullValue());
+            assertIndexSettings(1, Matchers.nullValue());
+            assertIndexSettings(2, Matchers.nullValue());
+            assertIndexSettings(3, Matchers.equalTo("logsdb"));
         }
     }
 
@@ -183,13 +175,11 @@ static void enableLogsdbByDefault() throws IOException {
         assertOK(client().performRequest(request));
     }
 
-    private void assertIndexMappingsAndSettings(int backingIndex, final Matcher<Object> indexModeMatcher, final MapMatcher mappingsMatcher)
-        throws IOException {
+    private void assertIndexSettings(int backingIndex, final Matcher<Object> indexModeMatcher) throws IOException {
         assertThat(
             getSettings(client(), getWriteBackingIndex(client(), "logs-apache-production", backingIndex)).get("index.mode"),
             indexModeMatcher
         );
-        assertMap(getIndexMappingAsMap(getWriteBackingIndex(client(), "logs-apache-production", backingIndex)), mappingsMatcher);
     }
 
     private static Request createDataStream(final String dataStreamName) {
diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle
index 439960228cef6..650d17e41de7f 100644
--- a/rest-api-spec/build.gradle
+++ b/rest-api-spec/build.gradle
@@ -60,4 +60,10 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task ->
   task.skipTest("cat.aliases/10_basic/Deprecated local parameter", "CAT APIs not covered by compatibility policy")
   task.skipTest("cat.shards/10_basic/Help", "sync_id is removed in 9.0")
   task.skipTest("search/500_date_range/from, to, include_lower, include_upper deprecated", "deprecated parameters are removed in 9.0")
+  task.skipTest("tsdb/20_mapping/stored source is supported", "no longer serialize source_mode")
+  task.skipTest("tsdb/20_mapping/Synthetic source", "no longer serialize source_mode")
+  task.skipTest("logsdb/10_settings/create logs index", "no longer serialize source_mode")
+  task.skipTest("logsdb/20_source_mapping/stored _source mode is supported", "no longer serialize source_mode")
+  task.skipTest("logsdb/20_source_mapping/include/exclude is supported with stored _source", "no longer serialize source_mode")
+  task.skipTest("logsdb/20_source_mapping/synthetic _source is default", "no longer serialize source_mode")
 })
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
index d0f89b1b8b6cb..463df7d2ab1bb 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/10_settings.yml
@@ -76,11 +76,6 @@ create logs index:
   - is_true: test
   - match: { test.settings.index.mode: "logsdb" }
 
-  - do:
-      indices.get_mapping:
-        index: test
-  - match: { test.mappings._source.mode: synthetic }
-
 ---
 using default timestamp field mapping:
   - requires:
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml
index 27146557bb1be..06a007b8aaca5 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/logsdb/20_source_mapping.yml
@@ -13,10 +13,10 @@ synthetic _source is default:
             index:
               mode: logsdb
   - do:
-      indices.get:
+      indices.get_settings:
         index: test-default-source
-
-  - match: { test-default-source.mappings._source.mode: "synthetic" }
+  - match: { test-default-source.settings.index.mode: logsdb }
+  - match: { test-default-source.settings.index.mapping.source.mode: null }
 
 ---
 stored _source mode is supported:
@@ -28,11 +28,12 @@ stored _source mode is supported:
             index:
               mode: logsdb
               mapping.source.mode: stored
+
   - do:
-      indices.get:
+      indices.get_settings:
         index: test-stored-source
-
-  - match: { test-stored-source.mappings._source.mode: "stored" }
+  - match: { test-stored-source.settings.index.mode: logsdb }
+  - match: { test-stored-source.settings.index.mapping.source.mode: stored }
 
 ---
 disabled _source is not supported:
@@ -110,7 +111,6 @@ include/exclude is supported with stored _source:
       indices.get:
         index: test-includes
 
-  - match: { test-includes.mappings._source.mode: "stored" }
   - match: { test-includes.mappings._source.includes: ["a"] }
 
   - do:
@@ -129,5 +129,4 @@ include/exclude is supported with stored _source:
       indices.get:
         index: test-excludes
 
-  - match: { test-excludes.mappings._source.mode: "stored" }
   - match: { test-excludes.mappings._source.excludes: ["b"] }
diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
index 4d8f03a6e5e18..9fe3f5e0b7272 100644
--- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
+++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/20_mapping.yml
@@ -450,11 +450,6 @@ nested fields:
                 type: long
                 time_series_metric: gauge
 
-  - do:
-      indices.get_mapping: {}
-
-  - match: {tsdb-synthetic.mappings._source.mode: synthetic}
-
 ---
 stored source is supported:
   - requires:
@@ -486,12 +481,6 @@ stored source is supported:
                         type: keyword
                         time_series_dimension: true
 
-  - do:
-      indices.get:
-        index: tsdb_index
-
-  - match: { tsdb_index.mappings._source.mode: "stored" }
-
 ---
 disabled source is not supported:
   - requires:
diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java
index 6a553d5dc5440..8c997a9766baa 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexSettingProvider.java
@@ -11,6 +11,7 @@
 
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.compress.CompressedXContent;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedFunction;
@@ -54,7 +55,7 @@ Settings getAdditionalIndexSettings(
     /**
      * Infrastructure class that holds services that can be used by {@link IndexSettingProvider} instances.
      */
-    record Parameters(CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory) {
+    record Parameters(ClusterService clusterService, CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory) {
 
     }
 
diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java
index 5746bea12a2d8..7a5f469a57fa1 100644
--- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java
+++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java
@@ -134,6 +134,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion UPGRADE_TO_LUCENE_10_0_0 = def(9_000_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion LOGSDB_DEFAULT_IGNORE_DYNAMIC_BEYOND_LIMIT = def(9_001_00_0, Version.LUCENE_10_0_0);
     public static final IndexVersion TIME_BASED_K_ORDERED_DOC_ID = def(9_002_00_0, Version.LUCENE_10_0_0);
+    public static final IndexVersion DEPRECATE_SOURCE_MODE_MAPPER = def(9_003_00_0, Version.LUCENE_10_0_0);
     /*
      * STOP! READ THIS FIRST! No, really,
      *        ____ _____ ___  ____  _        ____  _____    _    ____    _____ _   _ ___ ____    _____ ___ ____  ____ _____ _
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
index dd25cd6eb80a3..e5b12f748543f 100644
--- a/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
+++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceFieldMapper.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
@@ -38,6 +39,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 
 public class SourceFieldMapper extends MetadataFieldMapper {
     public static final NodeFeature SYNTHETIC_SOURCE_FALLBACK = new NodeFeature("mapper.source.synthetic_source_fallback");
@@ -68,6 +70,9 @@ public class SourceFieldMapper extends MetadataFieldMapper {
         return indexMode.defaultSourceMode().name();
     }, "index.mapping.source.mode", value -> {}, Setting.Property.Final, Setting.Property.IndexScope);
 
+    public static final String DEPRECATION_WARNING = "Configuring source mode in mappings is deprecated and will be removed "
+        + "in future versions. Use [index.mapping.source.mode] index setting instead.";
+
     /** The source mode */
     public enum Mode {
         DISABLED,
@@ -79,28 +84,32 @@ public enum Mode {
         null,
         Explicit.IMPLICIT_TRUE,
         Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY
+        Strings.EMPTY_ARRAY,
+        false
     );
 
     private static final SourceFieldMapper STORED = new SourceFieldMapper(
         Mode.STORED,
         Explicit.IMPLICIT_TRUE,
         Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY
+        Strings.EMPTY_ARRAY,
+        false
     );
 
     private static final SourceFieldMapper SYNTHETIC = new SourceFieldMapper(
         Mode.SYNTHETIC,
         Explicit.IMPLICIT_TRUE,
         Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY
+        Strings.EMPTY_ARRAY,
+        false
     );
 
     private static final SourceFieldMapper DISABLED = new SourceFieldMapper(
         Mode.DISABLED,
         Explicit.IMPLICIT_TRUE,
         Strings.EMPTY_ARRAY,
-        Strings.EMPTY_ARRAY
+        Strings.EMPTY_ARRAY,
+        false
     );
 
     public static class Defaults {
@@ -134,16 +143,7 @@ public static class Builder extends MetadataFieldMapper.Builder {
          * The default mode for TimeSeries is left empty on purpose, so that mapping printings include the synthetic
          * source mode.
          */
-        private final Parameter<Mode> mode = new Parameter<>(
-            "mode",
-            true,
-            () -> null,
-            (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)),
-            m -> toType(m).enabled.explicit() ? null : toType(m).mode,
-            (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)),
-            v -> v.toString().toLowerCase(Locale.ROOT)
-        ).setMergeValidator((previous, current, conflicts) -> (previous == current) || current != Mode.STORED)
-            .setSerializerCheck((includeDefaults, isConfigured, value) -> value != null); // don't emit if `enabled` is configured
+        private final Parameter<Mode> mode;
         private final Parameter<List<String>> includes = Parameter.stringArrayParam(
             "includes",
             false,
@@ -158,15 +158,28 @@ public static class Builder extends MetadataFieldMapper.Builder {
         private final Settings settings;
 
         private final IndexMode indexMode;
+        private boolean serializeMode;
 
         private final boolean supportsNonDefaultParameterValues;
 
-        public Builder(IndexMode indexMode, final Settings settings, boolean supportsCheckForNonDefaultParams) {
+        public Builder(IndexMode indexMode, final Settings settings, boolean supportsCheckForNonDefaultParams, boolean serializeMode) {
             super(Defaults.NAME);
             this.settings = settings;
             this.indexMode = indexMode;
             this.supportsNonDefaultParameterValues = supportsCheckForNonDefaultParams == false
                 || settings.getAsBoolean(LOSSY_PARAMETERS_ALLOWED_SETTING_NAME, true);
+            this.serializeMode = serializeMode;
+            this.mode = new Parameter<>(
+                "mode",
+                true,
+                () -> null,
+                (n, c, o) -> Mode.valueOf(o.toString().toUpperCase(Locale.ROOT)),
+                m -> toType(m).enabled.explicit() ? null : toType(m).mode,
+                (b, n, v) -> b.field(n, v.toString().toLowerCase(Locale.ROOT)),
+                v -> v.toString().toLowerCase(Locale.ROOT)
+            ).setMergeValidator((previous, current, conflicts) -> (previous == current) || current != Mode.STORED)
+                // don't emit if `enabled` is configured
+                .setSerializerCheck((includeDefaults, isConfigured, value) -> serializeMode && value != null);
         }
 
         public Builder setSynthetic() {
@@ -219,21 +232,22 @@ public SourceFieldMapper build() {
             if (sourceMode == Mode.SYNTHETIC && (includes.getValue().isEmpty() == false || excludes.getValue().isEmpty() == false)) {
                 throw new IllegalArgumentException("filtering the stored _source is incompatible with synthetic source");
             }
-
-            SourceFieldMapper sourceFieldMapper;
-            if (isDefault()) {
+            if (mode.isConfigured()) {
+                serializeMode = true;
+            }
+            final SourceFieldMapper sourceFieldMapper;
+            if (isDefault() && sourceMode == null) {
                 // Needed for bwc so that "mode" is not serialized in case of a standard index with stored source.
-                if (sourceMode == null) {
-                    sourceFieldMapper = DEFAULT;
-                } else {
-                    sourceFieldMapper = resolveStaticInstance(sourceMode);
-                }
+                sourceFieldMapper = DEFAULT;
+            } else if (isDefault() && serializeMode == false && sourceMode != null) {
+                sourceFieldMapper = resolveStaticInstance(sourceMode);
             } else {
                 sourceFieldMapper = new SourceFieldMapper(
                     sourceMode,
                     enabled.get(),
                     includes.getValue().toArray(Strings.EMPTY_ARRAY),
-                    excludes.getValue().toArray(Strings.EMPTY_ARRAY)
+                    excludes.getValue().toArray(Strings.EMPTY_ARRAY),
+                    serializeMode
                 );
             }
             if (indexMode != null) {
@@ -283,15 +297,29 @@ private static SourceFieldMapper resolveStaticInstance(final Mode sourceMode) {
         if (indexMode == IndexMode.STANDARD && settingSourceMode == Mode.STORED) {
             return DEFAULT;
         }
-
-        return resolveStaticInstance(settingSourceMode);
+        if (c.indexVersionCreated().onOrAfter(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER)) {
+            return resolveStaticInstance(settingSourceMode);
+        } else {
+            return new SourceFieldMapper(settingSourceMode, Explicit.IMPLICIT_TRUE, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, true);
+        }
     },
         c -> new Builder(
             c.getIndexSettings().getMode(),
             c.getSettings(),
-            c.indexVersionCreated().onOrAfter(IndexVersions.SOURCE_MAPPER_LOSSY_PARAMS_CHECK)
+            c.indexVersionCreated().onOrAfter(IndexVersions.SOURCE_MAPPER_LOSSY_PARAMS_CHECK),
+            c.indexVersionCreated().before(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER)
         )
-    );
+    ) {
+        @Override
+        public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, MappingParserContext parserContext)
+            throws MapperParsingException {
+            assert name.equals(SourceFieldMapper.NAME) : name;
+            if (parserContext.indexVersionCreated().after(IndexVersions.DEPRECATE_SOURCE_MODE_MAPPER) && node.containsKey("mode")) {
+                deprecationLogger.critical(DeprecationCategory.MAPPINGS, "mapping_source_mode", SourceFieldMapper.DEPRECATION_WARNING);
+            }
+            return super.parse(name, node, parserContext);
+        }
+    };
 
     static final class SourceFieldType extends MappedFieldType {
         private final boolean enabled;
@@ -330,8 +358,9 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
         }
     }
 
-    // nullable for bwc reasons
+    // nullable for bwc reasons - TODO: fold this into serializeMode
     private final @Nullable Mode mode;
+    private final boolean serializeMode;
     private final Explicit<Boolean> enabled;
 
     /** indicates whether the source will always exist and be complete, for use by features like the update API */
@@ -341,7 +370,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
     private final String[] excludes;
     private final SourceFilter sourceFilter;
 
-    private SourceFieldMapper(Mode mode, Explicit<Boolean> enabled, String[] includes, String[] excludes) {
+    private SourceFieldMapper(Mode mode, Explicit<Boolean> enabled, String[] includes, String[] excludes, boolean serializeMode) {
         super(new SourceFieldType((enabled.explicit() && enabled.value()) || (enabled.explicit() == false && mode != Mode.DISABLED)));
         this.mode = mode;
         this.enabled = enabled;
@@ -349,6 +378,7 @@ private SourceFieldMapper(Mode mode, Explicit<Boolean> enabled, String[] include
         this.includes = includes;
         this.excludes = excludes;
         this.complete = stored() && sourceFilter == null;
+        this.serializeMode = serializeMode;
     }
 
     private static SourceFilter buildSourceFilter(String[] includes, String[] excludes) {
@@ -419,7 +449,7 @@ protected String contentType() {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(null, Settings.EMPTY, false).init(this);
+        return new Builder(null, Settings.EMPTY, false, serializeMode).init(this);
     }
 
     /**
diff --git a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
index 842fa773ce330..caf65c05cf27d 100644
--- a/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
+++ b/server/src/main/java/org/elasticsearch/node/NodeConstruction.java
@@ -820,7 +820,7 @@ private void construct(
             .searchOperationListeners(searchOperationListeners)
             .build();
 
-        final var parameters = new IndexSettingProvider.Parameters(indicesService::createIndexMapperServiceForValidation);
+        final var parameters = new IndexSettingProvider.Parameters(clusterService, indicesService::createIndexMapperServiceForValidation);
         IndexSettingProviders indexSettingProviders = new IndexSettingProviders(
             Sets.union(
                 builtinIndexSettingProviders(),
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java
index 399740e6200e6..d4d0e67ff4141 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/DynamicFieldsBuilderTests.java
@@ -69,7 +69,7 @@ public void testCreateDynamicStringFieldAsKeywordForDimension() throws IOExcepti
         XContentParser parser = createParser(JsonXContent.jsonXContent, source);
         SourceToParse sourceToParse = new SourceToParse("test", new BytesArray(source), XContentType.JSON);
 
-        SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false).setSynthetic().build();
+        SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false, false).setSynthetic().build();
         RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
             new PassThroughObjectMapper.Builder("labels").setPriority(0).setContainsDimensions().dynamic(ObjectMapper.Dynamic.TRUE)
         ).build(MapperBuilderContext.root(false, false));
diff --git a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
index df6d9380fd141..d7f33b9cdb3ba 100644
--- a/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
+++ b/server/src/test/java/org/elasticsearch/index/mapper/SourceFieldMapperTests.java
@@ -247,14 +247,14 @@ public void testSyntheticSourceInTimeSeries() throws IOException {
         });
         DocumentMapper mapper = createTimeSeriesModeDocumentMapper(mapping);
         assertTrue(mapper.sourceMapper().isSynthetic());
-        assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString());
+        assertEquals("{\"_source\":{}}", mapper.sourceMapper().toString());
     }
 
     public void testSyntheticSourceWithLogsIndexMode() throws IOException {
         XContentBuilder mapping = fieldMapping(b -> { b.field("type", "keyword"); });
         DocumentMapper mapper = createLogsModeDocumentMapper(mapping);
         assertTrue(mapper.sourceMapper().isSynthetic());
-        assertEquals("{\"_source\":{\"mode\":\"synthetic\"}}", mapper.sourceMapper().toString());
+        assertEquals("{\"_source\":{}}", mapper.sourceMapper().toString());
     }
 
     public void testSupportsNonDefaultParameterValues() throws IOException {
diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
index fdc18264e2299..dc70c44a89128 100644
--- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
+++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java
@@ -384,7 +384,7 @@ public void testSearchRequestRuntimeFieldsAndMultifieldDetection() {
 
     public void testSyntheticSourceSearchLookup() throws IOException {
         // Build a mapping using synthetic source
-        SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false).setSynthetic().build();
+        SourceFieldMapper sourceMapper = new SourceFieldMapper.Builder(null, Settings.EMPTY, false, false).setSynthetic().build();
         RootObjectMapper root = new RootObjectMapper.Builder("_doc", Optional.empty()).add(
             new KeywordFieldMapper.Builder("cat", IndexVersion.current()).ignoreAbove(100)
         ).build(MapperBuilderContext.root(true, false));
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
index c20aded9280fc..dd08107bd67fb 100644
--- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
+++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -69,6 +69,7 @@
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.seqno.ReplicationTracker;
 import org.elasticsearch.rest.RestStatus;
@@ -112,6 +113,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
@@ -1827,8 +1829,9 @@ public static CreateIndexResponse createIndex(RestClient client, String name, Se
 
         if (settings != null && settings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true) == false) {
             expectSoftDeletesWarning(request, name);
+        } else if (isSyntheticSourceConfiguredInMapping(mapping)) {
+            request.setOptions(expectVersionSpecificWarnings(v -> v.compatible(SourceFieldMapper.DEPRECATION_WARNING)));
         }
-
         final Response response = client.performRequest(request);
         try (var parser = responseAsParser(response)) {
             return TestResponseParsers.parseCreateIndexResponse(parser);
@@ -1872,6 +1875,27 @@ protected static void expectSoftDeletesWarning(Request request, String indexName
         }));
     }
 
+    @SuppressWarnings("unchecked")
+    protected static boolean isSyntheticSourceConfiguredInMapping(String mapping) {
+        if (mapping == null) {
+            return false;
+        }
+        var mappings = XContentHelper.convertToMap(
+            JsonXContent.jsonXContent,
+            mapping.trim().startsWith("{") ? mapping : '{' + mapping + '}',
+            false
+        );
+        if (mappings.containsKey("_doc")) {
+            mappings = (Map<String, Object>) mappings.get("_doc");
+        }
+        Map<String, Object> sourceMapper = (Map<String, Object>) mappings.get(SourceFieldMapper.NAME);
+        if (sourceMapper == null) {
+            return false;
+        }
+        Object mode = sourceMapper.get("mode");
+        return mode != null && mode.toString().toLowerCase(Locale.ROOT).equals("synthetic");
+    }
+
     protected static Map<String, Object> getIndexSettings(String index) throws IOException {
         Request request = new Request("GET", "/" + index + "/_settings");
         request.addParameter("flat_settings", "true");
@@ -2269,7 +2293,7 @@ protected static Map<String, Set<String>> getClusterStateFeatures(RestClient adm
      */
     protected static IndexVersion minimumIndexVersion() throws IOException {
         final Request request = new Request("GET", "_nodes");
-        request.addParameter("filter_path", "nodes.*.version,nodes.*.max_index_version");
+        request.addParameter("filter_path", "nodes.*.version,nodes.*.max_index_version,nodes.*.index_version");
 
         final Response response = adminClient().performRequest(request);
         final Map<String, Object> nodes = ObjectPath.createFromResponse(response).evaluate("nodes");
@@ -2277,10 +2301,13 @@ protected static IndexVersion minimumIndexVersion() throws IOException {
         IndexVersion minVersion = null;
         for (Map.Entry<String, Object> node : nodes.entrySet()) {
             Map<?, ?> nodeData = (Map<?, ?>) node.getValue();
-            String versionStr = (String) nodeData.get("max_index_version");
+            Object versionStr = nodeData.get("index_version");
+            if (versionStr == null) {
+                versionStr = nodeData.get("max_index_version");
+            }
             // fallback on version if index version is not there
             IndexVersion indexVersion = versionStr != null
-                ? IndexVersion.fromId(Integer.parseInt(versionStr))
+                ? IndexVersion.fromId(Integer.parseInt(versionStr.toString()))
                 : IndexVersion.fromId(
                     parseLegacyVersion((String) nodeData.get("version")).map(Version::id).orElse(IndexVersions.MINIMUM_COMPATIBLE.id())
                 );
diff --git a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
index 8243dcdc9de94..627554f6b261d 100644
--- a/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
+++ b/test/yaml-rest-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/DoSection.java
@@ -20,6 +20,7 @@
 import org.elasticsearch.common.logging.HeaderWarning;
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.core.UpdateForV9;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.rest.action.admin.indices.RestPutIndexTemplateAction;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestResponse;
@@ -495,6 +496,8 @@ public void checkWarningHeaders(final List<String> warningHeaders, String testPa
             }
         }
 
+        unexpected.removeIf(s -> s.endsWith(SourceFieldMapper.DEPRECATION_WARNING + "\""));
+
         if (unexpected.isEmpty() == false
             || unmatched.isEmpty() == false
             || missing.isEmpty() == false
diff --git a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java
index 53e068ae6126e..0bb4afe51b85a 100644
--- a/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java
+++ b/x-pack/plugin/ccr/qa/multi-cluster/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java
@@ -18,6 +18,7 @@
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.repositories.fs.FsRepository;
 import org.elasticsearch.rest.RestStatus;
 
@@ -366,8 +367,10 @@ public void testSyntheticSource() throws Exception {
         final String leaderIndexName = "synthetic_leader";
         if ("leader".equals(targetCluster)) {
             logger.info("Running against leader cluster");
-            createIndex(adminClient(), leaderIndexName, Settings.EMPTY, """
-                "_source": {"mode": "synthetic"},
+            Settings settings = Settings.builder()
+                .put(SourceFieldMapper.INDEX_MAPPER_SOURCE_MODE_SETTING.getKey(), SourceFieldMapper.Mode.SYNTHETIC)
+                .build();
+            createIndex(adminClient(), leaderIndexName, settings, """
                 "properties": {"kwd": {"type": "keyword"}}}""", null);
             for (int i = 0; i < numDocs; i++) {
                 logger.info("Indexing doc [{}]", i);
@@ -392,7 +395,6 @@ public void testSyntheticSource() throws Exception {
             }
             assertBusy(() -> {
                 verifyDocuments(client(), followIndexName, numDocs);
-                assertMap(getIndexMappingAsMap(followIndexName), matchesMap().extraOk().entry("_source", Map.of("mode", "synthetic")));
                 if (overrideNumberOfReplicas) {
                     assertMap(getIndexSettingsAsMap(followIndexName), matchesMap().extraOk().entry("index.number_of_replicas", "0"));
                 } else {
diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java
index d124fdb5755c3..6f45c9d92fd12 100644
--- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java
+++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java
@@ -14,6 +14,7 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.network.NetworkAddress;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.CheckedConsumer;
 import org.elasticsearch.geo.GeometryTestUtils;
 import org.elasticsearch.index.mapper.BlockLoader;
@@ -1456,16 +1457,12 @@ private static void index(String name, String... docs) throws IOException {
     }
 
     private static void createIndex(String name, CheckedConsumer<XContentBuilder, IOException> mapping) throws IOException {
-        Request request = new Request("PUT", "/" + name);
         XContentBuilder index = JsonXContent.contentBuilder().prettyPrint().startObject();
-        index.startObject("mappings");
         mapping.accept(index);
         index.endObject();
-        index.endObject();
         String configStr = Strings.toString(index);
         logger.info("index: {} {}", name, configStr);
-        request.setJsonEntity(configStr);
-        client().performRequest(request);
+        ESRestTestCase.createIndex(name, Settings.EMPTY, configStr);
     }
 
     /**
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
index 93ba126e4196f..04d12fd51bae7 100644
--- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
+++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsDBPlugin.java
@@ -67,10 +67,13 @@ public Collection<IndexSettingProvider> getAdditionalIndexSettingProviders(Index
         if (DiscoveryNode.isStateless(settings)) {
             return List.of(logsdbIndexModeSettingsProvider);
         }
-        return List.of(
-            new SyntheticSourceIndexSettingsProvider(licenseService, parameters.mapperServiceFactory(), logsdbIndexModeSettingsProvider),
-            logsdbIndexModeSettingsProvider
+        var syntheticSettingProvider = new SyntheticSourceIndexSettingsProvider(
+            licenseService,
+            parameters.mapperServiceFactory(),
+            logsdbIndexModeSettingsProvider,
+            () -> parameters.clusterService().state().nodes().getMinSupportedIndexVersion()
         );
+        return List.of(syntheticSettingProvider, logsdbIndexModeSettingsProvider);
     }
 
     @Override
diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java
index e87f10ec19916..1f38ecda19515 100644
--- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java
+++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProvider.java
@@ -26,6 +26,7 @@
 import java.io.IOException;
 import java.time.Instant;
 import java.util.List;
+import java.util.function.Supplier;
 
 import static org.elasticsearch.cluster.metadata.IndexMetadata.INDEX_ROUTING_PATH;
 
@@ -39,15 +40,18 @@ final class SyntheticSourceIndexSettingsProvider implements IndexSettingProvider
     private final SyntheticSourceLicenseService syntheticSourceLicenseService;
     private final CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory;
     private final LogsdbIndexModeSettingsProvider logsdbIndexModeSettingsProvider;
+    private final Supplier<IndexVersion> createdIndexVersion;
 
     SyntheticSourceIndexSettingsProvider(
         SyntheticSourceLicenseService syntheticSourceLicenseService,
         CheckedFunction<IndexMetadata, MapperService, IOException> mapperServiceFactory,
-        LogsdbIndexModeSettingsProvider logsdbIndexModeSettingsProvider
+        LogsdbIndexModeSettingsProvider logsdbIndexModeSettingsProvider,
+        Supplier<IndexVersion> createdIndexVersion
     ) {
         this.syntheticSourceLicenseService = syntheticSourceLicenseService;
         this.mapperServiceFactory = mapperServiceFactory;
         this.logsdbIndexModeSettingsProvider = logsdbIndexModeSettingsProvider;
+        this.createdIndexVersion = createdIndexVersion;
     }
 
     @Override
@@ -148,7 +152,7 @@ private IndexMetadata buildIndexMetadataForMapperService(
         );
         int shardReplicas = indexTemplateAndCreateRequestSettings.getAsInt(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0);
         var finalResolvedSettings = Settings.builder()
-            .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current())
+            .put(IndexMetadata.SETTING_VERSION_CREATED, createdIndexVersion.get())
             .put(indexTemplateAndCreateRequestSettings)
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, dummyShards)
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, shardReplicas)
diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java
index 2d8723a0d8c25..1f5d26eaedf34 100644
--- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java
+++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/SyntheticSourceIndexSettingsProviderTests.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
+import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.MapperTestUtils;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
 import org.elasticsearch.license.MockLicenseState;
@@ -54,7 +55,7 @@ public void setup() {
         provider = new SyntheticSourceIndexSettingsProvider(syntheticSourceLicenseService, im -> {
             newMapperServiceCounter.incrementAndGet();
             return MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName());
-        }, getLogsdbIndexModeSettingsProvider(false));
+        }, getLogsdbIndexModeSettingsProvider(false), IndexVersion::current);
         newMapperServiceCounter.set(0);
     }
 
@@ -336,7 +337,8 @@ public void testGetAdditionalIndexSettingsDowngradeFromSyntheticSourceFileMatch(
         provider = new SyntheticSourceIndexSettingsProvider(
             syntheticSourceLicenseService,
             im -> MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), im.getSettings(), im.getIndex().getName()),
-            getLogsdbIndexModeSettingsProvider(true)
+            getLogsdbIndexModeSettingsProvider(true),
+            IndexVersion::current
         );
         final Settings settings = Settings.EMPTY;
 
diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml
index 33fedce3b59c1..792df4dbf639e 100644
--- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml
+++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/40_source_mode_setting.yml
@@ -470,13 +470,7 @@ create an index with time_series index mode and synthetic source:
       indices.get_settings:
         index: "test_time_series_index_mode_synthetic"
   - match: { test_time_series_index_mode_synthetic.settings.index.mode: time_series }
-
-
-  - do:
-      indices.get_mapping:
-        index: test_time_series_index_mode_synthetic
-
-  - match: { test_time_series_index_mode_synthetic.mappings._source.mode: synthetic }
+  - match: { test_time_series_index_mode_synthetic.settings.index.mapping.source.mode: synthetic }
 
 ---
 create an index with logsdb index mode and synthetic source:
@@ -493,12 +487,7 @@ create an index with logsdb index mode and synthetic source:
       indices.get_settings:
         index: "test_logsdb_index_mode_synthetic"
   - match: { test_logsdb_index_mode_synthetic.settings.index.mode: logsdb }
-
-  - do:
-      indices.get_mapping:
-        index: test_logsdb_index_mode_synthetic
-
-  - match: { test_logsdb_index_mode_synthetic.mappings._source.mode: synthetic }
+  - match: { test_logsdb_index_mode_synthetic.settings.index.mapping.source.mode: synthetic }
 
 ---
 create an index with time_series index mode and stored source:
@@ -524,12 +513,7 @@ create an index with time_series index mode and stored source:
       indices.get_settings:
         index: "test_time_series_index_mode_undefined"
   - match: { test_time_series_index_mode_undefined.settings.index.mode: time_series }
-
-  - do:
-      indices.get_mapping:
-        index: test_time_series_index_mode_undefined
-
-  - match: { test_time_series_index_mode_undefined.mappings._source.mode: stored }
+  - match: { test_time_series_index_mode_undefined.settings.index.mapping.source.mode: stored }
 
 ---
 create an index with logsdb index mode and stored source:
@@ -546,12 +530,7 @@ create an index with logsdb index mode and stored source:
       indices.get_settings:
         index: "test_logsdb_index_mode_undefined"
   - match: { test_logsdb_index_mode_undefined.settings.index.mode: logsdb }
-
-  - do:
-      indices.get_mapping:
-        index: test_logsdb_index_mode_undefined
-
-  - match: { test_logsdb_index_mode_undefined.mappings._source.mode: stored }
+  - match: { test_logsdb_index_mode_undefined.settings.index.mapping.source.mode: stored }
 
 ---
 create an index with time_series index mode and disabled source: