Merge remote-tracking branch 'es/main' into block_source_loader_ignor…

…e_above

build-tools/src/main/java/org/elasticsearch/gradle/util/PlatformUtils.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.util;
+
+import java.util.stream.Collectors;
+
+public class PlatformUtils {
+
+    public static String normalize(String input) {
+        return input.lines()
+            .map(it -> it.replace('\\', '/'))
+            .map(it -> it.replaceAll("\\d+\\.\\d\\ds", "0.00s"))
+            .map(it -> it.replace("file:/./", "file:./"))
+            .collect(Collectors.joining("\n"));
+    }
+}

docs/changelog/113920.yaml

@@ -0,0 +1,5 @@
+pr: 113920
+summary: Add initial support for `semantic_text` field type
+area: Search
+type: enhancement
+issues: []

docs/changelog/114742.yaml

@@ -0,0 +1,5 @@
+pr: 114742
+summary: Adding support for additional mapping to simulate ingest API
+area: Ingest Node
+type: enhancement
+issues: []

docs/changelog/114774.yaml

@@ -0,0 +1,5 @@
+pr: 114774
+summary: "ESQL: Add support for multivalue fields in Arrow output"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/114819.yaml

@@ -0,0 +1,6 @@
+pr: 114819
+summary: Don't use a `BytesStreamOutput` to copy keys in `BytesRefBlockHash`
+area: EQL
+type: bug
+issues:
+ - 114599

docs/changelog/115181.yaml

@@ -0,0 +1,5 @@
+pr: 115181
+summary: Always check the parent breaker with zero bytes in `PreallocatedCircuitBreakerService`
+area: Aggregations
+type: bug
+issues: []

docs/changelog/115194.yaml

@@ -0,0 +1,7 @@
+pr: 115194
+summary: Update APM Java Agent to support JDK 23
+area: Infra/Metrics
+type: upgrade
+issues:
+ - 115101
+ - 115100

docs/changelog/115245.yaml

@@ -0,0 +1,8 @@
+pr: 115245
+summary: "ESQL: Fix `REVERSE` with backspace character"
+area: ES|QL
+type: bug
+issues:
+ - 114372
+ - 115227
+ - 115228

docs/reference/ingest/apis/simulate-ingest.asciidoc

@@ -108,6 +108,14 @@ POST /_ingest/_simulate
       "index_patterns": ["my-index-*"],
       "composed_of": ["component_template_1", "component_template_2"]
     }
+  },
+  "mapping_addition": { <4>
+    "dynamic": "strict",
+    "properties": {
+      "foo": {
+        "type": "keyword"
+      }
+    }
   }
 }
 ----
@@ -117,6 +125,7 @@ POST /_ingest/_simulate
 These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result.
 <3> This replaces the existing `my-index-template` index template with the contents given here for the duration of this request.
 These templates can be used to change the pipeline(s) used, or to modify the mapping that will be used to validate the result.
+<4> This mapping is merged into the index's final mapping just before validation. It is used only for the duration of this request.
 
 [[simulate-ingest-api-request]]
 ==== {api-request-title}
@@ -246,6 +255,10 @@ include::{es-ref-dir}/indices/put-index-template.asciidoc[tag=request-body]
 
 ====
 
+`mapping_addition`::
+(Optional, <<mapping,mapping object>>)
+Definition of a mapping that will be merged into the index's mapping for validation during the course of this request.
+
 [[simulate-ingest-api-example]]
 ==== {api-examples-title}
 

docs/reference/mapping/types/binary.asciidoc

@@ -68,8 +68,16 @@ Synthetic source may sort `binary` values in order of their byte representation.
 ----
 PUT idx
 {
+  "settings": {
+    "index": {
+      "mapping": {
+        "source": {
+          "mode": "synthetic"
+        }
+      }
+    }
+  },
   "mappings": {
-    "_source": { "mode": "synthetic" },
     "properties": {
       "binary": { "type": "binary", "doc_values": true }
     }

docs/reference/search/search.asciidoc

@@ -38,7 +38,7 @@ must have the `read` index privilege for the alias's data streams or indices.
 
 Allows you to execute a search query and get back search hits that match the
 query. You can provide search queries using the <<search-api-query-params-q,`q`
-query string parameter>> or <<search-search,request body>>.
+query string parameter>> or <<search-search-api-request-body,request body>>.
 
 [[search-search-api-path-params]]
 ==== {api-path-parms-title}

docs/reference/security/limitations.asciidoc

@@ -81,12 +81,13 @@ including the following queries:
 * A search request cannot be profiled if document level security is enabled.
 * The <<search-terms-enum,terms enum API>> does not return terms if document
 level security is enabled.
+* The <<query-dsl-multi-match-query, `multi_match`>> query does not support specifying fields using wildcards.
 
 NOTE: While document-level security prevents users from viewing restricted documents,
 it's still possible to write search requests that return aggregate information about the
 entire index. A user whose access is restricted to specific documents in an index could
 still learn about field names and terms that only exist in inaccessible
-documents, and count how many inaccessible documents contain a given term. 
+documents, and count how many inaccessible documents contain a given term.
 
 [discrete]
 [[alias-limitations]]

gradle/verification-metadata.xml

@@ -69,9 +69,9 @@
             <sha256 value="10fe288fd7a2cdaf5175332b73529f9abf8fd54dcfff317d6967c0c35ffb133b" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="co.elastic.apm" name="elastic-apm-agent" version="1.44.0">
-         <artifact name="elastic-apm-agent-1.44.0.jar">
-            <sha256 value="169dc01756a960692314c12ed8868baf937399f6eddc2d7e6ede9caad8bec0a2" origin="Generated by Gradle"/>
+      <component group="co.elastic.apm" name="elastic-apm-agent" version="1.52.0">
+         <artifact name="elastic-apm-agent-1.52.0.jar">
+            <sha256 value="ef6c8f75bd6181e717cdd172864441580708c7ee8543175621a3f404f4ba6429" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="co.elastic.logging" name="ecs-logging-core" version="1.2.0">

modules/apm/build.gradle

@@ -19,7 +19,7 @@ dependencies {
   implementation "io.opentelemetry:opentelemetry-api:${otelVersion}"
   implementation "io.opentelemetry:opentelemetry-context:${otelVersion}"
   implementation "io.opentelemetry:opentelemetry-semconv:${otelSemconvVersion}"
-  runtimeOnly    "co.elastic.apm:elastic-apm-agent:1.44.0"
+  runtimeOnly    "co.elastic.apm:elastic-apm-agent:1.52.0"
 }
 
 tasks.named("dependencyLicenses").configure {

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBIndexingIT.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction;
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
 import org.elasticsearch.action.bulk.BulkRequest;
+import org.elasticsearch.action.bulk.IndexDocFailureStoreStatus;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -170,7 +171,7 @@ public void testTimeRanges() throws Exception {
             var indexRequest = new IndexRequest("k8s").opType(DocWriteRequest.OpType.CREATE);
             time = randomBoolean() ? endTime : endTime.plusSeconds(randomIntBetween(1, 99));
             indexRequest.source(DOC.replace("$time", formatInstant(time)), XContentType.JSON);
-            expectThrows(IllegalArgumentException.class, () -> client().index(indexRequest).actionGet());
+            expectThrows(IndexDocFailureStoreStatus.ExceptionWithFailureStoreStatus.class, () -> client().index(indexRequest).actionGet());
         }
 
         // Fetch UpdateTimeSeriesRangeService and increment time range of latest backing index:

modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java

@@ -26,6 +26,7 @@
 public class DataStreamFeatures implements FeatureSpecification {
 
     public static final NodeFeature DATA_STREAM_LIFECYCLE = new NodeFeature("data_stream.lifecycle");
+    public static final NodeFeature DATA_STREAM_FAILURE_STORE_TSDB_FIX = new NodeFeature("data_stream.failure_store.tsdb_fix");
 
     @Override
     public Map<NodeFeature, Version> getHistoricalFeatures() {
@@ -41,4 +42,9 @@ public Set<NodeFeature> getFeatures() {
             DataStreamGlobalRetention.GLOBAL_RETENTION                       // Added in 8.14
         );
     }
+
+    @Override
+    public Set<NodeFeature> getTestFeatures() {
+        return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX);
+    }
 }

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml

@@ -182,6 +182,107 @@ index without timestamp:
         body:
           - '{"metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
 
+---
+TSDB failures go to failure store:
+  - requires:
+      cluster_features: ["data_stream.failure_store.tsdb_fix"]
+      reason: "tests tsdb failure store fixes in 8.16.0 that catch timestamp errors that happen earlier in the process and redirect them to the failure store."
+
+  - do:
+      allowed_warnings:
+        - "index template [my-template2] has index patterns [fs-k8s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-template2] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-template2
+        body:
+          index_patterns: [ "fs-k8s*" ]
+          data_stream:
+            failure_store: true
+          template:
+            settings:
+              index:
+                mode: time_series
+                number_of_replicas: 1
+                number_of_shards: 2
+                routing_path: [ metricset, time_series_dimension ]
+                time_series:
+                  start_time: 2021-04-28T00:00:00Z
+                  end_time: 2021-04-29T00:00:00Z
+            mappings:
+              properties:
+                "@timestamp":
+                  type: date
+                metricset:
+                  type: keyword
+                  time_series_dimension: true
+                k8s:
+                  properties:
+                    pod:
+                      properties:
+                        uid:
+                          type: keyword
+                          time_series_dimension: true
+                        name:
+                          type: keyword
+                        ip:
+                          type: ip
+                        network:
+                          properties:
+                            tx:
+                              type: long
+                            rx:
+                              type: long
+  - do:
+      index:
+        index: fs-k8s
+        body:
+          - '{"metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+  - match: { result : "created"}
+  - match: { failure_store : "used"}
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{ "create": { "_index": "fs-k8s"} }'
+          - '{"@timestamp":"2021-04-28T01:00:00ZZ", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{ "create": { "_index": "k8s"} }'
+          - '{ "@timestamp": "2021-04-28T01:00:00ZZ", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{ "create": { "_index": "fs-k8s"} }'
+          - '{ "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{ "create": { "_index": "fs-k8s"} }'
+          - '{ "@timestamp":"2000-04-28T01:00:00ZZ", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{ "create": { "_index": "k8s"} }'
+          - '{"metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+          - '{ "create": { "_index": "k8s"} }'
+          - '{ "@timestamp":"2000-04-28T01:00:00ZZ", "metricset": "pod", "k8s": {"pod": {"name": "cat", "uid":"947e4ced-1786-4e53-9e0c-5c447e959507", "ip": "10.10.55.1", "network": {"tx": 2001818691, "rx": 802133794}}}}'
+  - is_true: errors
+
+  # Successfully indexed to backing index
+  - match: { items.0.create._index: '/\.ds-fs-k8s-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { items.0.create.status: 201 }
+  - is_false: items.0.create.failure_store
+  - match: { items.1.create._index: '/\.ds-k8s-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
+  - match: { items.1.create.status: 201 }
+  - is_false: items.1.create.failure_store
+
+  # Successfully indexed to failure store
+  - match: { items.2.create._index: '/\.fs-fs-k8s-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
+  - match: { items.2.create.status: 201 }
+  - match: { items.2.create.failure_store: used }
+  - match: { items.3.create._index: '/\.fs-fs-k8s-(\d{4}\.\d{2}\.\d{2}-)?000002/' }
+  - match: { items.3.create.status: 201 }
+  - match: { items.3.create.failure_store: used }
+
+  # Rejected, eligible to go to failure store, but failure store not enabled
+  - match: { items.4.create._index: 'k8s' }
+  - match: { items.4.create.status: 400 }
+  - match: { items.4.create.error.type: timestamp_error }
+  - match: { items.4.create.failure_store: not_enabled }
+  - match: { items.4.create._index: 'k8s' }
+  - match: { items.4.create.status: 400 }
+  - match: { items.4.create.error.type: timestamp_error }
+  - match: { items.4.create.failure_store: not_enabled }
+
 ---
 index without timestamp with pipeline:
   - do:

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/190_failure_store_redirection.yml

@@ -879,7 +879,7 @@ teardown:
   # Successfully indexed to backing index
   - match: { items.0.create._index: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }
   - match: { items.0.create.status: 201 }
-  - is_false: items.1.create.failure_store
+  - is_false: items.0.create.failure_store
 
   # Rejected but not eligible to go to failure store
   - match: { items.1.create._index: '/\.ds-logs-foobar-(\d{4}\.\d{2}\.\d{2}-)?000001/' }