Merge remote-tracking branch 'origin/main' into profiling-enterprise-…

…license

docs/changelog/100232.yaml

@@ -0,0 +1,5 @@
+pr: 100232
+summary: "Tracing: Use `doPriv` when working with spans, use `SpanId`"
+area: Infra/Core
+type: bug
+issues: []

docs/changelog/100238.yaml

@@ -0,0 +1,6 @@
+pr: 100238
+summary: "ESQL: Remove aliasing inside Eval"
+area: ES|QL
+type: bug
+issues:
+ - 100174

docs/changelog/100273.yaml

@@ -0,0 +1,5 @@
+pr: 100273
+summary: Propagate cancellation in `GetHealthAction`
+area: Health
+type: bug
+issues: []

docs/changelog/100284.yaml

@@ -0,0 +1,5 @@
+pr: 100284
+summary: Defend against negative datafeed start times
+area: Machine Learning
+type: bug
+issues: []

docs/changelog/100287.yaml

@@ -0,0 +1,5 @@
+pr: 100287
+summary: Add an assertion to the testTransformFeatureReset test case
+area: Transform
+type: bug
+issues: []

modules/apm/src/main/java/org/elasticsearch/telemetry/apm/internal/tracing/APMTracer.java

@@ -283,7 +283,7 @@ private Context getParentContext(ThreadContext threadContext) {
     public Releasable withScope(SpanId spanId) {
         final Context context = spans.get(spanId);
         if (context != null) {
-            var scope = context.makeCurrent();
+            var scope = AccessController.doPrivileged((PrivilegedAction<Scope>) context::makeCurrent);
             return scope::close;
         }
         return () -> {};
@@ -381,7 +381,10 @@ public void stopTrace(SpanId spanId) {
         final var span = Span.fromContextOrNull(spans.remove(spanId));
         if (span != null) {
             logger.trace("Finishing trace [{}]", spanId);
-            span.end();
+            AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
+                span.end();
+                return null;
+            });
         }
     }
 
@@ -390,7 +393,10 @@ public void stopTrace(SpanId spanId) {
      */
     @Override
     public void stopTrace() {
-        Span.current().end();
+        AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
+            Span.current().end();
+            return null;
+        });
     }
 
     @Override

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java

@@ -20,6 +20,7 @@
 import static org.hamcrest.Matchers.contains;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.matchesRegex;
 
 public class LogsDataStreamIT extends DisabledSecurityDataStreamTestCase {
 
@@ -100,6 +101,189 @@ public void testDefaultLogsSettingAndMapping() throws Exception {
         }
     }
 
+    @SuppressWarnings("unchecked")
+    public void testCustomMapping() throws Exception {
+        RestClient client = client();
+        waitForLogs(client);
+
+        {
+            Request request = new Request("POST", "/_component_template/logs@custom");
+            request.setJsonEntity("""
+                {
+                  "template": {
+                    "settings": {
+                      "index": {
+                        "query": {
+                          "default_field": ["custom-message"]
+                        }
+                      }
+                    },
+                    "mappings": {
+                      "properties": {
+                        "numeric_field": {
+                          "type": "integer"
+                        },
+                        "socket": {
+                          "properties": {
+                            "ip": {
+                              "type": "keyword"
+                            }
+                          }
+                        }
+                      }
+                    }
+                  }
+                }
+                """);
+            assertOK(client.performRequest(request));
+        }
+
+        String dataStreamName = "logs-generic-default";
+        createDataStream(client, dataStreamName);
+        String backingIndex = getWriteBackingIndex(client, dataStreamName);
+
+        // Verify that the custom settings.index.query.default_field overrides the default query field - "message"
+        Map<String, Object> settings = getSettings(client, backingIndex);
+        assertThat(settings.get("index.query.default_field"), is(List.of("custom-message")));
+
+        // Verify that the new field from the custom component template is applied
+        putMapping(client, backingIndex);
+        Map<String, Object> mappingProperties = getMappingProperties(client, backingIndex);
+        assertThat(((Map<String, Object>) mappingProperties.get("numeric_field")).get("type"), equalTo("integer"));
+        assertThat(
+            ((Map<String, Object>) mappingProperties.get("socket")).get("properties"),
+            equalTo(Map.of("ip", Map.of("type", "keyword")))
+        );
+
+        // Insert valid doc and verify successful indexing
+        {
+            indexDoc(client, dataStreamName, """
+                {
+                  "test": "doc-with-ip",
+                  "socket": {
+                    "ip": "127.0.0.1"
+                  }
+                }
+                """);
+            List<Object> results = searchDocs(client, dataStreamName, """
+                {
+                  "query": {
+                    "term": {
+                      "test": {
+                        "value": "doc-with-ip"
+                      }
+                    }
+                  },
+                  "fields": ["socket.ip"]
+                }
+                """);
+            Map<String, Object> fields = ((Map<String, Map<String, Object>>) results.get(0)).get("_source");
+            assertThat(fields.get("socket"), is(Map.of("ip", "127.0.0.1")));
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testLogsDefaultPipeline() throws Exception {
+        RestClient client = client();
+        waitForLogs(client);
+
+        {
+            Request request = new Request("POST", "/_component_template/logs@custom");
+            request.setJsonEntity("""
+                {
+                  "template": {
+                    "mappings": {
+                      "properties": {
+                        "custom_timestamp": {
+                          "type": "date"
+                        }
+                      }
+                    }
+                  }
+                }
+                """);
+            assertOK(client.performRequest(request));
+        }
+        {
+            Request request = new Request("PUT", "/_ingest/pipeline/logs@custom");
+            request.setJsonEntity("""
+                    {
+                      "processors": [
+                        {
+                          "set" : {
+                            "field": "custom_timestamp",
+                            "copy_from": "_ingest.timestamp"
+                          }
+                        }
+                      ]
+                    }
+                """);
+            assertOK(client.performRequest(request));
+        }
+
+        String dataStreamName = "logs-generic-default";
+        createDataStream(client, dataStreamName);
+        String backingIndex = getWriteBackingIndex(client, dataStreamName);
+
+        // Verify mapping from custom logs
+        Map<String, Object> mappingProperties = getMappingProperties(client, backingIndex);
+        assertThat(((Map<String, Object>) mappingProperties.get("@timestamp")).get("type"), equalTo("date"));
+
+        // no timestamp - testing default pipeline's @timestamp set processor
+        {
+            indexDoc(client, dataStreamName, """
+                {
+                  "message": "no_timestamp"
+                }
+                """);
+            List<Object> results = searchDocs(client, dataStreamName, """
+                {
+                  "query": {
+                    "term": {
+                      "message": {
+                        "value": "no_timestamp"
+                      }
+                    }
+                  },
+                  "fields": ["@timestamp", "custom_timestamp"]
+                }
+                """);
+            Map<String, Object> source = ((Map<String, Map<String, Object>>) results.get(0)).get("_source");
+            String timestamp = (String) source.get("@timestamp");
+            assertThat(timestamp, matchesRegex("[0-9-]+T[0-9:.]+Z"));
+            assertThat(source.get("custom_timestamp"), is(timestamp));
+
+            Map<String, Object> fields = ((Map<String, Map<String, Object>>) results.get(0)).get("fields");
+            timestamp = ((List<String>) fields.get("@timestamp")).get(0);
+            assertThat(timestamp, matchesRegex("[0-9-]+T[0-9:.]+Z"));
+            assertThat(((List<Object>) fields.get("custom_timestamp")).get(0), is(timestamp));
+        }
+
+        // verify that when a document is ingested with a timestamp, it does not get overridden
+        {
+            indexDoc(client, dataStreamName, """
+                {
+                  "message": "with_timestamp",
+                  "@timestamp": "2023-05-10"
+                }
+                """);
+            List<Object> results = searchDocs(client, dataStreamName, """
+                {
+                  "query": {
+                    "term": {
+                      "message": {
+                        "value": "with_timestamp"
+                      }
+                    }
+                  },
+                  "fields": ["@timestamp", "custom_timestamp"]
+                }
+                """);
+            Map<String, Object> fields = ((Map<String, Map<String, Object>>) results.get(0)).get("fields");
+            assertThat(fields.get("@timestamp"), is(List.of("2023-05-10T00:00:00.000Z")));
+        }
+    }
+
     private static void waitForLogs(RestClient client) throws Exception {
         assertBusy(() -> {
             try {