Add OpenTelemetry mappings

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/AbstractDataStreamIT.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.datastreams;
 
 import org.elasticsearch.client.Request;
+import org.elasticsearch.client.Response;
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.elasticsearch.common.settings.SecureString;
@@ -25,6 +26,7 @@
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 /**
  * This base class provides the boilerplate to simplify the development of integration tests.
@@ -104,12 +106,26 @@ static void indexDoc(RestClient client, String dataStreamName, String doc) throw
         assertOK(client.performRequest(request));
     }
 
+    static void bulk(RestClient client, String dataStreamName, List<String> bulkLines) throws IOException {
+        Request request = new Request("POST", "/" + dataStreamName + "/_bulk?refresh=true&pretty&error_trace");
+        request.setJsonEntity(bulkLines.stream().map(s -> s.replace("\n", "")).collect(Collectors.joining("\n")) + "\n");
+        Response response = client.performRequest(request);
+        assertOK(response);
+        Map<String, Object> bulkResponse = entityAsMap(response.getEntity());
+        assertFalse("Bulk response contains errors: " + bulkResponse, (boolean) bulkResponse.get("errors"));
+    }
+
     @SuppressWarnings("unchecked")
     static List<Object> searchDocs(RestClient client, String dataStreamName, String query) throws IOException {
+        Map<String, Object> hits = (Map<String, Object>) search(client, dataStreamName, query).get("hits");
+        return (List<Object>) hits.get("hits");
+    }
+
+    @SuppressWarnings("unchecked")
+    static Map<String, Object> search(RestClient client, String dataStreamName, String query) throws IOException {
         Request request = new Request("GET", "/" + dataStreamName + "/_search");
         request.setJsonEntity(query);
-        Map<String, Object> hits = (Map<String, Object>) entityAsMap(client.performRequest(request)).get("hits");
-        return (List<Object>) hits.get("hits");
+        return entityAsMap(client.performRequest(request));
     }
 
     @SuppressWarnings("unchecked")

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/OtelLogsDataStreamIT.java

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.datastreams;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.is;
+
+public class OtelLogsDataStreamIT extends AbstractDataStreamIT {
+
+    @Override
+    protected String indexTemplateName() {
+        return "logs-otel@template";
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testOtelMapping() throws Exception {
+        String dataStream = "logs-generic.otel-default";
+        createDataStream(client, dataStream);
+
+        indexDoc(client, dataStream, """
+            {
+               "@timestamp": "1688394864123.456789",
+               "body_text": "This is a log message",
+               "data_stream": {
+                 "type": "logs",
+                 "dataset": "generic.otel",
+                 "namespace": "default"
+               },
+               "dropped_attributes_count": 1,
+               "severity_text": "Info",
+               "severity_number": 9,
+               "resource": {
+                 "dropped_attributes_count": 1,
+                 "attributes": {
+                   "service.name": "my-service"
+                 }
+               },
+               "scope": {
+                 "dropped_attributes_count": 1,
+                 "attributes": {
+                   "scope-attr": "scope-attr-val-1"
+                 }
+               },
+               "attributes": {
+                 "foo.attr": "bar",
+                 "complex.attribute": {
+                   "foo": {
+                     "bar": {
+                       "baz": "qux"
+                     }
+                   }
+                 }
+               },
+               "trace_flags": 1,
+               "span_id": "0102040800000000",
+               "trace_id": "08040201000000000000000000000000"
+             }
+            """);
+        List<Object> hits = searchDocs(client, dataStream, """
+            {
+              "query": {
+                "term": {
+                  "foo.attr": "bar"
+                }
+              },
+              "fields": [
+                "*"
+              ]
+            }
+            """);
+        assertThat(hits.size(), is(1));
+        Map<String, Object> fields = ((Map<String, Map<String, Object>>) hits.get(0)).get("fields");
+
+        assertThat(fields.get("data_stream.type"), is(List.of("logs")));
+        assertThat(fields.get("foo.attr"), is(List.of("bar")));
+        assertThat(fields.get("attributes.foo.attr"), is(List.of("bar")));
+        assertThat(fields.get("service.name"), is(List.of("my-service")));
+        assertThat(fields.get("resource.attributes.service.name"), is(List.of("my-service")));
+        assertThat(fields.get("@timestamp"), is(List.of("2023-07-03T14:34:24.123456789Z")));
+
+        Map<String, Object> properties = getMappingProperties(client, getWriteBackingIndex(client, dataStream));
+        assertThat(getValueFromPath(properties, List.of("@timestamp", "type")), is("date_nanos"));
+        assertThat(
+            getValueFromPath(properties, List.of("resource", "properties", "attributes", "properties", "service.name", "type")),
+            is("keyword")
+        );
+        assertThat(
+            getValueFromPath(
+                properties,
+                List.of("resource", "properties", "attributes", "properties", "service.name", "fields", "text", "type")
+            ),
+            is("match_only_text")
+        );
+        assertThat(getValueFromPath(properties, List.of("attributes", "properties", "complex.attribute", "type")), is("flattened"));
+    }
+}

modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/OtelMetircsDataStreamIT.java

@@ -0,0 +1,94 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.datastreams;
+
+import org.elasticsearch.script.field.WriteField;
+
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+public class OtelMetircsDataStreamIT extends AbstractDataStreamIT {
+
+    @Override
+    protected String indexTemplateName() {
+        return "logs-otel@template";
+    }
+
+    @SuppressWarnings("unchecked")
+    public void testOtelMapping() throws Exception {
+        String dataStream = "metrics-generic.otel-default";
+        createDataStream(client, dataStream);
+
+        bulk(client, dataStream, List.of("""
+            { "create" : {"dynamic_templates": {"metrics.my.gauge": "gauge_long"} } }
+            """, """
+              {
+              "@timestamp": "%s",
+              "data_stream": {
+                "type": "metrics",
+                "dataset": "generic.otel",
+                "namespace": "default"
+              },
+              "resource": {
+                "attributes": {
+                  "service.name": "my-service"
+                }
+              },
+              "attributes": {
+                "foo": "bar",
+                "numeric": 42,
+                "host.ip": "127.0.0.1"
+              },
+              "unit": "{thingies}",
+              "metrics": {
+                "my.gauge": 42
+              }
+            }
+            """.formatted(System.currentTimeMillis() + ".123456")));
+        Map<String, Object> response = search(client, dataStream, """
+            {
+              "query": {
+                "exists": {
+                  "field": "my.gauge"
+                }
+              },
+              "size": 0,
+              "aggs": {
+                "avg_value": {
+                  "avg": {
+                    "field": "my.gauge"
+                  }
+                }
+              }
+            }
+            """);
+
+        assertThat(new WriteField("aggregations.avg_value.value", () -> response).get(-1), equalTo(42.0));
+
+        Map<String, Object> properties = getMappingProperties(client, getWriteBackingIndex(client, dataStream));
+        assertThat(getValueFromPath(properties, List.of("@timestamp", "type")), is("date_nanos"));
+        assertThat(getValueFromPath(properties, List.of("scope", "properties", "name", "type")), is("keyword"));
+        assertThat(getValueFromPath(properties, List.of("metrics", "properties", "my.gauge", "type")), is("long"));
+        assertThat(getValueFromPath(properties, List.of("metrics", "properties", "my.gauge", "time_series_metric")), is("gauge"));
+        assertThat(getValueFromPath(properties, List.of("attributes", "properties", "numeric", "type")), is("long"));
+        assertThat(getValueFromPath(properties, List.of("attributes", "properties", "host.ip", "type")), is("ip"));
+        assertThat(getValueFromPath(properties, List.of("attributes", "properties", "foo", "type")), is("keyword"));
+        assertThat(getValueFromPath(properties, List.of("attributes", "properties", "foo", "ignore_above")), is(1024));
+        assertThat(
+            getValueFromPath(
+                properties,
+                List.of("resource", "properties", "attributes", "properties", "service.name", "time_series_dimension")
+            ),
+            is(true)
+        );
+    }
+}

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -210,7 +210,7 @@ public Builder(String name, IndexAnalyzers indexAnalyzers, ScriptCompiler script
                             + "] are true"
                     );
                 }
-            }).precludesParameters(normalizer, ignoreAbove);
+            }).precludesParameters(normalizer);
         }
 
         public Builder(String name, IndexVersion indexCreatedVersion) {

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -801,6 +801,7 @@ protected Set<String> preserveILMPolicyIds() {
             "logs@lifecycle",
             "metrics",
             "metrics@lifecycle",
+            "traces@lifecycle",
             "profiling-60-days",
             "profiling-60-days@lifecycle",
             "synthetics",

x-pack/plugin/apm-data/build.gradle

@@ -27,6 +27,7 @@ dependencies {
   clusterModules project(':modules:ingest-user-agent')
   clusterModules project(':modules:lang-mustache')
   clusterModules project(':modules:mapper-extras')
+  clusterModules project(xpackModule('mapper-version'))
   clusterModules project(xpackModule('analytics'))
   clusterModules project(xpackModule('ilm'))
   clusterModules project(xpackModule('mapper-aggregate-metric'))

x-pack/plugin/apm-data/src/main/resources/resources.yaml

@@ -23,7 +23,6 @@ component-templates:
   - metrics-apm.service_summary@mappings
   - metrics-apm.service_transaction@mappings
   - metrics-apm.transaction@mappings
-  - traces@mappings
   - traces-apm@mappings
   - traces-apm.rum@mappings
 

x-pack/plugin/apm-data/src/yamlRestTest/java/org/elasticsearch/xpack/apmdata/APMYamlTestSuiteIT.java

@@ -27,6 +27,7 @@ public class APMYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
         .module("ingest-user-agent")
         .module("lang-mustache")
         .module("mapper-extras")
+        .module("mapper-version")
         .module("wildcard")
         .module("x-pack-analytics")
         .module("x-pack-apm-data")

x-pack/plugin/core/template-resources/src/main/resources/ecs-tsdb@mappings.json

@@ -0,0 +1,36 @@
+{
+  "template": {
+    "mappings": {
+      "dynamic_templates": [
+        {
+          "ecs_ip": {
+            "mapping": {
+              "type": "ip"
+            },
+            "path_match": [
+              "ip",
+              "*.ip",
+              "*_ip"
+            ],
+            "match_mapping_type": "string"
+          }
+        },
+        {
+          "all_strings_to_keywords": {
+            "mapping": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "match_mapping_type": "string"
+          }
+        }
+      ]
+    }
+  },
+  "_meta": {
+    "description": "dynamic mappings based on ECS for time series data streams, installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version},
+  "deprecated": ${xpack.stack.template.deprecated}
+}

x-pack/plugin/core/template-resources/src/main/resources/logs-otel@mappings.json

@@ -0,0 +1,60 @@
+{
+  "template": {
+    "mappings": {
+      "date_detection": false,
+      "dynamic": "strict",
+      "properties": {
+        "data_stream.type": {
+          "type": "constant_keyword",
+          "value": "logs"
+        },
+        "observed_timestamp": {
+          "type": "date_nanos"
+        },
+        "severity_number": {
+          "type": "byte"
+        },
+        "severity_text": {
+          "type": "keyword"
+        },
+        "log.level": {
+          "type": "alias",
+          "path": "severity_text"
+        },
+        "body_text": {
+          "type": "match_only_text"
+        },
+        "message": {
+          "type": "alias",
+          "path": "body_text"
+        },
+        "body_structured": {
+          "type": "flattened"
+        },
+        "trace_flags": {
+          "type": "byte"
+        },
+        "trace_id": {
+          "type": "keyword"
+        },
+        "trace.id": {
+          "type": "alias",
+          "path": "trace_id"
+        },
+        "span_id": {
+          "type": "keyword"
+        },
+        "span.id": {
+          "type": "alias",
+          "path": "span_id"
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "default mappings for the logs index template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version},
+  "deprecated": ${xpack.stack.template.deprecated}
+}