Adjust template and pipeline names to convention

See elastic#96267 (comment) - Index templates have a "@template" suffix - Component templates are split into settings and mappings, and have a @settings and @mappings suffix respectively - Ingest pipelines generally have a @pipeline suffix, except for one special case where we use @default-pipeline We no longer have a component template for every data stream, rather we have reusable component templates where it makes sense. For example, we now have metrics-apm@settings and metrics-apm@mappings, which are included by all APM metrics data streams. We now set both default_pipeline and final_pipeline in the index templates, preventing users from overriding them completely. The default pipeline is always apm@default-pipeline, which performs some rerouting of legacy data, and invokes user-defined @Custom pipelines based on the data_stream.type and data_stream.dataset fields. The final pipeline just performs built-in processing.
axw · Sep 13, 2023 · db19f78 · db19f78
1 parent bac147f
commit db19f78
Show file tree

Hide file tree

Showing 47 changed files with 457 additions and 337 deletions.
diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -1851,7 +1851,7 @@ protected static boolean isXPackTemplate(String name) {
         if (name.startsWith("elastic-connectors")) {
             return true;
         }
-        if (name.equals("apm")
+        if (name.startsWith("apm@")
             || name.startsWith("apm-")
             || name.startsWith("traces-apm")
             || name.startsWith("metrics-apm")

diff --git a/...omponent-templates/apm-retention-10d.yaml → ...omponent-templates/[email protected] b/...omponent-templates/apm-retention-10d.yaml → ...omponent-templates/[email protected]
diff --git a/...mponent-templates/apm-retention-180d.yaml → ...mponent-templates/[email protected] b/...mponent-templates/apm-retention-180d.yaml → ...mponent-templates/[email protected]
diff --git a/...mponent-templates/apm-retention-390d.yaml → ...mponent-templates/[email protected] b/...mponent-templates/apm-retention-390d.yaml → ...mponent-templates/[email protected]
diff --git a/...omponent-templates/apm-retention-90d.yaml → ...omponent-templates/[email protected] b/...omponent-templates/apm-retention-90d.yaml → ...omponent-templates/[email protected]
diff --git a/...in/resources/component-templates/apm.yaml → ...ces/component-templates/[email protected] b/...in/resources/component-templates/apm.yaml → ...ces/component-templates/[email protected]
@@ -1,11 +1,5 @@
 version: ${xpack.apm.template.version}
 template:
-  settings:
-    index:
-      default_pipeline: apm
-      sort:
-        field: "@timestamp"
-        order: desc
   mappings:
     dynamic: true
     dynamic_templates:

diff --git a/x-pack/plugin/apm/src/main/resources/component-templates/[email protected] b/x-pack/plugin/apm/src/main/resources/component-templates/[email protected]
@@ -0,0 +1,7 @@
+version: ${xpack.apm.template.version}
+template:
+  settings:
+    index:
+      sort:
+        field: "@timestamp"
+        order: desc
diff --git a/...s/component-templates/logs-apm.error.yaml → ...nt-templates/[email protected] b/...s/component-templates/logs-apm.error.yaml → ...nt-templates/[email protected]
@@ -3,9 +3,6 @@ version: ${xpack.apm.template.version}
 template:
   mappings:
     properties:
-      processor.event:
-        type: constant_keyword
-        value: error
       error.custom:
         type: flattened
       error.exception.attributes:

diff --git a/x-pack/plugin/apm/src/main/resources/component-templates/metrics-apm.internal.yaml b/x-pack/plugin/apm/src/main/resources/component-templates/metrics-apm.internal.yaml
diff --git a/...ates/metrics-apm.service_destination.yaml → [email protected] b/...ates/metrics-apm.service_destination.yaml → [email protected]
@@ -1,17 +1,8 @@
 ---
 version: ${xpack.apm.template.version}
 template:
-  settings:
-    codec: best_compression
-    index:
-      default_pipeline: metrics-apm
   mappings:
-    _source:
-      mode: synthetic
     properties:
-      processor.event:
-        type: constant_keyword
-        value: metric
       metricset.name:
         type: constant_keyword
         value: service_destination

diff --git a/...emplates/metrics-apm.service_summary.yaml → [email protected] b/...emplates/metrics-apm.service_summary.yaml → [email protected]
@@ -1,17 +1,8 @@
 ---
 version: ${xpack.apm.template.version}
 template:
-  settings:
-    codec: best_compression
-    index:
-      default_pipeline: metrics-apm
   mappings:
-    _source:
-      mode: synthetic
     properties:
-      processor.event:
-        type: constant_keyword
-        value: metric
       metricset.name:
         type: constant_keyword
         value: service_summary

diff --git a/...ates/metrics-apm.service_transaction.yaml → [email protected] b/...ates/metrics-apm.service_transaction.yaml → [email protected]
@@ -1,16 +1,7 @@
 version: ${xpack.apm.template.version}
 template:
-  settings:
-    codec: best_compression
-    index:
-      default_pipeline: metrics-apm
   mappings:
-    _source:
-      mode: synthetic
     properties:
-      processor.event:
-        type: constant_keyword
-        value: metric
       metricset.name:
         type: constant_keyword
         value: service_transaction

diff --git a/...nt-templates/metrics-apm.transaction.yaml → ...tes/[email protected] b/...nt-templates/metrics-apm.transaction.yaml → ...tes/[email protected]
@@ -1,16 +1,7 @@
 version: ${xpack.apm.template.version}
 template:
-  settings:
-    codec: best_compression
-    index:
-      default_pipeline: metrics-apm
   mappings:
-    _source:
-      mode: synthetic
     properties:
-      processor.event:
-        type: constant_keyword
-        value: metric
       metricset.name:
         type: constant_keyword
         value: transaction

diff --git a/.../component-templates/metrics-apm.app.yaml → ...onent-templates/[email protected] b/.../component-templates/metrics-apm.app.yaml → ...onent-templates/[email protected]
@@ -1,10 +1,6 @@
 ---
 version: ${xpack.apm.template.version}
 template:
-  settings:
-    codec: best_compression
-    index:
-      default_pipeline: metrics-apm
   mappings:
     _source:
       mode: synthetic

diff --git a/x-pack/plugin/apm/src/main/resources/component-templates/[email protected] b/x-pack/plugin/apm/src/main/resources/component-templates/[email protected]
@@ -0,0 +1,5 @@
+---
+version: ${xpack.apm.template.version}
+template:
+  settings:
+    codec: best_compression
diff --git a/...s/component-templates/traces-apm.rum.yaml → ...nt-templates/[email protected] b/...s/component-templates/traces-apm.rum.yaml → ...nt-templates/[email protected]
@@ -3,9 +3,6 @@ version: ${xpack.apm.template.version}
 template:
   mappings:
     properties:
-      data_stream.dataset:
-        type: constant_keyword
-        value: apm.rum
       transaction.experience.cls:
         type: scaled_float
         scaling_factor: 1000

diff --git a/...urces/component-templates/traces-apm.yaml → ...ponent-templates/[email protected] b/...urces/component-templates/traces-apm.yaml → ...ponent-templates/[email protected]
@@ -1,19 +1,10 @@
 ---
 version: ${xpack.apm.template.version}
 template:
-  settings:
-    index:
-      default_pipeline: traces-apm
   mappings:
     properties:
       processor.event:
         type: keyword
-      data_stream.type:
-        type: constant_keyword
-        value: traces
-      data_stream.dataset:
-        type: constant_keyword
-        value: apm
       event.success_count:
         type: byte
         index: false

diff --git a/x-pack/plugin/apm/src/main/resources/index-templates/README.md b/x-pack/plugin/apm/src/main/resources/index-templates/README.md
@@ -0,0 +1,7 @@
+Mappings and settings that must not be overridden are to be defined
+in the index templates, as these will take precedence over component
+template mappings and settings. This includes:
+
+ - `index.default_pipeline`
+ - `index.final_pipeline`
+ - values for `data_stream.*`
diff --git a/...sources/index-templates/logs-apm.app.yaml → ...ndex-templates/[email protected] b/...sources/index-templates/logs-apm.app.yaml → ...ndex-templates/[email protected]
@@ -2,17 +2,23 @@ version: ${xpack.apm.template.version}
 index_patterns: ["logs-apm.app.*-*"]
 priority: 150
 data_stream: {}
+allow_auto_create: true
+_meta:
+  description: Index template for logs-apm.app.*-*
+  managed: true
 composed_of:
 - logs-mappings
-- apm
-- apm-retention-10d
+- apm@mappings
+- apm@settings
+- apm-10d@lifecycle
 - apm@custom
 - logs-apm.app@custom
 - ecs@dynamic_templates
 ignore_missing_component_templates:
 - logs-apm.app@custom
 - apm@custom
-allow_auto_create: true
-_meta:
-  description: Index template for logs-apm.app.*-*
-  managed: true
+template:
+  settings:
+    index:
+      default_pipeline: apm@default-pipeline
+      final_pipeline: apm@pipeline
diff --git a/...urces/index-templates/logs-apm.error.yaml → ...ex-templates/[email protected] b/...urces/index-templates/logs-apm.error.yaml → ...ex-templates/[email protected]
@@ -3,18 +3,29 @@ version: ${xpack.apm.template.version}
 index_patterns: ["logs-apm.error-*"]
 priority: 150
 data_stream: {}
+allow_auto_create: true
+_meta:
+  description: Index template for logs-apm.error-*
+  managed: true
 composed_of:
 - logs-mappings
-- apm
-- apm-retention-10d
-- logs-apm.error
+- apm@mappings
+- apm@settings
+- apm-10d@lifecycle
+- logs-apm.error@mappings
 - apm@custom
 - logs-apm.error@custom
 - ecs@dynamic_templates
 ignore_missing_component_templates:
 - logs-apm.error@custom
 - apm@custom
-allow_auto_create: true
-_meta:
-  description: Index template for logs-apm.error-*
-  managed: true
+template:
+  mappings:
+    properties:
+      processor.event:
+        type: constant_keyword
+        value: error
+  settings:
+    index:
+      default_pipeline: apm@default-pipeline
+      final_pipeline: apm@pipeline
diff --git a/...rces/index-templates/metrics-apm.app.yaml → ...x-templates/[email protected] b/...rces/index-templates/metrics-apm.app.yaml → ...x-templates/[email protected]
@@ -2,18 +2,25 @@ version: ${xpack.apm.template.version}
 index_patterns: ["metrics-apm.app.*-*"]
 priority: 150
 data_stream: {}
+allow_auto_create: true
+_meta:
+  description: "Index template for metrics-apm.app.*-*"
+  managed: true
 composed_of:
 - metrics-mappings
-- apm
-- apm-retention-90d
-- metrics-apm.app
+- apm@mappings
+- apm@settings
+- apm-90d@lifecycle
+- metrics-apm@mappings
+- metrics-apm@settings
 - apm@custom
 - metrics-apm.app@custom
 - ecs@dynamic_templates
 ignore_missing_component_templates:
 - apm@custom
 - metrics-apm.app@custom
-allow_auto_create: true
-_meta:
-  description: "Index template for metrics-apm.app.*-*"
-  managed: true
+template:
+  settings:
+    index:
+      default_pipeline: apm@default-pipeline
+      final_pipeline: metrics-apm@pipeline
diff --git a/...index-templates/metrics-apm.internal.yaml → ...plates/[email protected] b/...index-templates/metrics-apm.internal.yaml → ...plates/[email protected]
@@ -3,18 +3,30 @@ version: ${xpack.apm.template.version}
 index_patterns: ["metrics-apm.internal-*"]
 priority: 150
 data_stream: {}
+allow_auto_create: true
+_meta:
+  description: "Index template for metrics-apm.internal-*"
+  managed: true
 composed_of:
 - metrics-mappings
-- apm
-- apm-retention-90d
-- metrics-apm.internal
+- apm@mappings
+- apm@settings
+- apm-90d@lifecycle
+- metrics-apm@mappings
+- metrics-apm@settings
 - apm@custom
 - metrics-apm.internal@custom
 - ecs@dynamic_templates
 ignore_missing_component_templates:
 - apm@custom
 - metrics-apm.internal@custom
-allow_auto_create: true
-_meta:
-  description: "Index template for metrics-apm.internal-*"
-  managed: true
+template:
+  settings:
+    index:
+      default_pipeline: apm@default-pipeline
+      final_pipeline: metrics-apm@pipeline
+  mappings:
+    properties:
+      data_stream.dataset:
+        type: constant_keyword
+        value: apm.internal
diff --git a/.../metrics-apm.service_destination.10m.yaml → [email protected] b/.../metrics-apm.service_destination.10m.yaml → [email protected]
@@ -4,18 +4,26 @@ index_patterns: [metrics-apm.service_destination.10m-*]
 priority: 150
 data_stream:
   hidden: true
+allow_auto_create: true
+_meta:
+  description: Index template for metrics-apm.service_destination.10m-*
+  managed: true
 composed_of:
 - metrics-mappings
-- apm
-- apm-retention-180d
-- metrics-apm.service_destination
+- apm@mappings
+- apm@settings
+- apm-180d@lifecycle
+- metrics-apm@mappings
+- metrics-apm@settings
+- metrics-apm.service_destination@mappings
 - apm@custom
 - metrics-apm.service_destination@custom
 - ecs@dynamic_templates
 ignore_missing_component_templates:
 - apm@custom
 - metrics-apm.service_destination@custom
-allow_auto_create: true
-_meta:
-  description: Index template for metrics-apm.service_destination.10m-*
-  managed: true
+template:
+  settings:
+    index:
+      default_pipeline: apm@default-pipeline
+      final_pipeline: metrics-apm@pipeline
diff --git a/...s/metrics-apm.service_destination.1m.yaml → [email protected] b/...s/metrics-apm.service_destination.1m.yaml → [email protected]
@@ -3,18 +3,26 @@ version: ${xpack.apm.template.version}
 index_patterns: [metrics-apm.service_destination.1m-*]
 priority: 150
 data_stream: {}
+allow_auto_create: true
+_meta:
+  description: Index template for metrics-apm.service_destination.1m-*
+  managed: true
 composed_of:
 - metrics-mappings
-- apm
-- apm-retention-90d
-- metrics-apm.service_destination
+- apm@mappings
+- apm@settings
+- apm-90d@lifecycle
+- metrics-apm@mappings
+- metrics-apm@settings
+- metrics-apm.service_destination@mappings
 - apm@custom
 - metrics-apm.service_destination@custom
 - ecs@dynamic_templates
 ignore_missing_component_templates:
 - apm@custom
 - metrics-apm.service_destination@custom
-allow_auto_create: true
-_meta:
-  description: Index template for metrics-apm.service_destination.1m-*
-  managed: true
+template:
+  settings:
+    index:
+      default_pipeline: apm@default-pipeline
+      final_pipeline: metrics-apm@pipeline