[7.x] Add "synthetics-*-*" templates for synthetics fleet data (#62193)…

… (#62346)

* Add "synthetics-*-*" templates for synthetics fleet data

For the Elastic Agent we currently have `logs` and `metrics`, however, synthetic data doesn't belong
with those and thus we should have a place for it to live. This would be data reported from
heartbeat and under the 'monitoring' category.

This commit adds a composable index template for `synthetics-*-*` indices similar to the work in
 #56709 and #57629.

Resolves #61665

docs/reference/data-streams/set-up-a-data-stream.asciidoc

@@ -110,7 +110,7 @@ template for a data stream must specify:
 
 [IMPORTANT]
 ====
-{es} has built-in index templates for the `metrics-*-*` and `logs-*-*` index
+{es} has built-in index templates for the `metrics-*-*`, `logs-*-*`, and `synthetics-*-*` index
 patterns, each with a priority of `100`.
 {ingest-guide}/ingest-management-overview.html[{agent}] uses these templates to
 create data streams. If you use {agent}, assign your index templates a priority

docs/reference/docs/index_.asciidoc

@@ -171,7 +171,7 @@ the operation automatically creates the index and applies any matching
 
 [IMPORTANT]
 ====
-{es} has built-in index templates for the `metrics-*-*` and `logs-*-*` index
+{es} has built-in index templates for the `metrics-*-*`, `logs-*-*`, and `synthetics-*-*` index
 patterns, each with a priority of `100`.
 {ingest-guide}/ingest-management-overview.html[{agent}] uses these templates to
 create data streams. If you use {agent}, assign your index templates a priority

docs/reference/indices/index-templates.asciidoc

@@ -22,7 +22,7 @@ If a new data stream or index matches more than one index template, the index te
 
 [IMPORTANT]
 ====
-{es} has built-in index templates for the `metrics-*-*` and `logs-*-*` index
+{es} has built-in index templates for the `metrics-*-*`, `logs-*-*`, and `synthetics-*-*` index
 patterns, each with a priority of `100`.
 {ingest-guide}/ingest-management-overview.html[{agent}] uses these templates to
 create data streams. If you use {agent}, assign your index templates a priority

docs/reference/indices/put-index-template.asciidoc

@@ -86,7 +86,7 @@ used to match the names of data streams and indices during creation.
 +
 [IMPORTANT]
 ====
-{es} has built-in index templates for the `metrics-*-*` and `logs-*-*` index
+{es} has built-in index templates for the `metrics-*-*`, `logs-*-*`, and `synthetics-*-*` index
 patterns, each with a priority of `100`.
 {ingest-guide}/ingest-management-overview.html[{agent}] uses these templates to
 create data streams. If you use {agent}, assign your index templates a priority

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -1314,6 +1314,9 @@ protected static boolean isXPackTemplate(String name) {
             case "metrics":
             case "metrics-settings":
             case "metrics-mappings":
+            case "synthetics":
+            case "synthetics-settings":
+            case "synthetics-mappings":
             case ".snapshot-blob-cache":
                 return true;
             default:

x-pack/plugin/core/src/main/resources/synthetics-mappings.json

@@ -0,0 +1,50 @@
+{
+  "template": {
+    "mappings": {
+      "dynamic_templates": [
+        {
+          "strings_as_keyword": {
+            "mapping": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            },
+            "match_mapping_type": "string"
+          }
+        }
+      ],
+      "date_detection": false,
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "data_stream": {
+          "properties": {
+            "type": {
+              "type": "constant_keyword",
+              "value": "synthetics"
+            },
+            "dataset": {
+              "type": "constant_keyword"
+            },
+            "namespace": {
+              "type": "constant_keyword"
+            }
+          }
+        },
+        "ecs": {
+          "properties": {
+            "version": {
+              "ignore_above": 1024,
+              "type": "keyword"
+            }
+          }
+        }
+      }
+    }
+  },
+  "_meta": {
+    "description": "default mappings for the synthetics index template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version}
+}

x-pack/plugin/core/src/main/resources/synthetics-policy.json

@@ -0,0 +1,12 @@
+{
+  "phases": {
+    "hot": {
+      "actions": {
+        "rollover": {
+          "max_size": "50gb",
+          "max_age": "30d"
+        }
+      }
+    }
+  }
+}

x-pack/plugin/core/src/main/resources/synthetics-settings.json

@@ -0,0 +1,17 @@
+{
+  "template": {
+    "settings": {
+      "index": {
+        "lifecycle": {
+          "name": "synthetics"
+        },
+        "codec": "best_compression"
+      }
+    }
+  },
+  "_meta": {
+    "description": "default settings for the synthetics index template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version}
+}

x-pack/plugin/core/src/main/resources/synthetics-template.json

@@ -0,0 +1,14 @@
+{
+  "index_patterns": ["synthetics-*-*"],
+  "priority": 100,
+  "data_stream": {},
+  "composed_of": [
+    "synthetics-mappings",
+    "synthetics-settings"
+  ],
+  "_meta": {
+    "description": "default synthetics template installed by x-pack",
+    "managed": true
+  },
+  "version": ${xpack.stack.template.version}
+}

x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java

@@ -90,6 +90,37 @@ public class StackTemplateRegistry extends IndexTemplateRegistry {
         TEMPLATE_VERSION_VARIABLE
     );
 
+    //////////////////////////////////////////////////////////
+    // Synthetics components (for matching synthetics-*-* indices)
+    //////////////////////////////////////////////////////////
+    public static final String SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "synthetics-mappings";
+    public static final String SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME = "synthetics-settings";
+    public static final String SYNTHETICS_ILM_POLICY_NAME = "synthetics";
+    public static final String SYNTHETICS_INDEX_TEMPLATE_NAME = "synthetics";
+
+    public static final IndexTemplateConfig SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig(
+        SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
+        "/synthetics-mappings.json",
+        REGISTRY_VERSION,
+        TEMPLATE_VERSION_VARIABLE
+    );
+    public static final IndexTemplateConfig SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig(
+        SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME,
+        "/synthetics-settings.json",
+        REGISTRY_VERSION,
+        TEMPLATE_VERSION_VARIABLE
+    );
+    public static final LifecyclePolicyConfig SYNTHETICS_ILM_POLICY = new LifecyclePolicyConfig(
+        SYNTHETICS_ILM_POLICY_NAME,
+        "/synthetics-policy.json"
+    );
+    public static final IndexTemplateConfig SYNTHETICS_INDEX_TEMPLATE = new IndexTemplateConfig(
+        SYNTHETICS_INDEX_TEMPLATE_NAME,
+        "/synthetics-template.json",
+        REGISTRY_VERSION,
+        TEMPLATE_VERSION_VARIABLE
+    );
+
     public StackTemplateRegistry(
         Settings nodeSettings,
         ClusterService clusterService,
@@ -104,7 +135,7 @@ public StackTemplateRegistry(
     @Override
     protected List<LifecyclePolicyConfig> getPolicyConfigs() {
         if (stackTemplateEnabled) {
-            return Arrays.asList(LOGS_ILM_POLICY, METRICS_ILM_POLICY);
+            return Arrays.asList(LOGS_ILM_POLICY, METRICS_ILM_POLICY, SYNTHETICS_ILM_POLICY);
         } else {
             return Collections.emptyList();
         }
@@ -117,7 +148,9 @@ protected List<IndexTemplateConfig> getComponentTemplateConfigs() {
                 LOGS_MAPPINGS_COMPONENT_TEMPLATE,
                 LOGS_SETTINGS_COMPONENT_TEMPLATE,
                 METRICS_MAPPINGS_COMPONENT_TEMPLATE,
-                METRICS_SETTINGS_COMPONENT_TEMPLATE
+                METRICS_SETTINGS_COMPONENT_TEMPLATE,
+                SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE,
+                SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE
             );
         } else {
             return Collections.emptyList();
@@ -127,7 +160,7 @@ protected List<IndexTemplateConfig> getComponentTemplateConfigs() {
     @Override
     protected List<IndexTemplateConfig> getComposableTemplateConfigs() {
         if (stackTemplateEnabled) {
-            return Arrays.asList(LOGS_INDEX_TEMPLATE, METRICS_INDEX_TEMPLATE);
+            return Arrays.asList(LOGS_INDEX_TEMPLATE, METRICS_INDEX_TEMPLATE, SYNTHETICS_INDEX_TEMPLATE);
         } else {
             return Collections.emptyList();
         }

x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java

@@ -150,7 +150,11 @@ public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception {
                 final PutLifecycleAction.Request putRequest = (PutLifecycleAction.Request) request;
                 assertThat(
                     putRequest.getPolicy().getName(),
-                    anyOf(equalTo(StackTemplateRegistry.LOGS_ILM_POLICY_NAME), equalTo(StackTemplateRegistry.METRICS_ILM_POLICY_NAME))
+                    anyOf(
+                        equalTo(StackTemplateRegistry.LOGS_ILM_POLICY_NAME),
+                        equalTo(StackTemplateRegistry.METRICS_ILM_POLICY_NAME),
+                        equalTo(StackTemplateRegistry.SYNTHETICS_ILM_POLICY_NAME)
+                    )
                 );
                 assertNotNull(listener);
                 return new PutLifecycleAction.Response(true);
@@ -168,7 +172,7 @@ public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception {
 
         ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes);
         registry.clusterChanged(event);
-        assertBusy(() -> assertThat(calledTimes.get(), equalTo(2)));
+        assertBusy(() -> assertThat(calledTimes.get(), equalTo(3)));
     }
 
     public void testPolicyAlreadyExists() {
@@ -180,7 +184,7 @@ public void testPolicyAlreadyExists() {
             .stream()
             .map(policyConfig -> policyConfig.load(xContentRegistry))
             .collect(Collectors.toList());
-        assertThat(policies, hasSize(2));
+        assertThat(policies, hasSize(3));
         policies.forEach(p -> policyMap.put(p.getName(), p));
 
         client.setVerifier((action, request, listener) -> {
@@ -209,7 +213,7 @@ public void testPolicyAlreadyExistsButDiffers() throws IOException {
             .stream()
             .map(policyConfig -> policyConfig.load(xContentRegistry))
             .collect(Collectors.toList());
-        assertThat(policies, hasSize(2));
+        assertThat(policies, hasSize(3));
         policies.forEach(p -> policyMap.put(p.getName(), p));
 
         client.setVerifier((action, request, listener) -> {
@@ -276,6 +280,8 @@ public void testSameOrHigherVersionTemplateNotUpgraded() {
         versions.put(StackTemplateRegistry.LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         versions.put(StackTemplateRegistry.METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         versions.put(StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
+        versions.put(StackTemplateRegistry.SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
+        versions.put(StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION);
         ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(versions, nodes);
         client.setVerifier((action, request, listener) -> {
             if (action instanceof PutComponentTemplateAction) {
@@ -311,6 +317,14 @@ public void testSameOrHigherVersionTemplateNotUpgraded() {
             StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
             StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
         );
+        versions.put(
+            StackTemplateRegistry.SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME,
+            StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
+        );
+        versions.put(
+            StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME,
+            StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)
+        );
         ClusterChangedEvent higherVersionEvent = createClusterChangedEvent(versions, nodes);
         registry.clusterChanged(higherVersionEvent);
     }