From 6b2af30a62782350512d7af5e3c783f43181da0d Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Mon, 14 Sep 2020 17:14:34 -0600 Subject: [PATCH] [7.x] Add "synthetics-*-*" templates for synthetics fleet data (#62193) (#62346) * Add "synthetics-*-*" templates for synthetics fleet data For the Elastic Agent we currently have `logs` and `metrics`, however, synthetic data doesn't belong with those and thus we should have a place for it to live. This would be data reported from heartbeat and under the 'monitoring' category. This commit adds a composable index template for `synthetics-*-*` indices similar to the work in #56709 and #57629. Resolves #61665 --- .../set-up-a-data-stream.asciidoc | 2 +- docs/reference/docs/index_.asciidoc | 2 +- .../indices/index-templates.asciidoc | 2 +- .../indices/put-index-template.asciidoc | 2 +- .../test/rest/ESRestTestCase.java | 3 ++ .../main/resources/synthetics-mappings.json | 50 +++++++++++++++++++ .../src/main/resources/synthetics-policy.json | 12 +++++ .../main/resources/synthetics-settings.json | 17 +++++++ .../main/resources/synthetics-template.json | 14 ++++++ .../xpack/stack/StackTemplateRegistry.java | 39 +++++++++++++-- .../stack/StackTemplateRegistryTests.java | 22 ++++++-- 11 files changed, 154 insertions(+), 11 deletions(-) create mode 100644 x-pack/plugin/core/src/main/resources/synthetics-mappings.json create mode 100644 x-pack/plugin/core/src/main/resources/synthetics-policy.json create mode 100644 x-pack/plugin/core/src/main/resources/synthetics-settings.json create mode 100644 x-pack/plugin/core/src/main/resources/synthetics-template.json diff --git a/docs/reference/data-streams/set-up-a-data-stream.asciidoc b/docs/reference/data-streams/set-up-a-data-stream.asciidoc index 84f627c731bcf..4ba5c654d4783 100644 --- a/docs/reference/data-streams/set-up-a-data-stream.asciidoc +++ b/docs/reference/data-streams/set-up-a-data-stream.asciidoc @@ -110,7 +110,7 @@ template for a data stream must specify: [IMPORTANT] ==== -{es} has built-in index templates for the `metrics-*-*` and `logs-*-*` index +{es} has built-in index templates for the `metrics-*-*`, `logs-*-*`, and `synthetics-*-*` index patterns, each with a priority of `100`. {ingest-guide}/ingest-management-overview.html[{agent}] uses these templates to create data streams. If you use {agent}, assign your index templates a priority diff --git a/docs/reference/docs/index_.asciidoc b/docs/reference/docs/index_.asciidoc index ee69dcefb791b..3e3eac028990d 100644 --- a/docs/reference/docs/index_.asciidoc +++ b/docs/reference/docs/index_.asciidoc @@ -171,7 +171,7 @@ the operation automatically creates the index and applies any matching [IMPORTANT] ==== -{es} has built-in index templates for the `metrics-*-*` and `logs-*-*` index +{es} has built-in index templates for the `metrics-*-*`, `logs-*-*`, and `synthetics-*-*` index patterns, each with a priority of `100`. {ingest-guide}/ingest-management-overview.html[{agent}] uses these templates to create data streams. If you use {agent}, assign your index templates a priority diff --git a/docs/reference/indices/index-templates.asciidoc b/docs/reference/indices/index-templates.asciidoc index c5a0ce7aeac08..4e5aa636fc139 100644 --- a/docs/reference/indices/index-templates.asciidoc +++ b/docs/reference/indices/index-templates.asciidoc @@ -22,7 +22,7 @@ If a new data stream or index matches more than one index template, the index te [IMPORTANT] ==== -{es} has built-in index templates for the `metrics-*-*` and `logs-*-*` index +{es} has built-in index templates for the `metrics-*-*`, `logs-*-*`, and `synthetics-*-*` index patterns, each with a priority of `100`. {ingest-guide}/ingest-management-overview.html[{agent}] uses these templates to create data streams. If you use {agent}, assign your index templates a priority diff --git a/docs/reference/indices/put-index-template.asciidoc b/docs/reference/indices/put-index-template.asciidoc index 03748258d5576..d0989eac528e3 100644 --- a/docs/reference/indices/put-index-template.asciidoc +++ b/docs/reference/indices/put-index-template.asciidoc @@ -86,7 +86,7 @@ used to match the names of data streams and indices during creation. + [IMPORTANT] ==== -{es} has built-in index templates for the `metrics-*-*` and `logs-*-*` index +{es} has built-in index templates for the `metrics-*-*`, `logs-*-*`, and `synthetics-*-*` index patterns, each with a priority of `100`. {ingest-guide}/ingest-management-overview.html[{agent}] uses these templates to create data streams. If you use {agent}, assign your index templates a priority diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 2179e40ea7dae..bc41b9de2baaf 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1314,6 +1314,9 @@ protected static boolean isXPackTemplate(String name) { case "metrics": case "metrics-settings": case "metrics-mappings": + case "synthetics": + case "synthetics-settings": + case "synthetics-mappings": case ".snapshot-blob-cache": return true; default: diff --git a/x-pack/plugin/core/src/main/resources/synthetics-mappings.json b/x-pack/plugin/core/src/main/resources/synthetics-mappings.json new file mode 100644 index 0000000000000..ffaa374cc731a --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/synthetics-mappings.json @@ -0,0 +1,50 @@ +{ + "template": { + "mappings": { + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "date_detection": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "data_stream": { + "properties": { + "type": { + "type": "constant_keyword", + "value": "synthetics" + }, + "dataset": { + "type": "constant_keyword" + }, + "namespace": { + "type": "constant_keyword" + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + } + } + }, + "_meta": { + "description": "default mappings for the synthetics index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/synthetics-policy.json b/x-pack/plugin/core/src/main/resources/synthetics-policy.json new file mode 100644 index 0000000000000..936bfe6c0e2e3 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/synthetics-policy.json @@ -0,0 +1,12 @@ +{ + "phases": { + "hot": { + "actions": { + "rollover": { + "max_size": "50gb", + "max_age": "30d" + } + } + } + } +} diff --git a/x-pack/plugin/core/src/main/resources/synthetics-settings.json b/x-pack/plugin/core/src/main/resources/synthetics-settings.json new file mode 100644 index 0000000000000..27ced96be36e3 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/synthetics-settings.json @@ -0,0 +1,17 @@ +{ + "template": { + "settings": { + "index": { + "lifecycle": { + "name": "synthetics" + }, + "codec": "best_compression" + } + } + }, + "_meta": { + "description": "default settings for the synthetics index template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/synthetics-template.json b/x-pack/plugin/core/src/main/resources/synthetics-template.json new file mode 100644 index 0000000000000..9bbbe0ef9e3df --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/synthetics-template.json @@ -0,0 +1,14 @@ +{ + "index_patterns": ["synthetics-*-*"], + "priority": 100, + "data_stream": {}, + "composed_of": [ + "synthetics-mappings", + "synthetics-settings" + ], + "_meta": { + "description": "default synthetics template installed by x-pack", + "managed": true + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index e1ae7282650d9..424539b9aeda9 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -90,6 +90,37 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { TEMPLATE_VERSION_VARIABLE ); + ////////////////////////////////////////////////////////// + // Synthetics components (for matching synthetics-*-* indices) + ////////////////////////////////////////////////////////// + public static final String SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "synthetics-mappings"; + public static final String SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME = "synthetics-settings"; + public static final String SYNTHETICS_ILM_POLICY_NAME = "synthetics"; + public static final String SYNTHETICS_INDEX_TEMPLATE_NAME = "synthetics"; + + public static final IndexTemplateConfig SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( + SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/synthetics-mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + public static final IndexTemplateConfig SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( + SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME, + "/synthetics-settings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + public static final LifecyclePolicyConfig SYNTHETICS_ILM_POLICY = new LifecyclePolicyConfig( + SYNTHETICS_ILM_POLICY_NAME, + "/synthetics-policy.json" + ); + public static final IndexTemplateConfig SYNTHETICS_INDEX_TEMPLATE = new IndexTemplateConfig( + SYNTHETICS_INDEX_TEMPLATE_NAME, + "/synthetics-template.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + public StackTemplateRegistry( Settings nodeSettings, ClusterService clusterService, @@ -104,7 +135,7 @@ public StackTemplateRegistry( @Override protected List getPolicyConfigs() { if (stackTemplateEnabled) { - return Arrays.asList(LOGS_ILM_POLICY, METRICS_ILM_POLICY); + return Arrays.asList(LOGS_ILM_POLICY, METRICS_ILM_POLICY, SYNTHETICS_ILM_POLICY); } else { return Collections.emptyList(); } @@ -117,7 +148,9 @@ protected List getComponentTemplateConfigs() { LOGS_MAPPINGS_COMPONENT_TEMPLATE, LOGS_SETTINGS_COMPONENT_TEMPLATE, METRICS_MAPPINGS_COMPONENT_TEMPLATE, - METRICS_SETTINGS_COMPONENT_TEMPLATE + METRICS_SETTINGS_COMPONENT_TEMPLATE, + SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE, + SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE ); } else { return Collections.emptyList(); @@ -127,7 +160,7 @@ protected List getComponentTemplateConfigs() { @Override protected List getComposableTemplateConfigs() { if (stackTemplateEnabled) { - return Arrays.asList(LOGS_INDEX_TEMPLATE, METRICS_INDEX_TEMPLATE); + return Arrays.asList(LOGS_INDEX_TEMPLATE, METRICS_INDEX_TEMPLATE, SYNTHETICS_INDEX_TEMPLATE); } else { return Collections.emptyList(); } diff --git a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java index 7ef3d710df9fd..fcfa904f97464 100644 --- a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java +++ b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java @@ -150,7 +150,11 @@ public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception { final PutLifecycleAction.Request putRequest = (PutLifecycleAction.Request) request; assertThat( putRequest.getPolicy().getName(), - anyOf(equalTo(StackTemplateRegistry.LOGS_ILM_POLICY_NAME), equalTo(StackTemplateRegistry.METRICS_ILM_POLICY_NAME)) + anyOf( + equalTo(StackTemplateRegistry.LOGS_ILM_POLICY_NAME), + equalTo(StackTemplateRegistry.METRICS_ILM_POLICY_NAME), + equalTo(StackTemplateRegistry.SYNTHETICS_ILM_POLICY_NAME) + ) ); assertNotNull(listener); return new PutLifecycleAction.Response(true); @@ -168,7 +172,7 @@ public void testThatNonExistingPoliciesAreAddedImmediately() throws Exception { ClusterChangedEvent event = createClusterChangedEvent(Collections.emptyMap(), nodes); registry.clusterChanged(event); - assertBusy(() -> assertThat(calledTimes.get(), equalTo(2))); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(3))); } public void testPolicyAlreadyExists() { @@ -180,7 +184,7 @@ public void testPolicyAlreadyExists() { .stream() .map(policyConfig -> policyConfig.load(xContentRegistry)) .collect(Collectors.toList()); - assertThat(policies, hasSize(2)); + assertThat(policies, hasSize(3)); policies.forEach(p -> policyMap.put(p.getName(), p)); client.setVerifier((action, request, listener) -> { @@ -209,7 +213,7 @@ public void testPolicyAlreadyExistsButDiffers() throws IOException { .stream() .map(policyConfig -> policyConfig.load(xContentRegistry)) .collect(Collectors.toList()); - assertThat(policies, hasSize(2)); + assertThat(policies, hasSize(3)); policies.forEach(p -> policyMap.put(p.getName(), p)); client.setVerifier((action, request, listener) -> { @@ -276,6 +280,8 @@ public void testSameOrHigherVersionTemplateNotUpgraded() { versions.put(StackTemplateRegistry.LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); versions.put(StackTemplateRegistry.METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); versions.put(StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); + versions.put(StackTemplateRegistry.SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); + versions.put(StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); ClusterChangedEvent sameVersionEvent = createClusterChangedEvent(versions, nodes); client.setVerifier((action, request, listener) -> { if (action instanceof PutComponentTemplateAction) { @@ -311,6 +317,14 @@ public void testSameOrHigherVersionTemplateNotUpgraded() { StackTemplateRegistry.METRICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) ); + versions.put( + StackTemplateRegistry.SYNTHETICS_SETTINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) + ); + versions.put( + StackTemplateRegistry.SYNTHETICS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) + ); ClusterChangedEvent higherVersionEvent = createClusterChangedEvent(versions, nodes); registry.clusterChanged(higherVersionEvent); }