diff --git a/x-pack/plugin/core/src/main/resources/data-streams-mappings.json b/x-pack/plugin/core/src/main/resources/data-streams-mappings.json new file mode 100644 index 0000000000000..f87c0e79b7c45 --- /dev/null +++ b/x-pack/plugin/core/src/main/resources/data-streams-mappings.json @@ -0,0 +1,67 @@ +{ + "template": { + "mappings": { + "dynamic_templates": [ + { + "match_ip": { + "match_mapping_type": "string", + "match": "ip", + "mapping": { + "type": "ip" + } + } + }, + { + "match_message": { + "match_mapping_type": "string", + "match": "message", + "mapping": { + "type": "match_only_text" + } + } + }, + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "date_detection": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "data_stream": { + "properties": { + "dataset": { + "type": "constant_keyword" + }, + "namespace": { + "type": "constant_keyword" + } + } + }, + "ecs": { + "properties": { + "version": { + "ignore_above": 1024, + "type": "keyword" + } + } + }, + "host": { + "type": "object" + } + } + } + }, + "_meta": { + "description": "general mapping conventions for data streams", + "managed": true + }, + "version": ${xpack.stack.template.version} +} diff --git a/x-pack/plugin/core/src/main/resources/logs-mappings.json b/x-pack/plugin/core/src/main/resources/logs-mappings.json index 04ac179a04b5a..1851f36947cc8 100644 --- a/x-pack/plugin/core/src/main/resources/logs-mappings.json +++ b/x-pack/plugin/core/src/main/resources/logs-mappings.json @@ -1,53 +1,14 @@ { "template": { "mappings": { - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "date_detection": false, "properties": { - "@timestamp": { - "type": "date" - }, "data_stream": { "properties": { "type": { "type": "constant_keyword", "value": "logs" - }, - "dataset": { - "type": "constant_keyword" - }, - "namespace": { - "type": "constant_keyword" - } - } - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "host": { - "properties": { - "ip": { - "type": "ip" } } - }, - "message": { - "type": "text" } } } diff --git a/x-pack/plugin/core/src/main/resources/logs-template.json b/x-pack/plugin/core/src/main/resources/logs-template.json index 9c21a2b6ae504..198e0d04ab5af 100644 --- a/x-pack/plugin/core/src/main/resources/logs-template.json +++ b/x-pack/plugin/core/src/main/resources/logs-template.json @@ -4,6 +4,7 @@ "data_stream": {}, "composed_of": [ "logs-mappings", + "data-streams-mappings", "logs-settings" ], "allow_auto_create": true, diff --git a/x-pack/plugin/core/src/main/resources/metrics-mappings.json b/x-pack/plugin/core/src/main/resources/metrics-mappings.json index d37cdc3f4c53b..e41863f282dce 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-mappings.json +++ b/x-pack/plugin/core/src/main/resources/metrics-mappings.json @@ -1,48 +1,12 @@ { "template": { "mappings": { - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "date_detection": false, "properties": { - "@timestamp": { - "type": "date" - }, "data_stream": { "properties": { "type": { "type": "constant_keyword", "value": "metrics" - }, - "dataset": { - "type": "constant_keyword" - }, - "namespace": { - "type": "constant_keyword" - } - } - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "host": { - "properties": { - "ip": { - "type": "ip" } } } diff --git a/x-pack/plugin/core/src/main/resources/metrics-template.json b/x-pack/plugin/core/src/main/resources/metrics-template.json index 0da4169de79b7..b0c4308722912 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-template.json +++ b/x-pack/plugin/core/src/main/resources/metrics-template.json @@ -4,6 +4,7 @@ "data_stream": {}, "composed_of": [ "metrics-mappings", + "data-streams-mappings", "metrics-settings" ], "allow_auto_create": true, diff --git a/x-pack/plugin/core/src/main/resources/synthetics-mappings.json b/x-pack/plugin/core/src/main/resources/synthetics-mappings.json index ca159c4968bef..9e3e56e3261d0 100644 --- a/x-pack/plugin/core/src/main/resources/synthetics-mappings.json +++ b/x-pack/plugin/core/src/main/resources/synthetics-mappings.json @@ -1,55 +1,12 @@ { "template": { "mappings": { - "dynamic_templates": [ - { - "strings_as_keyword": { - "mapping": { - "ignore_above": 1024, - "type": "keyword" - }, - "match_mapping_type": "string" - } - } - ], - "date_detection": false, "properties": { - "@timestamp": { - "type": "date" - }, "data_stream": { "properties": { "type": { "type": "constant_keyword", "value": "synthetics" - }, - "dataset": { - "type": "constant_keyword" - }, - "namespace": { - "type": "constant_keyword" - } - } - }, - "ecs": { - "properties": { - "version": { - "ignore_above": 1024, - "type": "keyword" - } - } - }, - "host": { - "properties": { - "ip": { - "type": "ip" - } - } - }, - "observer": { - "properties": { - "ip": { - "type": "ip" } } } diff --git a/x-pack/plugin/core/src/main/resources/synthetics-template.json b/x-pack/plugin/core/src/main/resources/synthetics-template.json index 1036a29cf5a9d..0e292f3d8694f 100644 --- a/x-pack/plugin/core/src/main/resources/synthetics-template.json +++ b/x-pack/plugin/core/src/main/resources/synthetics-template.json @@ -4,6 +4,7 @@ "data_stream": {}, "composed_of": [ "synthetics-mappings", + "data-streams-mappings", "synthetics-settings" ], "allow_auto_create": true, diff --git a/x-pack/plugin/stack/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml b/x-pack/plugin/stack/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml index dda4652afc663..4138d1577bcf4 100644 --- a/x-pack/plugin/stack/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml +++ b/x-pack/plugin/stack/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml @@ -14,6 +14,10 @@ setup: ilm.get_lifecycle: policy: "metrics" + - do: + cluster.get_component_template: + name: data-streams-mappings + - do: cluster.get_component_template: name: logs-mappings @@ -46,6 +50,11 @@ setup: body: "@timestamp": "2020-01-01" message: "test-log-message" + source.ip: "10.1.2.3" + log.file.path: "/var/log/web/access.log" + data_stream.type: "logs" + data_stream.dataset: "foo" + data_stream.namespace: "bar" - do: indices.get_data_stream: @@ -66,7 +75,16 @@ setup: - is_true: .$idx0name.settings - is_true: .$idx0name.mappings - match: { .$idx0name.settings.index.lifecycle.name: "logs" } + - match: { .$idx0name.mappings.properties.data_stream.properties.type.type: "constant_keyword" } + - match: { .$idx0name.mappings.properties.data_stream.properties.type.value: "logs" } + - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.type: "constant_keyword" } + - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.value: "foo" } + - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.type: "constant_keyword" } + - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.value: "bar" } - is_true: .$idx0name.mappings.properties.message + - match: { .$idx0name.mappings.properties.message.type: "match_only_text" } + - match: { .$idx0name.mappings.properties.source.properties.ip.type: "ip" } + - match: { .$idx0name.mappings.properties.log.properties.file.properties.path.type: "keyword" } - match: { .$idx0name.data_stream: "logs-foo-bar" } - do: @@ -80,7 +98,10 @@ setup: index: metrics-foo-bar body: "@timestamp": "2020-01-01" - message: "test-log-message" + source.ip: "10.1.2.3" + data_stream.type: "metrics" + data_stream.dataset: "foo" + data_stream.namespace: "bar" - do: indices.get_data_stream: @@ -101,9 +122,92 @@ setup: - is_true: .$idx0name.settings - is_true: .$idx0name.mappings - match: { .$idx0name.settings.index.lifecycle.name: "metrics" } - - is_true: .$idx0name.mappings.properties.message + - match: { .$idx0name.mappings.properties.data_stream.properties.type.type: "constant_keyword" } + - match: { .$idx0name.mappings.properties.data_stream.properties.type.value: "metrics" } + - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.type: "constant_keyword" } + - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.value: "foo" } + - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.type: "constant_keyword" } + - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.value: "bar" } + - match: { .$idx0name.mappings.properties.source.properties.ip.type: "ip" } - match: { .$idx0name.data_stream: "metrics-foo-bar" } - do: indices.delete_data_stream: name: metrics-foo-bar + +--- +"Test synthetics index auto creation": + - do: + index: + index: synthetics-foo-bar + body: + "@timestamp": "2020-01-01" + source.ip: "10.1.2.3" + data_stream.type: "synthetics" + data_stream.dataset: "foo" + data_stream.namespace: "bar" + + - do: + indices.get_data_stream: + name: synthetics-foo-bar + + - match: { data_streams.0.name: synthetics-foo-bar } + - match: { data_streams.0.timestamp_field.name: '@timestamp' } + - match: { data_streams.0.generation: 1 } + - length: { data_streams.0.indices: 1 } + - match: { data_streams.0.indices.0.index_name: '/\.ds-synthetics-foo-bar-(\d{4}\.\d{2}\.\d{2}-)?000001/' } + + - set: { data_streams.0.indices.0.index_name: idx0name } + + - do: + indices.get: + index: $idx0name + + - is_true: .$idx0name.settings + - is_true: .$idx0name.mappings + - match: { .$idx0name.settings.index.lifecycle.name: "synthetics" } + - match: { .$idx0name.mappings.properties.data_stream.properties.type.type: "constant_keyword" } + - match: { .$idx0name.mappings.properties.data_stream.properties.type.value: "synthetics" } + - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.type: "constant_keyword" } + - match: { .$idx0name.mappings.properties.data_stream.properties.dataset.value: "foo" } + - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.type: "constant_keyword" } + - match: { .$idx0name.mappings.properties.data_stream.properties.namespace.value: "bar" } + - match: { .$idx0name.mappings.properties.source.properties.ip.type: "ip" } + - match: { .$idx0name.data_stream: "synthetics-foo-bar" } + + - do: + indices.delete_data_stream: + name: synthetics-foo-bar + +--- +"Test wrong data_stream type": + + - do: + catch: bad_request + index: + index: synthetics-dataset0-namespace1 + body: + "@timestamp": "2020-01-01" + data_stream.type: "logs" + data_stream.dataset: "dataset0" + data_stream.namespace: "namespace1" + + - do: + catch: bad_request + index: + index: logs-dataset0-namespace1 + body: + "@timestamp": "2020-01-01" + data_stream.type: "metrics" + data_stream.dataset: "dataset0" + data_stream.namespace: "namespace1" + + - do: + catch: bad_request + index: + index: metrics-dataset0-namespace1 + body: + "@timestamp": "2020-01-01" + data_stream.type: "synthetics" + data_stream.dataset: "dataset0" + data_stream.namespace: "namespace1" diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index c1bd65f70a20a..4170e338c8578 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -28,11 +28,9 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { private static final Logger logger = LogManager.getLogger(StackTemplateRegistry.class); - // The stack template registry should remain at version 0. This is because templates and - // policies will be changed by the ingest manager once they exist, and ES should only ever put - // the template in place if it does not exist. If this were incremented we could accidentally - // overwrite a template or policy changed by the ingest manager. - public static final int REGISTRY_VERSION = 0; + // The stack template registry version. This number must be incremented when we make changes + // to built-in templates. + public static final int REGISTRY_VERSION = 1; public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version"; public static final Setting STACK_TEMPLATES_ENABLED = Setting.boolSetting( @@ -45,6 +43,16 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { private final ClusterService clusterService; private volatile boolean stackTemplateEnabled; + // General mappings conventions for any data that ends up in a data stream + public static final String DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME = "data-streams-mappings"; + + public static final IndexTemplateConfig DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE = new IndexTemplateConfig( + DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + "/data-streams-mappings.json", + REGISTRY_VERSION, + TEMPLATE_VERSION_VARIABLE + ); + ////////////////////////////////////////////////////////// // Logs components (for matching logs-*-* indices) ////////////////////////////////////////////////////////// @@ -179,6 +187,7 @@ protected List getPolicyConfigs() { protected List getComponentTemplateConfigs() { if (stackTemplateEnabled) { return Arrays.asList( + DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE, LOGS_MAPPINGS_COMPONENT_TEMPLATE, LOGS_SETTINGS_COMPONENT_TEMPLATE, METRICS_MAPPINGS_COMPONENT_TEMPLATE, diff --git a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java index e069075e5795d..47c25ae239bc3 100644 --- a/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java +++ b/x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java @@ -277,6 +277,7 @@ public void testSameOrHigherVersionTemplateNotUpgraded() { DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); Map versions = new HashMap<>(); + versions.put(StackTemplateRegistry.DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); versions.put(StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); versions.put(StackTemplateRegistry.LOGS_MAPPINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); versions.put(StackTemplateRegistry.METRICS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION); @@ -302,6 +303,10 @@ public void testSameOrHigherVersionTemplateNotUpgraded() { registry.clusterChanged(sameVersionEvent); versions.clear(); + versions.put( + StackTemplateRegistry.DATA_STREAMS_MAPPINGS_COMPONENT_TEMPLATE_NAME, + StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000) + ); versions.put( StackTemplateRegistry.LOGS_SETTINGS_COMPONENT_TEMPLATE_NAME, StackTemplateRegistry.REGISTRY_VERSION + randomIntBetween(1, 1000)