From 15597b483f979a8466de1874add5f8fe74c1a404 Mon Sep 17 00:00:00 2001 From: weizijun Date: Tue, 26 Oct 2021 22:54:00 +0800 Subject: [PATCH] TSDB: Automatically add timestamp mapper (#79136) If tsdb is enabled we need an `@timestamp` field. This automatically maps the field if it is missing and fails to create indices in time_series mode that map `@timestamp` as anything other than `date` and `date_nanos`. --- .../test/tsdb/15_timestamp_mapping.yml | 179 ++++++++++++++++++ .../template/ComposableTemplateIT.java | 27 --- .../MetadataIndexTemplateService.java | 12 -- .../org/elasticsearch/index/IndexMode.java | 57 ++++++ .../DataStreamTimestampFieldMapper.java | 4 +- .../index/mapper/MappingParser.java | 7 +- .../index/mapper/ObjectMapper.java | 5 + .../elasticsearch/indices/IndicesModule.java | 2 + .../MetadataDataStreamsServiceTests.java | 7 - .../MetadataIndexTemplateServiceTests.java | 30 +-- ...tadataMigrateToDataStreamServiceTests.java | 7 - .../index/TimeSeriesModeTests.java | 133 +++++++++++++ .../index/mapper/DocumentMapperTests.java | 6 +- .../index/mapper/MapperServiceTests.java | 8 +- .../indices/IndicesModuleTests.java | 17 +- .../xpack/datastreams/DataStreamsPlugin.java | 11 +- .../MetadataCreateDataStreamServiceTests.java | 3 +- 17 files changed, 410 insertions(+), 105 deletions(-) create mode 100644 rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml new file mode 100644 index 0000000000000..9d61d4c359b6d --- /dev/null +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml @@ -0,0 +1,179 @@ + +--- +date: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 to be backported to 7.16.0 + + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + routing_path: [metricset] + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date + metricset: + type: keyword + time_series_dimension: true + + - do: + indices.get_mapping: + index: test + - match: { "test.mappings.properties.@timestamp.type": date } + - match: { 'test.mappings._data_stream_timestamp.enabled': true } + + - do: + bulk: + refresh: true + index: test_index + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod"}' + + - do: + search: + index: test_index + body: + docvalue_fields: [ '@timestamp' ] + - match: {hits.total.value: 1} + - match: { "hits.hits.0.fields.@timestamp": ["2021-04-28T18:50:04.467Z"] } + +--- +date_nanos: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 to be backported to 7.16.0 + + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + routing_path: [metricset] + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: date_nanos + metricset: + type: keyword + time_series_dimension: true + + - do: + indices.get_mapping: + index: test + - match: { "test.mappings.properties.@timestamp.type": date_nanos } + - match: { 'test.mappings._data_stream_timestamp.enabled': true } + + - do: + bulk: + refresh: true + index: test_index + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod"}' + + - do: + search: + index: test_index + body: + docvalue_fields: [ '@timestamp' ] + - match: {hits.total.value: 1} + - match: { "hits.hits.0.fields.@timestamp": ["2021-04-28T18:50:04.467Z"] } + +--- +automatically add with date: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 to be backported to 7.16.0 + + - do: + indices.create: + index: test + body: + settings: + index: + mode: time_series + routing_path: [metricset] + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + metricset: + type: keyword + time_series_dimension: true + + - do: + indices.get_mapping: + index: test + - match: { 'test.mappings.properties.@timestamp': { "type": date } } + - match: { 'test.mappings._data_stream_timestamp.enabled': true } + + - do: + bulk: + refresh: true + index: test_index + body: + - '{"index": {}}' + - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod"}' + + - do: + search: + index: test_index + body: + docvalue_fields: [ '@timestamp' ] + - match: {hits.total.value: 1} + - match: { "hits.hits.0.fields.@timestamp": ["2021-04-28T18:50:04.467Z"] } + +--- +reject @timestamp with wrong type: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 to be backported to 7.16.0 + + - do: + catch: /data stream timestamp field \[@timestamp\] is of type \[keyword\], but \[date,date_nanos\] is expected/ + indices.create: + index: test + body: + settings: + index: + mode: time_series + routing_path: [metricset] + number_of_replicas: 0 + number_of_shards: 2 + mappings: + properties: + "@timestamp": + type: keyword + +--- +reject timestamp meta field with wrong type: + - skip: + version: " - 7.99.99" + reason: introduced in 8.0.0 to be backported to 7.16.0 + + - do: + catch: /.* time series index \[_data_stream_timestamp\] meta field must be enabled/ + indices.create: + index: test + body: + settings: + index: + mode: time_series + routing_path: [metricset] + number_of_replicas: 0 + number_of_shards: 2 + mappings: + _data_stream_timestamp: + enabled: false diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java index b3d05e7b38f2e..8fdd61855c32c 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java @@ -13,18 +13,10 @@ import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Template; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.compress.CompressedXContent; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.test.ESIntegTestCase; -import java.io.IOException; import java.util.Collections; -import java.util.List; - -import static org.hamcrest.Matchers.equalTo; public class ComposableTemplateIT extends ESIntegTestCase { @@ -80,23 +72,4 @@ public void testComponentTemplatesCanBeUpdatedAfterRestart() throws Exception { client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(cit2)).get(); } - - public void testUsageOfDataStreamFails() throws IOException { - // Exception that would happen if a unknown field is provided in a composable template: - // The thrown exception will be used to compare against the exception that is thrown when providing - // a composable template with a data stream definition. - String content = "{\"index_patterns\":[\"logs-*-*\"],\"my_field\":\"bla\"}"; - XContentParser parser = - XContentHelper.createParser(xContentRegistry(), null, new BytesArray(content), XContentType.JSON); - Exception expectedException = expectThrows(Exception.class, () -> ComposableIndexTemplate.parse(parser)); - - ComposableIndexTemplate template = new ComposableIndexTemplate.Builder().indexPatterns(List.of("logs-*-*")) - .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()).build(); - Exception e = expectThrows(IllegalArgumentException.class, () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, - new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(template)).actionGet()); - Exception actualException = (Exception) e.getCause(); - assertThat(actualException.getMessage(), - equalTo(expectedException.getMessage().replace("[1:32] ", "").replace("my_field", "data_stream"))); - assertThat(actualException.getMessage(), equalTo("[index_template] unknown field [data_stream]")); - } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index 959f514c0256a..b39aac641c2a3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -42,11 +42,9 @@ import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperService.MergeReason; @@ -1231,16 +1229,6 @@ private static void validateCompositeTemplate(final ClusterState state, String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; // Parse mappings to ensure they are valid after being composed - if (template.getDataStreamTemplate() != null) { - // If there is no _data_stream meta field mapper and a data stream should be created then - // fail as if the data_stream field can't be parsed: - if (tempIndexService.mapperService().isMetadataField(DataStreamTimestampFieldMapper.NAME) == false) { - // Fail like a parsing expection, since we will be moving data_stream template out of server module and - // then we would fail with the same error message, like we do here. - throw new XContentParseException("[index_template] unknown field [data_stream]"); - } - } - List mappings = collectMappings(stateWithIndex, templateName, indexName, xContentRegistry); try { MapperService mapperService = tempIndexService.mapperService(); diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 5d3600242a7e3..1e1064a4a915b 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -11,13 +11,21 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.mapper.MappingParserContext; +import org.elasticsearch.index.mapper.RootObjectMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.stream.Stream; import static java.util.stream.Collectors.toSet; @@ -44,6 +52,9 @@ void validateWithOtherSettings(Map, Object> settings) { @Override public void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting) {} + + @Override + public void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder) {} }, TIME_SERIES { @Override @@ -88,6 +99,47 @@ private String routingRequiredBad() { private String tsdbMode() { return "[" + IndexSettings.MODE.getKey() + "=time_series]"; } + + @Override + public void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder) { + if (false == mapping.containsKey(DataStreamTimestampFieldMapper.NAME)) { + mapping.put(DataStreamTimestampFieldMapper.NAME, new HashMap<>(Map.of("enabled", true))); + } else { + validateTimeStampField(mapping.get(DataStreamTimestampFieldMapper.NAME)); + } + + Optional timestamp = builder.getBuilder(DataStreamTimestampFieldMapper.DEFAULT_PATH); + if (timestamp.isEmpty()) { + builder.add( + new DateFieldMapper.Builder( + DataStreamTimestampFieldMapper.DEFAULT_PATH, + DateFieldMapper.Resolution.MILLISECONDS, + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + context.scriptCompiler(), + DateFieldMapper.IGNORE_MALFORMED_SETTING.get(context.getSettings()), + context.getIndexSettings().getIndexVersionCreated(), + context.getIndexSettings().getIndex().getName() + ) + ); + } + } + + private void validateTimeStampField(Object timestampFieldValue) { + if (false == (timestampFieldValue instanceof Map)) { + throw new IllegalArgumentException( + "time series index [" + DataStreamTimestampFieldMapper.NAME + "] meta field format error" + ); + } + + @SuppressWarnings("unchecked") + Map timeStampFieldValueMap = (Map) timestampFieldValue; + if (false == Maps.deepEquals(timeStampFieldValueMap, Map.of("enabled", true)) + && false == Maps.deepEquals(timeStampFieldValueMap, Map.of("enabled", "true"))) { + throw new IllegalArgumentException( + "time series index [" + DataStreamTimestampFieldMapper.NAME + "] meta field must be enabled" + ); + } + } }; private static final List> TIME_SERIES_UNSUPPORTED = List.of( @@ -115,4 +167,9 @@ private String tsdbMode() { * Validate aliases targeting this index. */ public abstract void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting); + + /** + * Validate and/or modify the mappings after after they've been parsed. + */ + public abstract void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java index 88b8f6305e354..616c10b052408 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DataStreamTimestampFieldMapper.java @@ -32,9 +32,9 @@ public class DataStreamTimestampFieldMapper extends MetadataFieldMapper { public static final String NAME = "_data_stream_timestamp"; - private static final String DEFAULT_PATH = "@timestamp"; + public static final String DEFAULT_PATH = "@timestamp"; - private static final DataStreamTimestampFieldMapper ENABLED_INSTANCE = + public static final DataStreamTimestampFieldMapper ENABLED_INSTANCE = new DataStreamTimestampFieldMapper(TimestampFieldType.INSTANCE, true); private static final DataStreamTimestampFieldMapper DISABLED_INSTANCE = new DataStreamTimestampFieldMapper(TimestampFieldType.INSTANCE, false); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java index 83acc10a18a4c..70dedbea0e6e4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java @@ -10,8 +10,8 @@ import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.core.Nullable; +import org.elasticsearch.xcontent.XContentType; import java.util.Collections; import java.util.HashMap; @@ -94,7 +94,8 @@ Mapping parse(@Nullable String type, CompressedXContent source) throws MapperPar private Mapping parse(String type, Map mapping) throws MapperParsingException { MappingParserContext parserContext = parserContextSupplier.get(); - RootObjectMapper rootObjectMapper = rootObjectTypeParser.parse(type, mapping, parserContext).build(MapperBuilderContext.ROOT); + RootObjectMapper.Builder rootObjectMapperBuilder = rootObjectTypeParser.parse(type, mapping, parserContext); + parserContext.getIndexSettings().getMode().completeMappings(parserContext, mapping, rootObjectMapperBuilder); Map, MetadataFieldMapper> metadataMappers = metadataMappersSupplier.get(); Map meta = null; @@ -143,7 +144,7 @@ private Mapping parse(String type, Map mapping) throws MapperPar checkNoRemainingFields(mapping, "Root mapping definition has unsupported parameters: "); return new Mapping( - rootObjectMapper, + rootObjectMapperBuilder.build(MapperBuilderContext.ROOT), metadataMappers.values().toArray(new MetadataFieldMapper[0]), meta); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 524758243ca58..afa6a338c5273 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -28,6 +28,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; public class ObjectMapper extends Mapper implements Cloneable { private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(ObjectMapper.class); @@ -86,6 +87,10 @@ public Builder add(Mapper.Builder builder) { return this; } + public Optional getBuilder(String name) { + return mappersBuilders.stream().filter(b -> b.name().equals(name)).findFirst(); + } + protected final Map buildMappers(boolean root, MapperBuilderContext context) { if (root == false) { context = context.createChildContext(name); diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 37896dc1edb46..9cf04054bd92d 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -14,6 +14,7 @@ import org.elasticsearch.action.admin.indices.rollover.MaxPrimaryShardSizeCondition; import org.elasticsearch.action.admin.indices.rollover.MaxSizeCondition; import org.elasticsearch.action.resync.TransportResyncReplicationAction; +import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -183,6 +184,7 @@ private static Map initBuiltInMetadataMa builtInMetadataMappers.put(VersionFieldMapper.NAME, VersionFieldMapper.PARSER); builtInMetadataMappers.put(SeqNoFieldMapper.NAME, SeqNoFieldMapper.PARSER); builtInMetadataMappers.put(DocCountFieldMapper.NAME, DocCountFieldMapper.PARSER); + builtInMetadataMappers.put(DataStreamTimestampFieldMapper.NAME, DataStreamTimestampFieldMapper.PARSER); //_field_names must be added last so that it has a chance to see all the other mappers builtInMetadataMappers.put(FieldNamesFieldMapper.NAME, FieldNamesFieldMapper.PARSER); return Collections.unmodifiableMap(builtInMetadataMappers); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java index 0c861e1744525..2c6ab387d50a3 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java @@ -16,11 +16,9 @@ import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; -import org.elasticsearch.plugins.Plugin; import java.io.IOException; import java.util.Arrays; -import java.util.Collection; import java.util.List; import java.util.Locale; import java.util.stream.Collectors; @@ -403,9 +401,4 @@ private MapperService getMapperService(IndexMetadata im) { throw new IllegalStateException(e); } } - - @Override - protected Collection getPlugins() { - return List.of(new MetadataIndexTemplateServiceTests.DummyPlugin()); - } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index f9f623979f5af..f66194fe9ef57 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.cluster.metadata; import com.fasterxml.jackson.core.JsonParseException; + import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; @@ -23,27 +24,22 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; -import org.elasticsearch.test.ClusterServiceUtils; -import org.elasticsearch.threadpool.ThreadPool; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; -import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.indices.EmptySystemIndices; import org.elasticsearch.indices.IndexTemplateMissingException; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidIndexTemplateException; -import org.elasticsearch.plugins.MapperPlugin; -import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentFactory; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -78,11 +74,6 @@ public class MetadataIndexTemplateServiceTests extends ESSingleNodeTestCase { - @Override - protected Collection> getPlugins() { - return List.of(DummyPlugin.class); - } - public void testLegacyNoopUpdate() { ClusterState state = ClusterState.EMPTY_STATE; PutRequest pr = new PutRequest("api", "id"); @@ -1595,15 +1586,4 @@ clusterService, createIndexService, new AliasValidator(), indicesService, public static void assertTemplatesEqual(ComposableIndexTemplate actual, ComposableIndexTemplate expected) { assertTrue(Objects.equals(actual, expected)); } - - // Composable index template with data_stream definition need _timestamp meta field mapper, - // this is a dummy impl, so that tests don't fail with the fact that the _timestamp field can't be found. - // (tests using this dummy impl doesn't test the _timestamp validation, but need it to tests other functionality) - public static class DummyPlugin extends Plugin implements MapperPlugin { - - @Override - public Map getMetadataMappers() { - return Map.of(DataStreamTimestampFieldMapper.NAME, DataStreamTimestampFieldMapper.PARSER); - } - } } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java index d89825ddecf96..7f394801c8106 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataMigrateToDataStreamServiceTests.java @@ -18,10 +18,8 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.indices.EmptySystemIndices; -import org.elasticsearch.plugins.Plugin; import java.io.IOException; -import java.util.Collection; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -326,9 +324,4 @@ private MetadataCreateIndexService getMetadataCreateIndexService() { when(service.getSystemIndices()).thenReturn(EmptySystemIndices.INSTANCE); return service; } - - @Override - protected Collection getPlugins() { - return List.of(new MetadataIndexTemplateServiceTests.DummyPlugin()); - } } diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index a91edb8a94fb4..e48400182df95 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -10,19 +10,30 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.StringFieldScript; import org.elasticsearch.script.StringFieldScript.LeafFactory; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; import java.io.IOException; import java.util.Map; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; public class TimeSeriesModeTests extends MapperServiceTestCase { + public void testConfigureIndex() { Settings s = Settings.builder() .put(IndexSettings.MODE.getKey(), "time_series") @@ -67,6 +78,128 @@ public void testSortOrder() { assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.order]")); } + public void testAddsTimestamp() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + DocumentMapper mapper = createMapperService(s, mapping(b -> {})).documentMapper(); + MappedFieldType timestamp = mapper.mappers().getFieldType(DataStreamTimestampFieldMapper.DEFAULT_PATH); + assertThat(timestamp, instanceOf(DateFieldType.class)); + assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.MILLISECONDS)); + + Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); + assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); + assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); + } + + public void testTimestampMillis() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) + .documentMapper(); + MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); + assertThat(timestamp, instanceOf(DateFieldType.class)); + assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.MILLISECONDS)); + + Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); + assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); + assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); + } + + public void testTimestampNanos() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date_nanos").endObject())) + .documentMapper(); + MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); + assertThat(timestamp, instanceOf(DateFieldType.class)); + assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.NANOSECONDS)); + + Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); + assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); + assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); + } + + public void testBadTimestamp() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + String type = randomFrom("keyword", "integer", "long", "double", "text"); + Exception e = expectThrows( + IllegalArgumentException.class, + () -> createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", type).endObject())) + ); + assertThat( + e.getMessage(), + equalTo("data stream timestamp field [@timestamp] is of type [" + type + "], but [date,date_nanos] is expected") + ); + } + + public void testEnabledTimeStampMapper() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + XContentBuilder mappings = XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject(DataStreamTimestampFieldMapper.NAME); + if (randomBoolean()) { + mappings.field("enabled", true); + } else { + mappings.field("enabled", "true"); + } + mappings.endObject().endObject().endObject(); + + DocumentMapper mapper = createMapperService(s, mappings).documentMapper(); + Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); + assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); + assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); + } + + public void testDisabledTimeStampMapper() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + XContentBuilder mappings = XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject(DataStreamTimestampFieldMapper.NAME) + .field("enabled", false) + .endObject() + .endObject() + .endObject(); + + Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(s, mappings).documentMapper()); + assertThat( + e.getMessage(), + equalTo("Failed to parse mapping: time series index [_data_stream_timestamp] meta field must be enabled") + ); + } + + public void testBadTimeStampMapper() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + XContentBuilder mappings = XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .field(DataStreamTimestampFieldMapper.NAME, "enabled") + .endObject() + .endObject(); + + Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(s, mappings).documentMapper()); + assertThat(e.getMessage(), equalTo("Failed to parse mapping: time series index [_data_stream_timestamp] meta field format error")); + } + public void testWithoutRoutingPath() { Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); Exception e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java index 35a80686b8635..0c713e008b1ab 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentMapperTests.java @@ -294,7 +294,8 @@ public void testEmptyDocumentMapper() { Collections.sort(metadataMappers, Comparator.comparing(c -> c.getSimpleName())); assertMap( metadataMappers, - matchesList().item(DocCountFieldMapper.class) + matchesList().item(DataStreamTimestampFieldMapper.class) + .item(DocCountFieldMapper.class) .item(FieldNamesFieldMapper.class) .item(IdFieldMapper.class) .item(IgnoredFieldMapper.class) @@ -309,7 +310,8 @@ public void testEmptyDocumentMapper() { Collections.sort(matching); assertMap( matching, - matchesList().item(DocCountFieldMapper.CONTENT_TYPE) + matchesList().item(DataStreamTimestampFieldMapper.NAME) + .item(DocCountFieldMapper.CONTENT_TYPE) .item(FieldNamesFieldMapper.CONTENT_TYPE) .item(IdFieldMapper.CONTENT_TYPE) .item(IgnoredFieldMapper.CONTENT_TYPE) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 00575dcab32a9..08b366b8c3e4f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -177,7 +177,7 @@ public void testTotalFieldsLimitWithFieldAlias() throws Throwable { } public void testFieldNameLengthLimit() throws Throwable { - int maxFieldNameLength = randomIntBetween(15, 20); + int maxFieldNameLength = randomIntBetween(25, 30); String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a"); Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength) .build(); @@ -191,7 +191,7 @@ public void testFieldNameLengthLimit() throws Throwable { } public void testObjectNameLengthLimit() throws Throwable { - int maxFieldNameLength = randomIntBetween(15, 20); + int maxFieldNameLength = randomIntBetween(25, 30); String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a"); Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength) .build(); @@ -205,7 +205,7 @@ public void testObjectNameLengthLimit() throws Throwable { } public void testAliasFieldNameLengthLimit() throws Throwable { - int maxFieldNameLength = randomIntBetween(15, 20); + int maxFieldNameLength = randomIntBetween(25, 30); String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a"); Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength) .build(); @@ -221,7 +221,7 @@ public void testAliasFieldNameLengthLimit() throws Throwable { } public void testMappingRecoverySkipFieldNameLengthLimit() throws Throwable { - int maxFieldNameLength = randomIntBetween(15, 20); + int maxFieldNameLength = randomIntBetween(25, 30); String testString = new String(new char[maxFieldNameLength + 1]).replace("\0", "a"); Settings settings = Settings.builder().put(MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING.getKey(), maxFieldNameLength) .build(); diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java index f9acded6340b5..a9c1329657cbf 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesModuleTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.indices; import org.elasticsearch.Version; +import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DocCountFieldMapper; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.IdFieldMapper; @@ -67,10 +68,18 @@ public Map getMetadataMappers() { } }); - private static final String[] EXPECTED_METADATA_FIELDS = new String[]{ IgnoredFieldMapper.NAME, IdFieldMapper.NAME, - RoutingFieldMapper.NAME, IndexFieldMapper.NAME, SourceFieldMapper.NAME, - NestedPathFieldMapper.NAME, VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, DocCountFieldMapper.NAME, - FieldNamesFieldMapper.NAME }; + private static final String[] EXPECTED_METADATA_FIELDS = new String[] { + IgnoredFieldMapper.NAME, + IdFieldMapper.NAME, + RoutingFieldMapper.NAME, + IndexFieldMapper.NAME, + SourceFieldMapper.NAME, + NestedPathFieldMapper.NAME, + VersionFieldMapper.NAME, + SeqNoFieldMapper.NAME, + DocCountFieldMapper.NAME, + DataStreamTimestampFieldMapper.NAME, + FieldNamesFieldMapper.NAME }; public void testBuiltinMappers() { IndicesModule module = new IndicesModule(Collections.emptyList()); diff --git a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java index 3c5e954e68ce6..2aae909fea0e5 100644 --- a/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java +++ b/x-pack/plugin/data-streams/src/main/java/org/elasticsearch/xpack/datastreams/DataStreamsPlugin.java @@ -14,10 +14,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; -import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; -import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.plugins.ActionPlugin; -import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestHandler; @@ -45,15 +42,9 @@ import org.elasticsearch.xpack.datastreams.rest.RestPromoteDataStreamAction; import java.util.List; -import java.util.Map; import java.util.function.Supplier; -public class DataStreamsPlugin extends Plugin implements ActionPlugin, MapperPlugin { - - @Override - public Map getMetadataMappers() { - return Map.of(DataStreamTimestampFieldMapper.NAME, DataStreamTimestampFieldMapper.PARSER); - } +public class DataStreamsPlugin extends Plugin implements ActionPlugin { @Override public List> getActions() { diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java index f1d2a0bdaa4f5..dd0d350577ccd 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/mapper/MetadataCreateDataStreamServiceTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.indices.IndicesModule; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.datastreams.DataStreamsPlugin; import java.io.IOException; import java.util.List; @@ -74,7 +73,7 @@ MappingLookup createMappingLookup(String mapping) throws IOException { ) .putMapping(mapping) .build(); - IndicesModule indicesModule = new IndicesModule(List.of(new DataStreamsPlugin())); + IndicesModule indicesModule = new IndicesModule(List.of()); MapperService mapperService = MapperTestUtils.newMapperService( xContentRegistry(), createTempDir(),