diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml index 9d61d4c359b6d..7ac5bc4be4d7b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml @@ -164,7 +164,7 @@ reject timestamp meta field with wrong type: reason: introduced in 8.0.0 to be backported to 7.16.0 - do: - catch: /.* time series index \[_data_stream_timestamp\] meta field must be enabled/ + catch: /\[_data_stream_timestamp\] meta field has been disabled/ indices.create: index: test body: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/TimeSeriesModeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/TimeSeriesModeIT.java new file mode 100644 index 0000000000000..74f58fe2364ae --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/TimeSeriesModeIT.java @@ -0,0 +1,598 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index; + +import org.elasticsearch.action.DocWriteResponse.Result; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; + +import java.io.IOException; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class TimeSeriesModeIT extends ESIntegTestCase { + public void testDisabledTimeStampMapper() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + XContentBuilder mappings = XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject(DataStreamTimestampFieldMapper.NAME) + .field("enabled", false) + .endObject() + .endObject() + .endObject(); + + Exception e = expectThrows(IllegalStateException.class, () -> prepareCreate("test").setSettings(s).setMapping(mappings).get()); + assertThat(e.getMessage(), equalTo("[_data_stream_timestamp] meta field has been disabled")); + } + + public void testBadTimeStampMapper() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + XContentBuilder mappings = XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .field(DataStreamTimestampFieldMapper.NAME, "enabled") + .endObject() + .endObject(); + + Exception e = expectThrows(MapperParsingException.class, () -> prepareCreate("test").setSettings(s).setMapping(mappings).get()); + assertThat(e.getMessage(), equalTo("Failed to parse mapping: [_data_stream_timestamp] config must be an object")); + } + + public void testBadTimestamp() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + String type = randomFrom("keyword", "integer", "long", "double", "text"); + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", type); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + Exception e = expectThrows(IllegalArgumentException.class, () -> prepareCreate("test").setSettings(s).setMapping(mappings).get()); + assertThat( + e.getMessage(), + equalTo("data stream timestamp field [@timestamp] is of type [" + type + "], but [date,date_nanos] is expected") + ); + } + + public void testAddsTimestamp() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).get(); + ensureGreen(index); + + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).get(); + assertThat(getMappingsResponse.getMappings().size(), equalTo(1)); + + XContentBuilder expect = XContentFactory.jsonBuilder(); + expect.startObject(); + { + expect.startObject("_doc"); + { + expect.startObject(DataStreamTimestampFieldMapper.NAME); + { + expect.field("enabled", true); + } + expect.endObject(); + expect.startObject("properties"); + { + expect.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + expect.field("type", "date"); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + assertThat(getMappingsResponse.getMappings().get(index).source().string(), equalTo(Strings.toString(expect))); + } + + public void testTimestampMillis() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", "date"); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).get(); + XContentBuilder expect = XContentFactory.jsonBuilder(); + expect.startObject(); + { + expect.startObject("_doc"); + { + expect.startObject(DataStreamTimestampFieldMapper.NAME); + { + expect.field("enabled", true); + } + expect.endObject(); + expect.startObject("properties"); + { + expect.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + expect.field("type", "date"); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + assertThat(getMappingsResponse.getMappings().get(index).source().string(), equalTo(Strings.toString(expect))); + } + + public void testTimestampNanos() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", "date_nanos"); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).get(); + XContentBuilder expect = XContentFactory.jsonBuilder(); + expect.startObject(); + { + expect.startObject("_doc"); + { + expect.startObject(DataStreamTimestampFieldMapper.NAME); + { + expect.field("enabled", true); + } + expect.endObject(); + expect.startObject("properties"); + { + expect.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + expect.field("type", "date_nanos"); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + assertThat(getMappingsResponse.getMappings().get(index).source().string(), equalTo(Strings.toString(expect))); + } + + public void testWithoutTimestamp() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", "date"); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> index(index, XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject()) + ); + assertThat(e.getRootCause().getMessage(), containsString("data stream timestamp field [@timestamp] is missing")); + } + + public void testEnableTimestampRange() throws IOException { + long endTime = System.currentTimeMillis(); + long startTime = endTime - TimeUnit.DAYS.toMillis(1); + + Settings s = Settings.builder() + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", randomBoolean() ? "date" : "date_nanos"); + } + mappings.endObject(); + mappings.startObject("foo"); + { + mappings.field("type", "keyword"); + mappings.field("time_series_dimension", true); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + IndexResponse indexResponse = index( + index, + XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .field("@timestamp", randomLongBetween(startTime, endTime)) + .endObject() + ); + assertEquals(indexResponse.getResult(), Result.CREATED); + } + + public void testBadStartTime() throws IOException { + long endTime = System.currentTimeMillis(); + long startTime = endTime - TimeUnit.DAYS.toMillis(1); + + Settings s = Settings.builder() + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", randomBoolean() ? "date" : "date_nanos"); + } + mappings.endObject(); + mappings.startObject("foo"); + { + mappings.field("type", "keyword"); + mappings.field("time_series_dimension", true); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> index( + index, + XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .field("@timestamp", Math.max(startTime - randomLongBetween(1, 3), 0)) + .endObject() + ) + ); + assertThat(e.getRootCause().getMessage(), containsString("must be larger than")); + } + + public void testBadEndTime() throws IOException { + long endTime = System.currentTimeMillis(); + long startTime = endTime - TimeUnit.DAYS.toMillis(1); + + Settings s = Settings.builder() + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", randomBoolean() ? "date" : "date_nanos"); + } + mappings.endObject(); + mappings.startObject("foo"); + { + mappings.field("type", "keyword"); + mappings.field("time_series_dimension", true); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> index( + index, + XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .field("@timestamp", endTime + randomLongBetween(0, 3)) + .endObject() + ) + ); + assertThat(e.getRootCause().getMessage(), containsString("must be smaller than")); + } + + public void testEnabledTimeStampMapper() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + XContentBuilder mappings = XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject(DataStreamTimestampFieldMapper.NAME); + if (randomBoolean()) { + mappings.field("enabled", true); + } else { + mappings.field("enabled", "true"); + } + mappings.endObject().endObject().endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).get(); + XContentBuilder expect = XContentFactory.jsonBuilder(); + expect.startObject(); + { + expect.startObject("_doc"); + { + expect.startObject(DataStreamTimestampFieldMapper.NAME); + { + expect.field("enabled", true); + } + expect.endObject(); + expect.startObject("properties"); + { + expect.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + expect.field("type", "date"); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + assertThat(getMappingsResponse.getMappings().get(index).source().string(), equalTo(Strings.toString(expect))); + } + + public void testAddTimeStampMeta() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject(DataStreamTimestampFieldMapper.NAME); + { + mappings.field("enabled", true); + } + mappings.endObject(); + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", "date"); + mappings.startObject("meta"); + { + mappings.field("field_meta", "time_series"); + } + mappings.endObject(); + } + mappings.endObject(); + mappings.startObject("foo"); + { + mappings.field("type", "keyword"); + mappings.field("time_series_dimension", true); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + IndexResponse indexResponse = index( + index, + XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .field("@timestamp", System.currentTimeMillis()) + .field("new_field", "value") + .endObject() + ); + assertEquals(indexResponse.getResult(), Result.CREATED); + + XContentBuilder expect = XContentFactory.jsonBuilder(); + expect.startObject(); + { + expect.startObject("_doc"); + { + expect.startObject(DataStreamTimestampFieldMapper.NAME); + { + expect.field("enabled", true); + } + expect.endObject(); + expect.startObject("properties"); + { + expect.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + expect.field("type", "date"); + expect.startObject("meta"); + { + expect.field("field_meta", "time_series"); + } + expect.endObject(); + } + expect.endObject(); + expect.startObject("foo"); + { + expect.field("type", "keyword"); + expect.field("time_series_dimension", true); + } + expect.endObject(); + expect.startObject("new_field"); + { + expect.field("type", "text"); + expect.startObject("fields"); + { + expect.startObject("keyword"); + { + expect.field("type", "keyword"); + expect.field("ignore_above", 256); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).get(); + assertThat(getMappingsResponse.getMappings().get(index).source().string(), equalTo(Strings.toString(expect))); + } + +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index b0e6ca7e6f0ba..ab0893aeaf5d1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -51,6 +51,7 @@ import org.elasticsearch.core.PathUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexService; @@ -100,7 +101,6 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveSettings; import static org.elasticsearch.index.IndexModule.INDEX_RECOVERY_TYPE_SETTING; import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; @@ -1195,18 +1195,23 @@ private static ClusterBlocks.Builder createClusterBlocksBuilder(ClusterState cur return blocksBuilder; } - private static void updateIndexMappingsAndBuildSortOrder( + private void updateIndexMappingsAndBuildSortOrder( IndexService indexService, CreateIndexClusterStateUpdateRequest request, List> mappings, @Nullable IndexMetadata sourceMetadata ) throws IOException { MapperService mapperService = indexService.mapperService(); - for (Map mapping : mappings) { + IndexMode indexMode = indexService.getIndexSettings() != null ? indexService.getIndexSettings().getMode() : IndexMode.STANDARD; + List> mergedMappings = new ArrayList<>(1 + mappings.size()); + mergedMappings.add(indexMode.getDefaultMapping()); + mergedMappings.addAll(mappings); + for (Map mapping : mergedMappings) { if (mapping.isEmpty() == false) { mapperService.merge(MapperService.SINGLE_MAPPING_NAME, mapping, MergeReason.INDEX_TEMPLATE); } } + indexMode.validateTimestampFieldMapping(request.dataStreamName() != null, mapperService.mappingLookup()); if (sourceMetadata == null) { // now that the mapping is merged we can validate the index sort. @@ -1215,9 +1220,6 @@ private static void updateIndexMappingsAndBuildSortOrder( // (when all shards are copied in a single place). indexService.getIndexSortSupplier().get(); } - if (request.dataStreamName() != null) { - validateTimestampFieldMapping(mapperService.mappingLookup()); - } } private static void validateActiveShardCount(ActiveShardCount waitForActiveShards, IndexMetadata indexMetadata) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 7c672d65632c3..0107a1ad817d8 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -9,23 +9,21 @@ package org.elasticsearch.index; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; -import org.elasticsearch.index.mapper.MappingParserContext; -import org.elasticsearch.index.mapper.RootObjectMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; -import java.util.HashMap; +import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.stream.Stream; import static java.util.stream.Collectors.toSet; @@ -59,7 +57,16 @@ private void settingRequiresTimeSeries(Map, Object> settings, Setting public void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting) {} @Override - public void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder) {} + public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup mappingLookup) throws IOException { + if (isDataStream) { + MetadataCreateDataStreamService.validateTimestampFieldMapping(mappingLookup); + } + } + + @Override + public Map getDefaultMapping() { + return Collections.emptyMap(); + } }, TIME_SERIES { @Override @@ -100,55 +107,35 @@ public void validateAlias(@Nullable String indexRouting, @Nullable String search } } - private String routingRequiredBad() { - return "routing is forbidden on CRUD operations that target indices in " + tsdbMode(); - } - - private String tsdbMode() { - return "[" + IndexSettings.MODE.getKey() + "=time_series]"; + @Override + public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup mappingLookup) throws IOException { + MetadataCreateDataStreamService.validateTimestampFieldMapping(mappingLookup); } @Override - public void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder) { - if (false == mapping.containsKey(DataStreamTimestampFieldMapper.NAME)) { - mapping.put(DataStreamTimestampFieldMapper.NAME, new HashMap<>(Map.of("enabled", true))); - } else { - validateTimeStampField(mapping.get(DataStreamTimestampFieldMapper.NAME)); - } - - Optional timestamp = builder.getBuilder(DataStreamTimestampFieldMapper.DEFAULT_PATH); - if (timestamp.isEmpty()) { - builder.add( - new DateFieldMapper.Builder( - DataStreamTimestampFieldMapper.DEFAULT_PATH, - DateFieldMapper.Resolution.MILLISECONDS, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, - context.scriptCompiler(), - DateFieldMapper.IGNORE_MALFORMED_SETTING.get(context.getSettings()), - context.getIndexSettings().getIndexVersionCreated() - ) - ); - } + public Map getDefaultMapping() { + return DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING; } - private void validateTimeStampField(Object timestampFieldValue) { - if (false == (timestampFieldValue instanceof Map)) { - throw new IllegalArgumentException( - "time series index [" + DataStreamTimestampFieldMapper.NAME + "] meta field format error" - ); - } + private String routingRequiredBad() { + return "routing is forbidden on CRUD operations that target indices in " + tsdbMode(); + } - @SuppressWarnings("unchecked") - Map timeStampFieldValueMap = (Map) timestampFieldValue; - if (false == Maps.deepEquals(timeStampFieldValueMap, Map.of("enabled", true)) - && false == Maps.deepEquals(timeStampFieldValueMap, Map.of("enabled", "true"))) { - throw new IllegalArgumentException( - "time series index [" + DataStreamTimestampFieldMapper.NAME + "] meta field must be enabled" - ); - } + private String tsdbMode() { + return "[" + IndexSettings.MODE.getKey() + "=time_series]"; } }; + public static final Map DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING = Map.of( + MapperService.SINGLE_MAPPING_NAME, + Map.of( + DataStreamTimestampFieldMapper.NAME, + Map.of("enabled", true), + "properties", + Map.of(DataStreamTimestampFieldMapper.DEFAULT_PATH, Map.of("type", DateFieldMapper.CONTENT_TYPE)) + ) + ); + private static final List> TIME_SERIES_UNSUPPORTED = List.of( IndexSortConfig.INDEX_SORT_FIELD_SETTING, IndexSortConfig.INDEX_SORT_ORDER_SETTING, @@ -181,7 +168,13 @@ private void validateTimeStampField(Object timestampFieldValue) { public abstract void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting); /** - * Validate and/or modify the mappings after after they've been parsed. + * validate timestamp mapping for this index. + */ + public abstract void validateTimestampFieldMapping(boolean isDataStream, MappingLookup mappingLookup) throws IOException; + + /** + * get default mapping for this index. + * @return */ - public abstract void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder); + public abstract Map getDefaultMapping(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java index 33c06221b43cd..19e18efaf24e0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java @@ -96,8 +96,7 @@ Mapping parse(@Nullable String type, CompressedXContent source) throws MapperPar private Mapping parse(String type, Map mapping) throws MapperParsingException { MappingParserContext parserContext = parserContextSupplier.get(); - RootObjectMapper.Builder rootObjectMapperBuilder = rootObjectTypeParser.parse(type, mapping, parserContext); - parserContext.getIndexSettings().getMode().completeMappings(parserContext, mapping, rootObjectMapperBuilder); + RootObjectMapper rootObjectMapper = rootObjectTypeParser.parse(type, mapping, parserContext).build(MapperBuilderContext.ROOT); Map, MetadataFieldMapper> metadataMappers = metadataMappersSupplier.get(); Map meta = null; @@ -145,10 +144,6 @@ private Mapping parse(String type, Map mapping) throws MapperPar } checkNoRemainingFields(mapping, "Root mapping definition has unsupported parameters: "); - return new Mapping( - rootObjectMapperBuilder.build(MapperBuilderContext.ROOT), - metadataMappers.values().toArray(new MetadataFieldMapper[0]), - meta - ); + return new Mapping(rootObjectMapper, metadataMappers.values().toArray(new MetadataFieldMapper[0]), meta); } } diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index d16c1d273207c..925bed8c7ab77 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -9,34 +9,18 @@ package org.elasticsearch.index; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; -import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperServiceTestCase; -import org.elasticsearch.index.mapper.ParsedDocument; -import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.StringFieldScript; import org.elasticsearch.script.StringFieldScript.LeafFactory; import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.Map; -import java.util.concurrent.TimeUnit; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; public class TimeSeriesModeTests extends MapperServiceTestCase { @@ -84,231 +68,6 @@ public void testSortOrder() { assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.order]")); } - public void testAddsTimestamp() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - DocumentMapper mapper = createMapperService(s, mapping(b -> {})).documentMapper(); - MappedFieldType timestamp = mapper.mappers().getFieldType(DataStreamTimestampFieldMapper.DEFAULT_PATH); - assertThat(timestamp, instanceOf(DateFieldType.class)); - assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.MILLISECONDS)); - - Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); - assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); - assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); - } - - public void testTimestampMillis() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) - .documentMapper(); - MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); - assertThat(timestamp, instanceOf(DateFieldType.class)); - assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.MILLISECONDS)); - - Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); - assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); - assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); - } - - public void testTimestampNanos() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date_nanos").endObject())) - .documentMapper(); - MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); - assertThat(timestamp, instanceOf(DateFieldType.class)); - assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.NANOSECONDS)); - - Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); - assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); - assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); - } - - public void testBadTimestamp() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - String type = randomFrom("keyword", "integer", "long", "double", "text"); - Exception e = expectThrows( - IllegalArgumentException.class, - () -> createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", type).endObject())) - ); - assertThat( - e.getMessage(), - equalTo("data stream timestamp field [@timestamp] is of type [" + type + "], but [date,date_nanos] is expected") - ); - } - - public void testWithoutTimestamp() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - - DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) - .documentMapper(); - MapperParsingException e = expectThrows( - MapperParsingException.class, - () -> mapper.parse( - new SourceToParse("1", BytesReference.bytes(XContentFactory.jsonBuilder().startObject().endObject()), XContentType.JSON) - ) - ); - assertThat(e.getRootCause().getMessage(), containsString("data stream timestamp field [@timestamp] is missing")); - } - - public void testEnableTimestampRange() throws IOException { - long endTime = System.currentTimeMillis(); - long startTime = endTime - TimeUnit.DAYS.toMillis(1); - - Settings s = Settings.builder() - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - DocumentMapper mapper = createMapperService( - s, - mapping(b -> b.startObject("@timestamp").field("type", randomBoolean() ? "date" : "date_nanos").endObject()) - ).documentMapper(); - ParsedDocument doc = mapper.parse( - new SourceToParse( - "1", - BytesReference.bytes( - XContentFactory.jsonBuilder().startObject().field("@timestamp", randomLongBetween(startTime, endTime)).endObject() - ), - XContentType.JSON - ) - ); - // Look, mah, no failure. - assertNotNull(doc.rootDoc().getNumericValue("@timestamp")); - } - - public void testBadStartTime() throws IOException { - long endTime = System.currentTimeMillis(); - long startTime = endTime - TimeUnit.DAYS.toMillis(1); - - Settings s = Settings.builder() - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - - DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) - .documentMapper(); - MapperParsingException e = expectThrows( - MapperParsingException.class, - () -> mapper.parse( - new SourceToParse( - "1", - BytesReference.bytes( - XContentFactory.jsonBuilder() - .startObject() - .field("@timestamp", Math.max(startTime - randomLongBetween(1, 3), 0)) - .endObject() - ), - XContentType.JSON - ) - ) - ); - assertThat(e.getRootCause().getMessage(), containsString("must be larger than")); - } - - public void testBadEndTime() throws IOException { - long endTime = System.currentTimeMillis(); - long startTime = endTime - TimeUnit.DAYS.toMillis(1); - - Settings s = Settings.builder() - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - - DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) - .documentMapper(); - MapperParsingException e = expectThrows( - MapperParsingException.class, - () -> mapper.parse( - new SourceToParse( - "1", - BytesReference.bytes( - XContentFactory.jsonBuilder().startObject().field("@timestamp", endTime + randomLongBetween(0, 3)).endObject() - ), - XContentType.JSON - ) - ) - ); - assertThat(e.getRootCause().getMessage(), containsString("must be smaller than")); - } - - public void testEnabledTimeStampMapper() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - XContentBuilder mappings = XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject(DataStreamTimestampFieldMapper.NAME); - if (randomBoolean()) { - mappings.field("enabled", true); - } else { - mappings.field("enabled", "true"); - } - mappings.endObject().endObject().endObject(); - - DocumentMapper mapper = createMapperService(s, mappings).documentMapper(); - Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); - assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); - assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); - } - - public void testDisabledTimeStampMapper() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - XContentBuilder mappings = XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject(DataStreamTimestampFieldMapper.NAME) - .field("enabled", false) - .endObject() - .endObject() - .endObject(); - - Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(s, mappings).documentMapper()); - assertThat( - e.getMessage(), - equalTo("Failed to parse mapping: time series index [_data_stream_timestamp] meta field must be enabled") - ); - } - - public void testBadTimeStampMapper() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - XContentBuilder mappings = XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .field(DataStreamTimestampFieldMapper.NAME, "enabled") - .endObject() - .endObject(); - - Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(s, mappings).documentMapper()); - assertThat(e.getMessage(), equalTo("Failed to parse mapping: time series index [_data_stream_timestamp] meta field format error")); - } - public void testWithoutRoutingPath() { Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); Exception e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s));