From 2136af2846b4d07244d5e4debe8c884e2f451966 Mon Sep 17 00:00:00 2001 From: weizijun Date: Mon, 22 Nov 2021 21:06:37 +0800 Subject: [PATCH] Fix time series timestamp meta missing (#80695) I find a time series timestamp meta missing case. here is the reproduce steps: 1. create a time_series index, and set the timestamp field some meta. 2. index a doc with a new field that is not in mappings, it will call mappings marge. 3. then the timestamp field meta is missing. the reason that meta is missing is when a new field comes, `MappingParser.parse` will build a new mapping with new fields. And merge the new mappings with exist mapping. the new mapping have no timestamp field, so it will auto add timestamp field, the timestamp is without user's meta info. And merge method build a new timestamp field to override the user's timestamp field. It cause the timestamp meta missing. I fixed the case, by move timestamp logic from MappingParser.parse to create index logic. And move the tests to a new IT test. I add a test to test case, TimeSeriesModeIT.testAddTimeStampMeta will fail in the pre-commit. --- .../test/tsdb/15_timestamp_mapping.yml | 2 +- .../elasticsearch/index/TimeSeriesModeIT.java | 598 ++++++++++++++++++ .../metadata/MetadataCreateIndexService.java | 14 +- .../org/elasticsearch/index/IndexMode.java | 91 ++- .../index/mapper/MappingParser.java | 9 +- .../index/TimeSeriesModeTests.java | 241 ------- 6 files changed, 651 insertions(+), 304 deletions(-) create mode 100644 server/src/internalClusterTest/java/org/elasticsearch/index/TimeSeriesModeIT.java diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml index 9d61d4c359b6d..7ac5bc4be4d7b 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/15_timestamp_mapping.yml @@ -164,7 +164,7 @@ reject timestamp meta field with wrong type: reason: introduced in 8.0.0 to be backported to 7.16.0 - do: - catch: /.* time series index \[_data_stream_timestamp\] meta field must be enabled/ + catch: /\[_data_stream_timestamp\] meta field has been disabled/ indices.create: index: test body: diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/TimeSeriesModeIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/TimeSeriesModeIT.java new file mode 100644 index 0000000000000..74f58fe2364ae --- /dev/null +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/TimeSeriesModeIT.java @@ -0,0 +1,598 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index; + +import org.elasticsearch.action.DocWriteResponse.Result; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.action.index.IndexResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentFactory; + +import java.io.IOException; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; + +public class TimeSeriesModeIT extends ESIntegTestCase { + public void testDisabledTimeStampMapper() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + XContentBuilder mappings = XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject(DataStreamTimestampFieldMapper.NAME) + .field("enabled", false) + .endObject() + .endObject() + .endObject(); + + Exception e = expectThrows(IllegalStateException.class, () -> prepareCreate("test").setSettings(s).setMapping(mappings).get()); + assertThat(e.getMessage(), equalTo("[_data_stream_timestamp] meta field has been disabled")); + } + + public void testBadTimeStampMapper() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + XContentBuilder mappings = XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .field(DataStreamTimestampFieldMapper.NAME, "enabled") + .endObject() + .endObject(); + + Exception e = expectThrows(MapperParsingException.class, () -> prepareCreate("test").setSettings(s).setMapping(mappings).get()); + assertThat(e.getMessage(), equalTo("Failed to parse mapping: [_data_stream_timestamp] config must be an object")); + } + + public void testBadTimestamp() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + String type = randomFrom("keyword", "integer", "long", "double", "text"); + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", type); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + Exception e = expectThrows(IllegalArgumentException.class, () -> prepareCreate("test").setSettings(s).setMapping(mappings).get()); + assertThat( + e.getMessage(), + equalTo("data stream timestamp field [@timestamp] is of type [" + type + "], but [date,date_nanos] is expected") + ); + } + + public void testAddsTimestamp() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).get(); + ensureGreen(index); + + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).get(); + assertThat(getMappingsResponse.getMappings().size(), equalTo(1)); + + XContentBuilder expect = XContentFactory.jsonBuilder(); + expect.startObject(); + { + expect.startObject("_doc"); + { + expect.startObject(DataStreamTimestampFieldMapper.NAME); + { + expect.field("enabled", true); + } + expect.endObject(); + expect.startObject("properties"); + { + expect.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + expect.field("type", "date"); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + assertThat(getMappingsResponse.getMappings().get(index).source().string(), equalTo(Strings.toString(expect))); + } + + public void testTimestampMillis() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", "date"); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).get(); + XContentBuilder expect = XContentFactory.jsonBuilder(); + expect.startObject(); + { + expect.startObject("_doc"); + { + expect.startObject(DataStreamTimestampFieldMapper.NAME); + { + expect.field("enabled", true); + } + expect.endObject(); + expect.startObject("properties"); + { + expect.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + expect.field("type", "date"); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + assertThat(getMappingsResponse.getMappings().get(index).source().string(), equalTo(Strings.toString(expect))); + } + + public void testTimestampNanos() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", "date_nanos"); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).get(); + XContentBuilder expect = XContentFactory.jsonBuilder(); + expect.startObject(); + { + expect.startObject("_doc"); + { + expect.startObject(DataStreamTimestampFieldMapper.NAME); + { + expect.field("enabled", true); + } + expect.endObject(); + expect.startObject("properties"); + { + expect.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + expect.field("type", "date_nanos"); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + assertThat(getMappingsResponse.getMappings().get(index).source().string(), equalTo(Strings.toString(expect))); + } + + public void testWithoutTimestamp() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", "date"); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> index(index, XContentFactory.jsonBuilder().startObject().field("foo", "bar").endObject()) + ); + assertThat(e.getRootCause().getMessage(), containsString("data stream timestamp field [@timestamp] is missing")); + } + + public void testEnableTimestampRange() throws IOException { + long endTime = System.currentTimeMillis(); + long startTime = endTime - TimeUnit.DAYS.toMillis(1); + + Settings s = Settings.builder() + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", randomBoolean() ? "date" : "date_nanos"); + } + mappings.endObject(); + mappings.startObject("foo"); + { + mappings.field("type", "keyword"); + mappings.field("time_series_dimension", true); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + IndexResponse indexResponse = index( + index, + XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .field("@timestamp", randomLongBetween(startTime, endTime)) + .endObject() + ); + assertEquals(indexResponse.getResult(), Result.CREATED); + } + + public void testBadStartTime() throws IOException { + long endTime = System.currentTimeMillis(); + long startTime = endTime - TimeUnit.DAYS.toMillis(1); + + Settings s = Settings.builder() + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", randomBoolean() ? "date" : "date_nanos"); + } + mappings.endObject(); + mappings.startObject("foo"); + { + mappings.field("type", "keyword"); + mappings.field("time_series_dimension", true); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> index( + index, + XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .field("@timestamp", Math.max(startTime - randomLongBetween(1, 3), 0)) + .endObject() + ) + ); + assertThat(e.getRootCause().getMessage(), containsString("must be larger than")); + } + + public void testBadEndTime() throws IOException { + long endTime = System.currentTimeMillis(); + long startTime = endTime - TimeUnit.DAYS.toMillis(1); + + Settings s = Settings.builder() + .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) + .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", randomBoolean() ? "date" : "date_nanos"); + } + mappings.endObject(); + mappings.startObject("foo"); + { + mappings.field("type", "keyword"); + mappings.field("time_series_dimension", true); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> index( + index, + XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .field("@timestamp", endTime + randomLongBetween(0, 3)) + .endObject() + ) + ); + assertThat(e.getRootCause().getMessage(), containsString("must be smaller than")); + } + + public void testEnabledTimeStampMapper() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + XContentBuilder mappings = XContentFactory.jsonBuilder() + .startObject() + .startObject("_doc") + .startObject(DataStreamTimestampFieldMapper.NAME); + if (randomBoolean()) { + mappings.field("enabled", true); + } else { + mappings.field("enabled", "true"); + } + mappings.endObject().endObject().endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).get(); + XContentBuilder expect = XContentFactory.jsonBuilder(); + expect.startObject(); + { + expect.startObject("_doc"); + { + expect.startObject(DataStreamTimestampFieldMapper.NAME); + { + expect.field("enabled", true); + } + expect.endObject(); + expect.startObject("properties"); + { + expect.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + expect.field("type", "date"); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + assertThat(getMappingsResponse.getMappings().get(index).source().string(), equalTo(Strings.toString(expect))); + } + + public void testAddTimeStampMeta() throws IOException { + Settings s = Settings.builder() + .put(IndexSettings.MODE.getKey(), "time_series") + .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") + .build(); + + XContentBuilder mappings = XContentFactory.jsonBuilder(); + mappings.startObject(); + { + mappings.startObject("_doc"); + { + mappings.startObject(DataStreamTimestampFieldMapper.NAME); + { + mappings.field("enabled", true); + } + mappings.endObject(); + mappings.startObject("properties"); + { + mappings.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + mappings.field("type", "date"); + mappings.startObject("meta"); + { + mappings.field("field_meta", "time_series"); + } + mappings.endObject(); + } + mappings.endObject(); + mappings.startObject("foo"); + { + mappings.field("type", "keyword"); + mappings.field("time_series_dimension", true); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + } + mappings.endObject(); + + String index = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); + prepareCreate(index).setSettings(s).setMapping(mappings).get(); + ensureGreen(index); + + IndexResponse indexResponse = index( + index, + XContentFactory.jsonBuilder() + .startObject() + .field("foo", "bar") + .field("@timestamp", System.currentTimeMillis()) + .field("new_field", "value") + .endObject() + ); + assertEquals(indexResponse.getResult(), Result.CREATED); + + XContentBuilder expect = XContentFactory.jsonBuilder(); + expect.startObject(); + { + expect.startObject("_doc"); + { + expect.startObject(DataStreamTimestampFieldMapper.NAME); + { + expect.field("enabled", true); + } + expect.endObject(); + expect.startObject("properties"); + { + expect.startObject(DataStreamTimestampFieldMapper.DEFAULT_PATH); + { + expect.field("type", "date"); + expect.startObject("meta"); + { + expect.field("field_meta", "time_series"); + } + expect.endObject(); + } + expect.endObject(); + expect.startObject("foo"); + { + expect.field("type", "keyword"); + expect.field("time_series_dimension", true); + } + expect.endObject(); + expect.startObject("new_field"); + { + expect.field("type", "text"); + expect.startObject("fields"); + { + expect.startObject("keyword"); + { + expect.field("type", "keyword"); + expect.field("ignore_above", 256); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + } + expect.endObject(); + GetMappingsResponse getMappingsResponse = client().admin().indices().prepareGetMappings(index).get(); + assertThat(getMappingsResponse.getMappings().get(index).source().string(), equalTo(Strings.toString(expect))); + } + +} diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index b0e6ca7e6f0ba..ab0893aeaf5d1 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -51,6 +51,7 @@ import org.elasticsearch.core.PathUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexService; @@ -100,7 +101,6 @@ import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; -import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.validateTimestampFieldMapping; import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.resolveSettings; import static org.elasticsearch.index.IndexModule.INDEX_RECOVERY_TYPE_SETTING; import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; @@ -1195,18 +1195,23 @@ private static ClusterBlocks.Builder createClusterBlocksBuilder(ClusterState cur return blocksBuilder; } - private static void updateIndexMappingsAndBuildSortOrder( + private void updateIndexMappingsAndBuildSortOrder( IndexService indexService, CreateIndexClusterStateUpdateRequest request, List> mappings, @Nullable IndexMetadata sourceMetadata ) throws IOException { MapperService mapperService = indexService.mapperService(); - for (Map mapping : mappings) { + IndexMode indexMode = indexService.getIndexSettings() != null ? indexService.getIndexSettings().getMode() : IndexMode.STANDARD; + List> mergedMappings = new ArrayList<>(1 + mappings.size()); + mergedMappings.add(indexMode.getDefaultMapping()); + mergedMappings.addAll(mappings); + for (Map mapping : mergedMappings) { if (mapping.isEmpty() == false) { mapperService.merge(MapperService.SINGLE_MAPPING_NAME, mapping, MergeReason.INDEX_TEMPLATE); } } + indexMode.validateTimestampFieldMapping(request.dataStreamName() != null, mapperService.mappingLookup()); if (sourceMetadata == null) { // now that the mapping is merged we can validate the index sort. @@ -1215,9 +1220,6 @@ private static void updateIndexMappingsAndBuildSortOrder( // (when all shards are copied in a single place). indexService.getIndexSortSupplier().get(); } - if (request.dataStreamName() != null) { - validateTimestampFieldMapping(mapperService.mappingLookup()); - } } private static void validateActiveShardCount(ActiveShardCount waitForActiveShards, IndexMetadata indexMetadata) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 7c672d65632c3..0107a1ad817d8 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -9,23 +9,21 @@ package org.elasticsearch.index; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.Maps; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; -import org.elasticsearch.index.mapper.MappingParserContext; -import org.elasticsearch.index.mapper.RootObjectMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; -import java.util.HashMap; +import java.io.IOException; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.stream.Stream; import static java.util.stream.Collectors.toSet; @@ -59,7 +57,16 @@ private void settingRequiresTimeSeries(Map, Object> settings, Setting public void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting) {} @Override - public void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder) {} + public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup mappingLookup) throws IOException { + if (isDataStream) { + MetadataCreateDataStreamService.validateTimestampFieldMapping(mappingLookup); + } + } + + @Override + public Map getDefaultMapping() { + return Collections.emptyMap(); + } }, TIME_SERIES { @Override @@ -100,55 +107,35 @@ public void validateAlias(@Nullable String indexRouting, @Nullable String search } } - private String routingRequiredBad() { - return "routing is forbidden on CRUD operations that target indices in " + tsdbMode(); - } - - private String tsdbMode() { - return "[" + IndexSettings.MODE.getKey() + "=time_series]"; + @Override + public void validateTimestampFieldMapping(boolean isDataStream, MappingLookup mappingLookup) throws IOException { + MetadataCreateDataStreamService.validateTimestampFieldMapping(mappingLookup); } @Override - public void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder) { - if (false == mapping.containsKey(DataStreamTimestampFieldMapper.NAME)) { - mapping.put(DataStreamTimestampFieldMapper.NAME, new HashMap<>(Map.of("enabled", true))); - } else { - validateTimeStampField(mapping.get(DataStreamTimestampFieldMapper.NAME)); - } - - Optional timestamp = builder.getBuilder(DataStreamTimestampFieldMapper.DEFAULT_PATH); - if (timestamp.isEmpty()) { - builder.add( - new DateFieldMapper.Builder( - DataStreamTimestampFieldMapper.DEFAULT_PATH, - DateFieldMapper.Resolution.MILLISECONDS, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, - context.scriptCompiler(), - DateFieldMapper.IGNORE_MALFORMED_SETTING.get(context.getSettings()), - context.getIndexSettings().getIndexVersionCreated() - ) - ); - } + public Map getDefaultMapping() { + return DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING; } - private void validateTimeStampField(Object timestampFieldValue) { - if (false == (timestampFieldValue instanceof Map)) { - throw new IllegalArgumentException( - "time series index [" + DataStreamTimestampFieldMapper.NAME + "] meta field format error" - ); - } + private String routingRequiredBad() { + return "routing is forbidden on CRUD operations that target indices in " + tsdbMode(); + } - @SuppressWarnings("unchecked") - Map timeStampFieldValueMap = (Map) timestampFieldValue; - if (false == Maps.deepEquals(timeStampFieldValueMap, Map.of("enabled", true)) - && false == Maps.deepEquals(timeStampFieldValueMap, Map.of("enabled", "true"))) { - throw new IllegalArgumentException( - "time series index [" + DataStreamTimestampFieldMapper.NAME + "] meta field must be enabled" - ); - } + private String tsdbMode() { + return "[" + IndexSettings.MODE.getKey() + "=time_series]"; } }; + public static final Map DEFAULT_TIME_SERIES_TIMESTAMP_MAPPING = Map.of( + MapperService.SINGLE_MAPPING_NAME, + Map.of( + DataStreamTimestampFieldMapper.NAME, + Map.of("enabled", true), + "properties", + Map.of(DataStreamTimestampFieldMapper.DEFAULT_PATH, Map.of("type", DateFieldMapper.CONTENT_TYPE)) + ) + ); + private static final List> TIME_SERIES_UNSUPPORTED = List.of( IndexSortConfig.INDEX_SORT_FIELD_SETTING, IndexSortConfig.INDEX_SORT_ORDER_SETTING, @@ -181,7 +168,13 @@ private void validateTimeStampField(Object timestampFieldValue) { public abstract void validateAlias(@Nullable String indexRouting, @Nullable String searchRouting); /** - * Validate and/or modify the mappings after after they've been parsed. + * validate timestamp mapping for this index. + */ + public abstract void validateTimestampFieldMapping(boolean isDataStream, MappingLookup mappingLookup) throws IOException; + + /** + * get default mapping for this index. + * @return */ - public abstract void completeMappings(MappingParserContext context, Map mapping, RootObjectMapper.Builder builder); + public abstract Map getDefaultMapping(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java index 33c06221b43cd..19e18efaf24e0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java @@ -96,8 +96,7 @@ Mapping parse(@Nullable String type, CompressedXContent source) throws MapperPar private Mapping parse(String type, Map mapping) throws MapperParsingException { MappingParserContext parserContext = parserContextSupplier.get(); - RootObjectMapper.Builder rootObjectMapperBuilder = rootObjectTypeParser.parse(type, mapping, parserContext); - parserContext.getIndexSettings().getMode().completeMappings(parserContext, mapping, rootObjectMapperBuilder); + RootObjectMapper rootObjectMapper = rootObjectTypeParser.parse(type, mapping, parserContext).build(MapperBuilderContext.ROOT); Map, MetadataFieldMapper> metadataMappers = metadataMappersSupplier.get(); Map meta = null; @@ -145,10 +144,6 @@ private Mapping parse(String type, Map mapping) throws MapperPar } checkNoRemainingFields(mapping, "Root mapping definition has unsupported parameters: "); - return new Mapping( - rootObjectMapperBuilder.build(MapperBuilderContext.ROOT), - metadataMappers.values().toArray(new MetadataFieldMapper[0]), - meta - ); + return new Mapping(rootObjectMapper, metadataMappers.values().toArray(new MetadataFieldMapper[0]), meta); } } diff --git a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java index d16c1d273207c..925bed8c7ab77 100644 --- a/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java +++ b/server/src/test/java/org/elasticsearch/index/TimeSeriesModeTests.java @@ -9,34 +9,18 @@ package org.elasticsearch.index; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper; -import org.elasticsearch.index.mapper.DateFieldMapper; -import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; -import org.elasticsearch.index.mapper.DocumentMapper; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; -import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperServiceTestCase; -import org.elasticsearch.index.mapper.ParsedDocument; -import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.StringFieldScript; import org.elasticsearch.script.StringFieldScript.LeafFactory; import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xcontent.XContentFactory; -import org.elasticsearch.xcontent.XContentType; import java.io.IOException; import java.util.Map; -import java.util.concurrent.TimeUnit; -import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; public class TimeSeriesModeTests extends MapperServiceTestCase { @@ -84,231 +68,6 @@ public void testSortOrder() { assertThat(e.getMessage(), equalTo("[index.mode=time_series] is incompatible with [index.sort.order]")); } - public void testAddsTimestamp() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - DocumentMapper mapper = createMapperService(s, mapping(b -> {})).documentMapper(); - MappedFieldType timestamp = mapper.mappers().getFieldType(DataStreamTimestampFieldMapper.DEFAULT_PATH); - assertThat(timestamp, instanceOf(DateFieldType.class)); - assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.MILLISECONDS)); - - Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); - assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); - assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); - } - - public void testTimestampMillis() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) - .documentMapper(); - MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); - assertThat(timestamp, instanceOf(DateFieldType.class)); - assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.MILLISECONDS)); - - Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); - assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); - assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); - } - - public void testTimestampNanos() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date_nanos").endObject())) - .documentMapper(); - MappedFieldType timestamp = mapper.mappers().getFieldType("@timestamp"); - assertThat(timestamp, instanceOf(DateFieldType.class)); - assertThat(((DateFieldType) timestamp).resolution(), equalTo(DateFieldMapper.Resolution.NANOSECONDS)); - - Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); - assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); - assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); - } - - public void testBadTimestamp() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - String type = randomFrom("keyword", "integer", "long", "double", "text"); - Exception e = expectThrows( - IllegalArgumentException.class, - () -> createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", type).endObject())) - ); - assertThat( - e.getMessage(), - equalTo("data stream timestamp field [@timestamp] is of type [" + type + "], but [date,date_nanos] is expected") - ); - } - - public void testWithoutTimestamp() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - - DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) - .documentMapper(); - MapperParsingException e = expectThrows( - MapperParsingException.class, - () -> mapper.parse( - new SourceToParse("1", BytesReference.bytes(XContentFactory.jsonBuilder().startObject().endObject()), XContentType.JSON) - ) - ); - assertThat(e.getRootCause().getMessage(), containsString("data stream timestamp field [@timestamp] is missing")); - } - - public void testEnableTimestampRange() throws IOException { - long endTime = System.currentTimeMillis(); - long startTime = endTime - TimeUnit.DAYS.toMillis(1); - - Settings s = Settings.builder() - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - DocumentMapper mapper = createMapperService( - s, - mapping(b -> b.startObject("@timestamp").field("type", randomBoolean() ? "date" : "date_nanos").endObject()) - ).documentMapper(); - ParsedDocument doc = mapper.parse( - new SourceToParse( - "1", - BytesReference.bytes( - XContentFactory.jsonBuilder().startObject().field("@timestamp", randomLongBetween(startTime, endTime)).endObject() - ), - XContentType.JSON - ) - ); - // Look, mah, no failure. - assertNotNull(doc.rootDoc().getNumericValue("@timestamp")); - } - - public void testBadStartTime() throws IOException { - long endTime = System.currentTimeMillis(); - long startTime = endTime - TimeUnit.DAYS.toMillis(1); - - Settings s = Settings.builder() - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - - DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) - .documentMapper(); - MapperParsingException e = expectThrows( - MapperParsingException.class, - () -> mapper.parse( - new SourceToParse( - "1", - BytesReference.bytes( - XContentFactory.jsonBuilder() - .startObject() - .field("@timestamp", Math.max(startTime - randomLongBetween(1, 3), 0)) - .endObject() - ), - XContentType.JSON - ) - ) - ); - assertThat(e.getRootCause().getMessage(), containsString("must be larger than")); - } - - public void testBadEndTime() throws IOException { - long endTime = System.currentTimeMillis(); - long startTime = endTime - TimeUnit.DAYS.toMillis(1); - - Settings s = Settings.builder() - .put(IndexSettings.TIME_SERIES_START_TIME.getKey(), startTime) - .put(IndexSettings.TIME_SERIES_END_TIME.getKey(), endTime) - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - - DocumentMapper mapper = createMapperService(s, mapping(b -> b.startObject("@timestamp").field("type", "date").endObject())) - .documentMapper(); - MapperParsingException e = expectThrows( - MapperParsingException.class, - () -> mapper.parse( - new SourceToParse( - "1", - BytesReference.bytes( - XContentFactory.jsonBuilder().startObject().field("@timestamp", endTime + randomLongBetween(0, 3)).endObject() - ), - XContentType.JSON - ) - ) - ); - assertThat(e.getRootCause().getMessage(), containsString("must be smaller than")); - } - - public void testEnabledTimeStampMapper() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - XContentBuilder mappings = XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject(DataStreamTimestampFieldMapper.NAME); - if (randomBoolean()) { - mappings.field("enabled", true); - } else { - mappings.field("enabled", "true"); - } - mappings.endObject().endObject().endObject(); - - DocumentMapper mapper = createMapperService(s, mappings).documentMapper(); - Mapper timestampField = mapper.mappers().getMapper(DataStreamTimestampFieldMapper.NAME); - assertThat(timestampField, instanceOf(DataStreamTimestampFieldMapper.class)); - assertTrue(((DataStreamTimestampFieldMapper) timestampField).isEnabled()); - } - - public void testDisabledTimeStampMapper() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - XContentBuilder mappings = XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .startObject(DataStreamTimestampFieldMapper.NAME) - .field("enabled", false) - .endObject() - .endObject() - .endObject(); - - Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(s, mappings).documentMapper()); - assertThat( - e.getMessage(), - equalTo("Failed to parse mapping: time series index [_data_stream_timestamp] meta field must be enabled") - ); - } - - public void testBadTimeStampMapper() throws IOException { - Settings s = Settings.builder() - .put(IndexSettings.MODE.getKey(), "time_series") - .put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "foo") - .build(); - XContentBuilder mappings = XContentFactory.jsonBuilder() - .startObject() - .startObject("_doc") - .field(DataStreamTimestampFieldMapper.NAME, "enabled") - .endObject() - .endObject(); - - Exception e = expectThrows(MapperParsingException.class, () -> createMapperService(s, mappings).documentMapper()); - assertThat(e.getMessage(), equalTo("Failed to parse mapping: time series index [_data_stream_timestamp] meta field format error")); - } - public void testWithoutRoutingPath() { Settings s = Settings.builder().put(IndexSettings.MODE.getKey(), "time_series").build(); Exception e = expectThrows(IllegalArgumentException.class, () -> IndexSettings.MODE.get(s));