From 12e7cbe92bcda9767538baee7e1ce1cd934797ed Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 22 Jun 2015 00:16:53 -0700 Subject: [PATCH] Mappings: Lockdown _timestamp This is a follow up to #8143 and #6730 for _timestamp. It removes support for `path`, as well as any field type settings, and enables docvalues for _timestamp, for 2.0. Users who need to adjust these settings can use a date field. --- .../mapper/internal/TimestampFieldMapper.java | 31 ++++---- .../org/elasticsearch/document/BulkTests.java | 3 +- .../org/elasticsearch/get/GetActionTests.java | 74 ++++++------------- .../timestamp/TimestampMappingTests.java | 71 +++++++++--------- .../update/UpdateMappingOnClusterTests.java | 63 ---------------- .../mapper/update/UpdateMappingTests.java | 4 +- .../search/sort/SimpleSortTests.java | 4 +- .../test/ElasticsearchIntegrationTest.java | 6 +- .../timestamp/SimpleTimestampTests.java | 6 +- .../org/elasticsearch/ttl/SimpleTTLTests.java | 4 +- .../org/elasticsearch/update/UpdateTests.java | 8 +- .../mapping/fields/timestamp-field.asciidoc | 41 +--------- docs/reference/migration/migrate_2_0.asciidoc | 1 + .../test/index/70_timestamp.yaml | 1 - 14 files changed, 92 insertions(+), 225 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java index 66533827e1c47..056bf81893b04 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java @@ -161,7 +161,9 @@ public static class TypeParser implements Mapper.TypeParser { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { TimestampFieldMapper.Builder builder = timestamp(); - parseField(builder, builder.name, node, parserContext); + if (parserContext.indexVersionCreated().before(Version.V_2_0_0)) { + parseField(builder, builder.name, node, parserContext); + } boolean defaultSet = false; Boolean ignoreMissing = null; for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { @@ -172,7 +174,7 @@ public Mapper.Builder parse(String name, Map node, ParserContext EnabledAttributeMapper enabledState = nodeBooleanValue(fieldNode) ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED; builder.enabled(enabledState); iterator.remove(); - } else if (fieldName.equals("path")) { + } else if (fieldName.equals("path") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) { builder.path(fieldNode.toString()); iterator.remove(); } else if (fieldName.equals("format")) { @@ -265,11 +267,6 @@ public MappedFieldType defaultFieldType() { return defaultFieldType; } - @Override - public boolean defaultDocValues() { - return false; - } - public boolean enabled() { return this.enabledState.enabled; } @@ -340,14 +337,16 @@ && defaultDocValues() == fieldType().hasDocValues()) { if (includeDefaults || enabledState != Defaults.ENABLED) { builder.field("enabled", enabledState.enabled); } - if (includeDefaults || (indexed != indexedDefault) || (fieldType().tokenized() != Defaults.FIELD_TYPE.tokenized())) { + if (indexCreatedBefore2x && (includeDefaults || (indexed != indexedDefault) || (fieldType().tokenized() != Defaults.FIELD_TYPE.tokenized()))) { builder.field("index", indexTokenizeOptionToString(indexed, fieldType().tokenized())); } - if (includeDefaults || fieldType().stored() != Defaults.FIELD_TYPE.stored()) { + if (indexCreatedBefore2x && (includeDefaults || fieldType().stored() != Defaults.PRE_20_FIELD_TYPE.stored())) { builder.field("store", fieldType().stored()); } - doXContentDocValues(builder, includeDefaults); - if (includeDefaults || path != Defaults.PATH) { + if (indexCreatedBefore2x) { + doXContentDocValues(builder, includeDefaults); + } + if (indexCreatedBefore2x && (includeDefaults || path != Defaults.PATH)) { builder.field("path", path); } if (includeDefaults || !fieldType().dateTimeFormatter().format().equals(Defaults.DATE_TIME_FORMATTER.format())) { @@ -359,10 +358,12 @@ && defaultDocValues() == fieldType().hasDocValues()) { if (includeDefaults || ignoreMissing != null) { builder.field("ignore_missing", ignoreMissing); } - if (customFieldDataSettings != null) { - builder.field("fielddata", (Map) customFieldDataSettings.getAsMap()); - } else if (includeDefaults) { - builder.field("fielddata", (Map) fieldType().fieldDataType().getSettings().getAsMap()); + if (indexCreatedBefore2x) { + if (customFieldDataSettings != null) { + builder.field("fielddata", (Map) customFieldDataSettings.getAsMap()); + } else if (includeDefaults) { + builder.field("fielddata", (Map) fieldType().fieldDataType().getSettings().getAsMap()); + } } builder.endObject(); diff --git a/core/src/test/java/org/elasticsearch/document/BulkTests.java b/core/src/test/java/org/elasticsearch/document/BulkTests.java index 46cc704966fe7..8be32de6feb83 100644 --- a/core/src/test/java/org/elasticsearch/document/BulkTests.java +++ b/core/src/test/java/org/elasticsearch/document/BulkTests.java @@ -570,7 +570,8 @@ public void preParsingSourceDueToMappingShouldNotBreakCompleteBulkRequest() thro .endObject() .endObject() .endObject(); - assertAcked(prepareCreate("test").addMapping("type", builder)); + assertAcked(prepareCreate("test").addMapping("type", builder) + .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2_ID)); String brokenBuildRequestData = "{\"index\": {\"_id\": \"1\"}}\n" + "{\"name\": \"Malformed}\n" + diff --git a/core/src/test/java/org/elasticsearch/get/GetActionTests.java b/core/src/test/java/org/elasticsearch/get/GetActionTests.java index d73e91d849f3c..5f85a04a057fe 100644 --- a/core/src/test/java/org/elasticsearch/get/GetActionTests.java +++ b/core/src/test/java/org/elasticsearch/get/GetActionTests.java @@ -1051,28 +1051,32 @@ void indexSingleDocumentWithUngeneratedFieldsThatArePartOf_source(boolean stored client().prepareIndex("test", "doc").setId("1").setSource(doc).setRouting("1").get(); } - - @Test - public void testUngeneratedFieldsNotPartOfSourceUnstored() throws IOException { - indexSingleDocumentWithUngeneratedFieldsThatAreNeverPartOf_source(false, randomBoolean()); - String[] fieldsList = {"_timestamp"}; - String[] alwaysStoredFieldsList = {"_routing", "_size"}; - // before refresh - document is only in translog - assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList, "1"); - assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", alwaysStoredFieldsList, "1"); - refresh(); - //after refresh - document is in translog and also indexed - assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList, "1"); - assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", alwaysStoredFieldsList, "1"); - flush(); - //after flush - document is in not anymore translog - only indexed - assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList, "1"); - assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", alwaysStoredFieldsList, "1"); - } - @Test public void testUngeneratedFieldsNotPartOfSourceStored() throws IOException { - indexSingleDocumentWithUngeneratedFieldsThatAreNeverPartOf_source(true, randomBoolean()); + String createIndexSource = "{\n" + + " \"settings\": {\n" + + " \"index.translog.disable_flush\": true,\n" + + " \"refresh_interval\": \"-1\"\n" + + " },\n" + + " \"mappings\": {\n" + + " \"parentdoc\": {},\n" + + " \"doc\": {\n" + + " \"_timestamp\": {\n" + + " \"enabled\": true\n" + + " },\n" + + " \"_size\": {\n" + + " \"enabled\": true\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource)); + ensureGreen(); + String doc = "{\n" + + " \"text\": \"some text.\"\n" + + "}\n"; + client().prepareIndex("test", "doc").setId("1").setSource(doc).setRouting("1").get(); String[] fieldsList = {"_timestamp", "_size", "_routing"}; // before refresh - document is only in translog assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1"); @@ -1084,36 +1088,6 @@ public void testUngeneratedFieldsNotPartOfSourceStored() throws IOException { assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1"); } - void indexSingleDocumentWithUngeneratedFieldsThatAreNeverPartOf_source(boolean stored, boolean sourceEnabled) { - String storedString = stored ? "yes" : "no"; - String createIndexSource = "{\n" + - " \"settings\": {\n" + - " \"index.translog.disable_flush\": true,\n" + - " \"refresh_interval\": \"-1\"\n" + - " },\n" + - " \"mappings\": {\n" + - " \"parentdoc\": {},\n" + - " \"doc\": {\n" + - " \"_timestamp\": {\n" + - " \"store\": \"" + storedString + "\",\n" + - " \"enabled\": true\n" + - " },\n" + - " \"_size\": {\n" + - " \"enabled\": true\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; - - assertAcked(prepareCreate("test").addAlias(new Alias("alias")).setSource(createIndexSource)); - ensureGreen(); - String doc = "{\n" + - " \"text\": \"some text.\"\n" + - "}\n"; - client().prepareIndex("test", "doc").setId("1").setSource(doc).setRouting("1").get(); - } - - @Test public void testGeneratedStringFieldsUnstored() throws IOException { indexSingleDocumentWithStringFieldsGeneratedFromText(false, randomBoolean()); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java index 8c65418892d50..e8b0ce2db7b2f 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.Index; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MappedFieldType; @@ -69,6 +70,7 @@ /** */ public class TimestampMappingTests extends ElasticsearchSingleNodeTest { + Settings BWC_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); @Test public void testSimpleDisabled() throws Exception { @@ -87,7 +89,7 @@ public void testSimpleDisabled() throws Exception { @Test public void testEnabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("enabled", "yes").field("store", "yes").endObject() + .startObject("_timestamp").field("enabled", "yes").endObject() .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); BytesReference source = XContentFactory.jsonBuilder() @@ -110,19 +112,18 @@ public void testDefaultValues() throws Exception { XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_timestamp").endObject().endObject().string())) { DocumentMapper docMapper = createIndex("test", Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, version).build()).mapperService().documentMapperParser().parse(mapping); assertThat(docMapper.timestampFieldMapper().enabled(), equalTo(TimestampFieldMapper.Defaults.ENABLED.enabled)); - assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(version.onOrAfter(Version.V_2_0_0) ? true : false)); + assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(version.onOrAfter(Version.V_2_0_0))); assertThat(docMapper.timestampFieldMapper().fieldType().indexOptions(), equalTo(TimestampFieldMapper.Defaults.FIELD_TYPE.indexOptions())); assertThat(docMapper.timestampFieldMapper().path(), equalTo(TimestampFieldMapper.Defaults.PATH)); assertThat(docMapper.timestampFieldMapper().fieldType().dateTimeFormatter().format(), equalTo(TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT)); - assertThat(docMapper.timestampFieldMapper().fieldType().hasDocValues(), equalTo(false)); + assertThat(docMapper.timestampFieldMapper().fieldType().hasDocValues(), equalTo(version.onOrAfter(Version.V_2_0_0))); assertAcked(client().admin().indices().prepareDelete("test").execute().get()); } } } - @Test - public void testSetValues() throws Exception { + public void testBackcompatSetValues() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") .field("enabled", "yes").field("store", "no").field("index", "no") @@ -130,7 +131,7 @@ public void testSetValues() throws Exception { .field("doc_values", true) .endObject() .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + DocumentMapper docMapper = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser().parse(mapping); assertThat(docMapper.timestampFieldMapper().enabled(), equalTo(true)); assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(false)); assertEquals(IndexOptions.NONE, docMapper.timestampFieldMapper().fieldType().indexOptions()); @@ -142,7 +143,7 @@ public void testSetValues() throws Exception { @Test public void testThatDisablingDuringMergeIsWorking() throws Exception { String enabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("enabled", true).field("store", "yes").endObject() + .startObject("_timestamp").field("enabled", true).endObject() .endObject().endObject().string(); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); DocumentMapper enabledMapper = parser.parse(enabledMapping); @@ -162,7 +163,7 @@ public void testThatSerializationWorksCorrectlyForIndexField() throws Exception String enabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true).field("store", "yes").field("index", "no").endObject() .endObject().endObject().string(); - DocumentMapper enabledMapper = createIndex("test").mapperService().documentMapperParser().parse(enabledMapping); + DocumentMapper enabledMapper = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser().parse(enabledMapping); XContentBuilder builder = JsonXContent.contentBuilder().startObject(); enabledMapper.timestampFieldMapper().toXContent(builder, ToXContent.EMPTY_PARAMS).endObject(); @@ -176,7 +177,7 @@ public void testThatSerializationWorksCorrectlyForIndexField() throws Exception } @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] - public void testPathMissingDefaultValue() throws Exception { + public void testBackcompatPathMissingDefaultValue() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") .field("enabled", "yes") @@ -190,7 +191,7 @@ public void testPathMissingDefaultValue() throws Exception { .endObject(); MetaData metaData = MetaData.builder().build(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string()); + DocumentMapper docMapper = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser().parse(mapping.string()); MappingMetaData mappingMetaData = new MappingMetaData(docMapper); @@ -230,7 +231,7 @@ public void testTimestampDefaultValue() throws Exception { } @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] - public void testPathMissingDefaultToEpochValue() throws Exception { + public void testBackcompatPathMissingDefaultToEpochValue() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") .field("enabled", "yes") @@ -245,7 +246,7 @@ public void testPathMissingDefaultToEpochValue() throws Exception { .endObject(); MetaData metaData = MetaData.builder().build(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string()); + DocumentMapper docMapper = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser().parse(mapping.string()); MappingMetaData mappingMetaData = new MappingMetaData(docMapper); @@ -281,7 +282,7 @@ public void testTimestampMissingDefaultToEpochValue() throws Exception { } @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] - public void testPathMissingNowDefaultValue() throws Exception { + public void testBackcompatPathMissingNowDefaultValue() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") .field("enabled", "yes") @@ -296,7 +297,7 @@ public void testPathMissingNowDefaultValue() throws Exception { .endObject(); MetaData metaData = MetaData.builder().build(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string()); + DocumentMapper docMapper = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser().parse(mapping.string()); MappingMetaData mappingMetaData = new MappingMetaData(docMapper); @@ -355,7 +356,7 @@ public void testPathMissingWithForcedNullDefaultShouldFail() throws Exception { } @Test // Issue 4718: was throwing a TimestampParsingException: failed to parse timestamp [null] - public void testPathMissingShouldFail() throws Exception { + public void testBackcompatPathMissingShouldFail() throws Exception { XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") .field("enabled", "yes") @@ -369,7 +370,7 @@ public void testPathMissingShouldFail() throws Exception { .endObject(); MetaData metaData = MetaData.builder().build(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping.string()); + DocumentMapper docMapper = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser().parse(mapping.string()); MappingMetaData mappingMetaData = new MappingMetaData(docMapper); @@ -522,14 +523,10 @@ public void testMergingFielddataLoadingWorks() throws Exception { @Test public void testParsingNotDefaultTwiceDoesNotChangeMapping() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("enabled", true) - .field("index", randomBoolean() ? "no" : "analyzed") // default is "not_analyzed" which will be omitted when building the source again - .field("doc_values", true) - .field("path", "foo") - .field("default", "1970-01-01") - .startObject("fielddata").field("format", "doc_values").endObject() - .endObject() - .endObject().endObject().string(); + .startObject("_timestamp") + .field("enabled", true) + .field("default", "1970-01-01") + .endObject().endObject().endObject().string(); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); DocumentMapper docMapper = parser.parse(mapping); @@ -538,7 +535,7 @@ public void testParsingNotDefaultTwiceDoesNotChangeMapping() throws Exception { } @Test - public void testParsingTwiceDoesNotChangeTokenizeValue() throws Exception { + public void testBackcompatParsingTwiceDoesNotChangeTokenizeValue() throws Exception { String[] index_options = {"no", "analyzed", "not_analyzed"}; String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true) @@ -551,7 +548,7 @@ public void testParsingTwiceDoesNotChangeTokenizeValue() throws Exception { .startObject("properties") .endObject() .endObject().endObject().string(); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + DocumentMapperParser parser = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser(); DocumentMapper docMapper = parser.parse(mapping); boolean tokenized = docMapper.timestampFieldMapper().fieldType().tokenized(); @@ -603,7 +600,7 @@ public void testMergingConflicts() throws Exception { } @Test - public void testMergingConflictsForIndexValues() throws Exception { + public void testBackcompatMergingConflictsForIndexValues() throws Exception { List indexValues = new ArrayList<>(); indexValues.add("analyzed"); indexValues.add("no"); @@ -614,7 +611,7 @@ public void testMergingConflictsForIndexValues() throws Exception { .field("index", indexValues.remove(randomInt(2))) .endObject() .endObject().endObject().string(); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + DocumentMapperParser parser = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser(); DocumentMapper docMapper = parser.parse(mapping); mapping = XContentFactory.jsonBuilder().startObject() @@ -656,9 +653,9 @@ public void testInitMappers() throws IOException { } @Test - public void testMergePaths() throws Exception { + public void testBackcompatMergePaths() throws Exception { String[] possiblePathValues = {"some_path", "anotherPath", null}; - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + DocumentMapperParser parser = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser(); XContentBuilder mapping1 = XContentFactory.jsonBuilder().startObject() .startObject("type") .startObject("_timestamp"); @@ -691,7 +688,7 @@ void assertConflict(String mapping1, String mapping2, DocumentMapperParser parse } } - public void testDocValuesSerialization() throws Exception { + public void testBackcompatDocValuesSerialization() throws Exception { // default String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp") @@ -737,7 +734,7 @@ public void testDocValuesSerialization() throws Exception { } void assertDocValuesSerialization(String mapping) throws Exception { - DocumentMapperParser parser = createIndex("test_doc_values").mapperService().documentMapperParser(); + DocumentMapperParser parser = createIndex("test_doc_values", BWC_SETTINGS).mapperService().documentMapperParser(); DocumentMapper docMapper = parser.parse(mapping); boolean docValues = docMapper.timestampFieldMapper().fieldType().hasDocValues(); docMapper = parser.parse(docMapper.mappingSource().string()); @@ -745,11 +742,11 @@ void assertDocValuesSerialization(String mapping) throws Exception { assertAcked(client().admin().indices().prepareDelete("test_doc_values")); } - public void testPath() throws Exception { + public void testBackcompatPath() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true).field("path", "custom_timestamp").endObject() .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + DocumentMapper docMapper = createIndex("test", BWC_SETTINGS).mapperService().documentMapperParser().parse(mapping); XContentBuilder doc = XContentFactory.jsonBuilder().startObject().field("custom_timestamp", 1).endObject(); MappingMetaData mappingMetaData = new MappingMetaData(docMapper); @@ -778,12 +775,12 @@ public void testIncludeInObjectBackcompat() throws Exception { public void testThatEpochCanBeIgnoredWithCustomFormat() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("enabled", true).field("format", "yyyyMMddHH").field("path", "custom_timestamp").endObject() + .startObject("_timestamp").field("enabled", true).field("format", "yyyyMMddHH").endObject() .endObject().endObject().string(); DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); - XContentBuilder doc = XContentFactory.jsonBuilder().startObject().field("custom_timestamp", 2015060210).endObject(); - IndexRequest request = new IndexRequest("test", "type", "1").source(doc); + XContentBuilder doc = XContentFactory.jsonBuilder().startObject().endObject(); + IndexRequest request = new IndexRequest("test", "type", "1").source(doc).timestamp("2015060210"); MappingMetaData mappingMetaData = new MappingMetaData(docMapper); request.process(MetaData.builder().build(), mappingMetaData, true, "test"); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java index ecdad4ebe311b..e12efa6a58dc1 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java @@ -19,19 +19,16 @@ package org.elasticsearch.index.mapper.update; -import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.junit.Test; import java.util.HashMap; -import java.util.LinkedHashMap; import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; @@ -179,64 +176,4 @@ private void compareMappingOnNodes(GetMappingsResponse previousMapping) { assertThat(previousMapping.getMappings().get(INDEX).get(TYPE).source(), equalTo(currentMapping.getMappings().get(INDEX).get(TYPE).source())); } } - - @Test - public void testUpdateTimestamp() throws Exception { - boolean enabled = randomBoolean(); - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("enabled", enabled).startObject("fielddata").field("loading", "lazy").field("format", "doc_values").endObject().field("store", "no").endObject() - .endObject().endObject(); - client().admin().indices().prepareCreate("test").addMapping("type", mapping).get(); - GetMappingsResponse appliedMappings = client().admin().indices().prepareGetMappings("test").get(); - LinkedHashMap timestampMapping = (LinkedHashMap) appliedMappings.getMappings().get("test").get("type").getSourceAsMap().get("_timestamp"); - assertThat((Boolean) timestampMapping.get("store"), equalTo(false)); - assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("loading"), equalTo("lazy")); - assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("format"), equalTo("doc_values")); - mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("enabled", enabled).startObject("fielddata").field("loading", "eager").field("format", "array").endObject().field("store", "no").endObject() - .endObject().endObject(); - PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get(); - appliedMappings = client().admin().indices().prepareGetMappings("test").get(); - timestampMapping = (LinkedHashMap) appliedMappings.getMappings().get("test").get("type").getSourceAsMap().get("_timestamp"); - assertThat((Boolean) timestampMapping.get("store"), equalTo(false)); - assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("loading"), equalTo("eager")); - assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("format"), equalTo("array")); - } - - @Test - @LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/10297") - public void testTimestampMergingConflicts() throws Exception { - String mapping = XContentFactory.jsonBuilder().startObject().startObject(TYPE) - .startObject("_timestamp").field("enabled", true) - .startObject("fielddata").field("format", "doc_values").endObject() - .field("store", "yes") - .field("index", "analyzed") - .field("path", "foo") - .field("default", "1970-01-01") - .endObject() - .endObject().endObject().string(); - - client().admin().indices().prepareCreate(INDEX).addMapping(TYPE, mapping).get(); - - mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("enabled", false) - .startObject("fielddata").field("format", "array").endObject() - .field("store", "no") - .field("index", "no") - .field("path", "bar") - .field("default", "1970-01-02") - .endObject() - .endObject().endObject().string(); - GetMappingsResponse mappingsBeforeUpdateResponse = client().admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).get(); - try { - client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(mapping).get(); - fail("This should result in conflicts when merging the mapping"); - } catch (MergeMappingException e) { - String[] expectedConflicts = {"mapper [_timestamp] has different index values", "mapper [_timestamp] has different store values", "Cannot update default in _timestamp value. Value is 1970-01-01 now encountering 1970-01-02", "Cannot update path in _timestamp value. Value is foo path in merged mapping is bar"}; - for (String conflict : expectedConflicts) { - assertThat(e.getDetailedMessage(), containsString(conflict)); - } - } - compareMappingOnNodes(mappingsBeforeUpdateResponse); - } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java index 80b7cadce702f..0c8263d9cde7b 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java @@ -19,7 +19,9 @@ package org.elasticsearch.index.mapper.update; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; @@ -132,7 +134,7 @@ public void testIndexFieldParsing() throws IOException { @Test public void testTimestampParsing() throws IOException { - IndexService indexService = createIndex("test", Settings.settingsBuilder().build()); + IndexService indexService = createIndex("test", Settings.settingsBuilder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build()); XContentBuilder indexMapping = XContentFactory.jsonBuilder(); boolean enabled = randomBoolean(); indexMapping.startObject() diff --git a/core/src/test/java/org/elasticsearch/search/sort/SimpleSortTests.java b/core/src/test/java/org/elasticsearch/search/sort/SimpleSortTests.java index b3259ae439ab8..1cf81f87e990e 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/SimpleSortTests.java +++ b/core/src/test/java/org/elasticsearch/search/sort/SimpleSortTests.java @@ -1562,10 +1562,8 @@ public void testSortOnRareField() throws IOException { } public void testSortMetaField() throws Exception { - final boolean idDocValues = random().nextBoolean(); - final boolean timestampDocValues = random().nextBoolean(); XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("enabled", true).field("store", true).field("index", !timestampDocValues || randomBoolean() ? "not_analyzed" : "no").field("doc_values", timestampDocValues).endObject() + .startObject("_timestamp").field("enabled", true).endObject() .endObject().endObject(); assertAcked(prepareCreate("test") .addMapping("type", mapping)); diff --git a/core/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java b/core/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java index 8450dfd6ccf60..a86964002769e 100644 --- a/core/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java +++ b/core/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java @@ -373,12 +373,8 @@ public void randomIndexTemplate() throws IOException { if (frequently() && randomDynamicTemplates()) { mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_"); if (randomBoolean()) { - boolean timestampEnabled = randomBoolean(); mappings.startObject(TimestampFieldMapper.NAME) - .field("enabled", timestampEnabled); - if (timestampEnabled) { - mappings.field("doc_values", randomBoolean()); - } + .field("enabled", randomBoolean()); mappings.endObject(); } if (randomBoolean()) { diff --git a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampTests.java b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampTests.java index 8f10547096934..90365f68c15f2 100644 --- a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampTests.java +++ b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampTests.java @@ -46,7 +46,7 @@ public class SimpleTimestampTests extends ElasticsearchIntegrationTest { public void testSimpleTimestamp() throws Exception { client().admin().indices().prepareCreate("test") - .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("_timestamp").field("enabled", true).field("store", "yes").endObject().endObject().endObject()) + .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("_timestamp").field("enabled", true).endObject().endObject().endObject()) .execute().actionGet(); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); @@ -119,14 +119,14 @@ public void testThatTimestampCanBeSwitchedOnAndOff() throws Exception { String index = "foo"; String type = "mytype"; - XContentBuilder builder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", true).field("store", true).endObject().endObject(); + XContentBuilder builder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", true).endObject().endObject(); assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); // check mapping again assertTimestampMappingEnabled(index, type, true); // update some field in the mapping - XContentBuilder updateMappingBuilder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", false).field("store", true).endObject().endObject(); + XContentBuilder updateMappingBuilder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", false).endObject().endObject(); PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping(index).setType(type).setSource(updateMappingBuilder).get(); assertAcked(putMappingResponse); diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLTests.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLTests.java index dde0202f1584d..573c0090fc381 100644 --- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLTests.java +++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLTests.java @@ -69,14 +69,14 @@ public void testSimpleTTL() throws Exception { .addMapping("type1", XContentFactory.jsonBuilder() .startObject() .startObject("type1") - .startObject("_timestamp").field("enabled", true).field("store", "yes").endObject() + .startObject("_timestamp").field("enabled", true).endObject() .startObject("_ttl").field("enabled", true).endObject() .endObject() .endObject()) .addMapping("type2", XContentFactory.jsonBuilder() .startObject() .startObject("type2") - .startObject("_timestamp").field("enabled", true).field("store", "yes").endObject() + .startObject("_timestamp").field("enabled", true).endObject() .startObject("_ttl").field("enabled", true).field("default", "1d").endObject() .endObject() .endObject())); diff --git a/core/src/test/java/org/elasticsearch/update/UpdateTests.java b/core/src/test/java/org/elasticsearch/update/UpdateTests.java index 5c0e14c0db16b..d38f28b208c9f 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateTests.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateTests.java @@ -70,7 +70,7 @@ private void createTestIndex() throws Exception { .addMapping("type1", XContentFactory.jsonBuilder() .startObject() .startObject("type1") - .startObject("_timestamp").field("enabled", true).field("store", "yes").endObject() + .startObject("_timestamp").field("enabled", true).endObject() .startObject("_ttl").field("enabled", true).endObject() .endObject() .endObject())); @@ -476,7 +476,7 @@ public void testContextVariables() throws Exception { .addMapping("type1", XContentFactory.jsonBuilder() .startObject() .startObject("type1") - .startObject("_timestamp").field("enabled", true).field("store", "yes").endObject() + .startObject("_timestamp").field("enabled", true).endObject() .startObject("_ttl").field("enabled", true).endObject() .endObject() .endObject()) @@ -484,7 +484,7 @@ public void testContextVariables() throws Exception { .startObject() .startObject("subtype1") .startObject("_parent").field("type", "type1").endObject() - .startObject("_timestamp").field("enabled", true).field("store", "yes").endObject() + .startObject("_timestamp").field("enabled", true).endObject() .startObject("_ttl").field("enabled", true).endObject() .endObject() .endObject()) @@ -634,7 +634,7 @@ public void stressUpdateDeleteConcurrency() throws Exception { .addMapping("type1", XContentFactory.jsonBuilder() .startObject() .startObject("type1") - .startObject("_timestamp").field("enabled", true).field("store", "yes").endObject() + .startObject("_timestamp").field("enabled", true).endObject() .startObject("_ttl").field("enabled", true).endObject() .endObject() .endObject()) diff --git a/docs/reference/mapping/fields/timestamp-field.asciidoc b/docs/reference/mapping/fields/timestamp-field.asciidoc index 0fb1a91d7f500..b510cef9c2ec6 100644 --- a/docs/reference/mapping/fields/timestamp-field.asciidoc +++ b/docs/reference/mapping/fields/timestamp-field.asciidoc @@ -2,8 +2,7 @@ === `_timestamp` The `_timestamp` field allows to automatically index the timestamp of a -document. It can be provided externally via the index request or in the -`_source`. If it is not provided externally it will be automatically set +document. If it is not provided it will be automatically set to a <>. [float] @@ -21,44 +20,6 @@ should be defined: } -------------------------------------------------- -[float] -==== store / index - -By default the `_timestamp` field has `store` set to `true` and `index` -set to `not_analyzed`. It can be queried as a standard date field. - -[float] -==== path - -The `_timestamp` value can be provided as an external value when -indexing. But, it can also be automatically extracted from the document -to index based on a `path`. For example, having the following mapping: - -[source,js] --------------------------------------------------- -{ - "tweet" : { - "_timestamp" : { - "enabled" : true, - "path" : "post_date" - } - } -} --------------------------------------------------- - -Will cause `2009-11-15T14:12:12` to be used as the timestamp value for: - -[source,js] --------------------------------------------------- -{ - "message" : "You know, for Search", - "post_date" : "2009-11-15T14:12:12" -} --------------------------------------------------- - -Note, using `path` without explicit timestamp value provided requires an -additional (though quite fast) parsing phase. - [float] [[mapping-timestamp-field-format]] ==== format diff --git a/docs/reference/migration/migrate_2_0.asciidoc b/docs/reference/migration/migrate_2_0.asciidoc index 1d422c35963c8..f7604af4949da 100644 --- a/docs/reference/migration/migrate_2_0.asciidoc +++ b/docs/reference/migration/migrate_2_0.asciidoc @@ -295,6 +295,7 @@ to provide special features. They now have limited configuration options. * `_boost` has been removed. * `_field_names` configuration is limited to disabling the field. * `_size` configuration is limited to enabling the field. +* `_timestamp` configuration is limited to enabling the field, setting format and default value ==== Meta fields in documents Meta fields can no longer be specified within a document. They should be specified diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml index 863df9ebfa8ad..938a358b103e6 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml @@ -9,7 +9,6 @@ test: _timestamp: enabled: 1 - store: yes - do: cluster.health: wait_for_status: yellow