diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java index ec1024c07f54a..33d194a7892e9 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java @@ -21,7 +21,6 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; -import org.elasticsearch.Version; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -101,9 +100,6 @@ public TTLFieldMapper build(BuilderContext context) { public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha4)) { - throw new IllegalArgumentException("[_ttl] is removed in 5.0. As a replacement, you should use time based indexes or cron a delete-by-query with a range query on a timestamp field."); - } Builder builder = new Builder(); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); @@ -169,9 +165,6 @@ private TTLFieldMapper(Settings indexSettings) { private TTLFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, long defaultTTL, Settings indexSettings) { super(NAME, fieldType, Defaults.TTL_FIELD_TYPE, indexSettings); - if (enabled.enabled && Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha4)) { - throw new IllegalArgumentException("[_ttl] is removed in 5.0. As a replacement, you should use time based indexes or cron a delete-by-query with a range query on a timestamp field."); - } this.enabledState = enabled; this.defaultTTL = defaultTTL; } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java index 24a86b11392e8..09f1144db28a1 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java @@ -22,7 +22,6 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.IndexOptions; -import org.elasticsearch.Version; import org.elasticsearch.action.TimestampParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.joda.FormatDateTimeFormatter; @@ -127,9 +126,6 @@ public TimestampFieldMapper build(BuilderContext context) { public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { - if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha4)) { - throw new IllegalArgumentException("[_timestamp] is removed in 5.0. As a replacement, you can use an ingest pipeline to add a field with the current timestamp to your documents."); - } Builder builder = new Builder(parserContext.mapperService().fullName(NAME), parserContext.mapperService().getIndexSettings().getSettings()); boolean defaultSet = false; Boolean ignoreMissing = null; @@ -204,9 +200,6 @@ private TimestampFieldMapper(Settings indexSettings, MappedFieldType existing) { private TimestampFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, EnabledAttributeMapper enabledState, String defaultTimestamp, Boolean ignoreMissing, Settings indexSettings) { super(NAME, fieldType, defaultFieldType, indexSettings); - if (enabledState.enabled && Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha4)) { - throw new IllegalArgumentException("[_timestamp] is removed in 5.0. As a replacement, you can use an ingest pipeline to add a field with the current timestamp to your documents."); - } this.enabledState = enabledState; this.defaultTimestamp = defaultTimestamp; this.ignoreMissing = ignoreMissing; diff --git a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java index 5ac1bf40af6cc..2a16625d03726 100644 --- a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java +++ b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java @@ -1035,13 +1035,13 @@ public void testAddAliasWithFilterNoMapping() throws Exception { } public void testAliasFilterWithNowInRangeFilterAndQuery() throws Exception { - assertAcked(prepareCreate("my-index").addMapping("my-type", "timestamp", "type=date")); - assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", rangeQuery("timestamp").from("2016-12-01").to("2016-12-31"))); - assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter2", rangeQuery("timestamp").from("2016-01-01").to("2016-12-31"))); + assertAcked(prepareCreate("my-index").addMapping("my-type", "_timestamp", "enabled=true")); + assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", rangeQuery("_timestamp").from("now-1d").to("now"))); + assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter2", rangeQuery("_timestamp").from("now-1d").to("now"))); final int numDocs = scaledRandomIntBetween(5, 52); for (int i = 1; i <= numDocs; i++) { - client().prepareIndex("my-index", "my-type").setCreate(true).setSource("timestamp", "2016-12-12").get(); + client().prepareIndex("my-index", "my-type").setCreate(true).setSource("{}").get(); if (i % 2 == 0) { refresh(); SearchResponse response = client().prepareSearch("filter1").get(); diff --git a/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java index 4773aafbf3f9f..72db338ab809d 100644 --- a/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java @@ -30,7 +30,6 @@ import org.elasticsearch.test.ESIntegTestCase.Scope; import java.io.IOException; -import java.util.Map; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @@ -112,18 +111,16 @@ public void testCustomDefaultMapping() throws Exception { internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false)); createIndex("test"); - assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("timestamp", "type=date")); + assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("_timestamp", "enabled=true")); MappingMetaData defaultMapping = client().admin().cluster().prepareState().get().getState().getMetaData().getIndices().get("test").getMappings().get("_default_"); - Map properties = (Map) defaultMapping.getSourceAsMap().get("properties"); - assertThat(properties.get("timestamp"), notNullValue()); + assertThat(defaultMapping.getSourceAsMap().get("_timestamp"), notNullValue()); - assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("timestamp", "type=date")); + assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("_timestamp", "enabled=true")); assertAcked(client().admin().indices().preparePutMapping("test").setType("type1").setSource("foo", "enabled=true")); MappingMetaData type1Mapping = client().admin().cluster().prepareState().get().getState().getMetaData().getIndices().get("test").getMappings().get("type1"); - properties = (Map) type1Mapping.getSourceAsMap().get("properties"); - assertThat(properties.get("timestamp"), notNullValue()); + assertThat(type1Mapping.getSourceAsMap().get("_timestamp"), notNullValue()); } public void testAliasFilterValidation() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java index b73f563fbb3a0..8c4b699cbff54 100644 --- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java +++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java @@ -20,6 +20,7 @@ package org.elasticsearch.get; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.flush.FlushResponse; @@ -29,6 +30,7 @@ import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.get.MultiGetRequestBuilder; import org.elasticsearch.action.get.MultiGetResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; @@ -38,11 +40,15 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; import java.io.IOException; +import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.Map; import java.util.Set; import static java.util.Collections.singleton; @@ -51,6 +57,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -529,7 +536,7 @@ public void testMultiGetWithVersion() throws Exception { public void testGetFieldsMetaData() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") - .addMapping("my-type1", "_parent", "type=parent") + .addMapping("my-type1", "_timestamp", "enabled=true", "_ttl", "enabled=true", "_parent", "type=parent") .addAlias(new Alias("alias")) .setSettings(Settings.builder().put("index.refresh_interval", -1))); @@ -550,6 +557,12 @@ public void testGetFieldsMetaData() throws Exception { assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value")); assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true)); assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1")); + assertThat(getResponse.getField("_timestamp").isMetadataField(), equalTo(true)); + assertThat(getResponse.getField("_timestamp").getValue().toString(), equalTo("205097")); + assertThat(getResponse.getField("_ttl").isMetadataField(), equalTo(true)); + // TODO: _ttl should return the original value, but it does not work today because + // it would use now() instead of the value of _timestamp to rebase + // assertThat(getResponse.getField("_ttl").getValue().toString(), equalTo("10000000205097")); assertThat(getResponse.getField("_parent").isMetadataField(), equalTo(true)); assertThat(getResponse.getField("_parent").getValue().toString(), equalTo("parent_1")); @@ -564,6 +577,12 @@ public void testGetFieldsMetaData() throws Exception { assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value")); assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true)); assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1")); + assertThat(getResponse.getField("_timestamp").isMetadataField(), equalTo(true)); + assertThat(getResponse.getField("_timestamp").getValue().toString(), equalTo("205097")); + assertThat(getResponse.getField("_ttl").isMetadataField(), equalTo(true)); + // TODO: _ttl should return the original value, but it does not work today because + // it would use now() instead of the value of _timestamp to rebase + //assertThat(getResponse.getField("_ttl").getValue().toString(), equalTo("10000000000000")); assertThat(getResponse.getField("_parent").isMetadataField(), equalTo(true)); assertThat(getResponse.getField("_parent").getValue().toString(), equalTo("parent_1")); } @@ -760,10 +779,16 @@ public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException { " },\n" + " \"mappings\": {\n" + " \"parentdoc\": {\n" + + " \"_ttl\": {\n" + + " \"enabled\": true\n" + + " }\n" + " },\n" + " \"doc\": {\n" + " \"_parent\": {\n" + " \"type\": \"parentdoc\"\n" + + " },\n" + + " \"_ttl\": {\n" + + " \"enabled\": true\n" + " }\n" + " }\n" + " }\n" + @@ -773,7 +798,7 @@ public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException { client().prepareIndex("test", "doc").setId("1").setSource("{}").setParent("1").setTTL(TimeValue.timeValueHours(1).getMillis()).get(); - String[] fieldsList = {"_parent"}; + String[] fieldsList = {"_ttl", "_parent"}; // before refresh - document is only in translog assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1"); refresh(); @@ -789,6 +814,14 @@ public void testUngeneratedFieldsNotPartOfSourceStored() throws IOException { " \"settings\": {\n" + " \"index.translog.flush_threshold_size\": \"1pb\",\n" + " \"refresh_interval\": \"-1\"\n" + + " },\n" + + " \"mappings\": {\n" + + " \"parentdoc\": {},\n" + + " \"doc\": {\n" + + " \"_timestamp\": {\n" + + " \"enabled\": true\n" + + " }\n" + + " }\n" + " }\n" + "}"; @@ -798,7 +831,7 @@ public void testUngeneratedFieldsNotPartOfSourceStored() throws IOException { " \"text\": \"some text.\"\n" + "}\n"; client().prepareIndex("test", "doc").setId("1").setSource(doc).setRouting("1").get(); - String[] fieldsList = {"_routing"}; + String[] fieldsList = {"_timestamp", "_routing"}; // before refresh - document is only in translog assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1"); refresh(); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java index 165b49d3145c1..966146655b883 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.internal.AllFieldMapper; +import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -49,7 +50,9 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; @@ -427,6 +430,23 @@ public void testRootObjectMapperPropertiesDoNotCauseException() throws IOExcepti parser.parse("test", new CompressedXContent(mapping)); } + // issue https://github.com/elastic/elasticsearch/issues/5864 + public void testMetadataMappersStillWorking() throws MapperParsingException, IOException { + String mapping = "{"; + Map rootTypes = new HashMap<>(); + //just pick some example from DocumentMapperParser.rootTypeParsers + rootTypes.put(TimestampFieldMapper.NAME, "{\"enabled\" : true}"); + rootTypes.put("include_in_all", "true"); + rootTypes.put("dynamic_date_formats", "[\"yyyy-MM-dd\", \"dd-MM-yyyy\"]"); + rootTypes.put("numeric_detection", "true"); + rootTypes.put("dynamic_templates", "[]"); + for (String key : rootTypes.keySet()) { + mapping += "\"" + key+ "\"" + ":" + rootTypes.get(key) + ",\n"; + } + mapping += "\"properties\":{}}" ; + createIndex("test").mapperService().documentMapperParser().parse("test", new CompressedXContent(mapping)); + } + public void testDocValuesNotAllowed() throws IOException { String mapping = jsonBuilder().startObject().startObject("type") .startObject("_all") diff --git a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java index 78da5abb746b9..14d86290db41b 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java @@ -22,7 +22,6 @@ import org.apache.lucene.index.IndexOptions; import org.elasticsearch.Version; import org.elasticsearch.action.TimestampParsingException; -import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; @@ -32,31 +31,25 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.joda.Joda; -import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; -import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; -import org.elasticsearch.test.VersionUtils; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collection; -import java.util.LinkedHashMap; -import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; @@ -65,35 +58,19 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.startsWith; /** */ public class TimestampMappingTests extends ESSingleNodeTestCase { - private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build(); - @Override protected Collection> getPlugins() { return pluginList(InternalSettingsPlugin.class); } - public void testRejectedOn5x() throws IOException { - String mapping = XContentFactory.jsonBuilder().startObject() - .startObject("type") - .startObject("_timestamp") - .field("enabled", true) - .endObject() - .endObject().endObject().string(); - IndexService index = createIndex("test"); - IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, - () -> index.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false)); - assertThat(expected.getMessage(), startsWith("[_timestamp] is removed")); - } - public void testSimpleDisabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); BytesReference source = XContentFactory.jsonBuilder() .startObject() .field("field", "value") @@ -108,7 +85,7 @@ public void testEnabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", "yes").endObject() .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); BytesReference source = XContentFactory.jsonBuilder() .startObject() .field("field", "value") @@ -122,7 +99,10 @@ public void testEnabled() throws Exception { } public void testDefaultValues() throws Exception { - Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.V_5_0_0_alpha3); + Version version; + do { + version = randomVersion(random()); + } while (version.before(Version.V_2_0_0_beta1)); for (String mapping : Arrays.asList( XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(), XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_timestamp").endObject().endObject().endObject().string())) { @@ -140,7 +120,7 @@ public void testThatDisablingDuringMergeIsWorking() throws Exception { String enabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true).endObject() .endObject().endObject().string(); - MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService(); + MapperService mapperService = createIndex("test").mapperService(); DocumentMapper enabledMapper = mapperService.merge("type", new CompressedXContent(enabledMapping), MapperService.MergeReason.MAPPING_UPDATE, false); String disabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type") @@ -166,7 +146,7 @@ public void testTimestampMissingDefaultToEpochValue() throws Exception { .field("foo", "bar") .endObject(); - DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData(); MappingMetaData mappingMetaData = new MappingMetaData(docMapper); @@ -192,7 +172,7 @@ public void testTimestampMissingNowDefaultValue() throws Exception { .endObject(); MetaData metaData = MetaData.builder().build(); - DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); MappingMetaData mappingMetaData = new MappingMetaData(docMapper); @@ -215,7 +195,7 @@ public void testPathMissingWithForcedNullDefaultShouldFail() throws Exception { .endObject() .endObject().endObject(); try { - createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); + createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set to null"); } catch (TimestampParsingException e) { assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null")); @@ -232,7 +212,7 @@ public void testTimestampMissingWithForcedNullDefaultShouldFail() throws Excepti .endObject().endObject(); try { - createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); + createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set to null"); } catch (TimestampParsingException e) { assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null")); @@ -250,7 +230,7 @@ public void testTimestampDefaultAndIgnore() throws Exception { .endObject().endObject(); try { - createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); + createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set with ignore_missing set to false"); } catch (TimestampParsingException e) { assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set with ignore_missing set to false")); @@ -270,7 +250,7 @@ public void testTimestampMissingShouldNotFail() throws Exception { .endObject(); MetaData metaData = MetaData.builder().build(); - DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); MappingMetaData mappingMetaData = new MappingMetaData(docMapper); @@ -343,7 +323,7 @@ public void testParsingNotDefaultTwiceDoesNotChangeMapping() throws Exception { .field("enabled", true) .field("default", "1970-01-01") .endObject().endObject().endObject().string(); - DocumentMapperParser parser = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser(); + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping)); docMapper = parser.parse("type", docMapper.mappingSource()); @@ -380,7 +360,7 @@ public void testIncludeInObjectNotAllowed() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true).field("default", "1970").field("format", "YYYY").endObject() .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); try { docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -395,7 +375,7 @@ public void testThatEpochCanBeIgnoredWithCustomFormat() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true).field("format", "yyyyMMddHH").endObject() .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData(); XContentBuilder doc = XContentFactory.jsonBuilder().startObject().endObject(); @@ -412,7 +392,7 @@ public void testThatIndicesAfter2_0DontSupportUnixTimestampsInAnyDateFormat() th .endObject().endObject().string(); BytesReference source = XContentFactory.jsonBuilder().startObject().field("field", "value").endObject().bytes(); // test with 2.x - DocumentMapper currentMapper = createIndex("new-index", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper currentMapper = createIndex("new-index").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); // this works with 2.x IndexRequest request = new IndexRequest("new-index", "type", "1").source(source).timestamp("1970-01-01"); @@ -427,54 +407,4 @@ public void testThatIndicesAfter2_0DontSupportUnixTimestampsInAnyDateFormat() th assertThat(e.getMessage(), containsString("failed to parse timestamp [1234567890]")); } } - - public void testSizeTimestampIndexParsing() throws IOException { - IndexService indexService = createIndex("test", BW_SETTINGS); - String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json"); - DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(mapping), true); - assertThat(documentMapper.mappingSource().string(), equalTo(mapping)); - documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true); - assertThat(documentMapper.mappingSource().string(), equalTo(mapping)); - } - - public void testDefaultApplied() throws IOException { - createIndex("test1", BW_SETTINGS); - createIndex("test2", BW_SETTINGS); - XContentBuilder defaultMapping = XContentFactory.jsonBuilder().startObject() - .startObject(MapperService.DEFAULT_MAPPING).startObject("_timestamp").field("enabled", true).endObject().endObject() - .endObject(); - client().admin().indices().preparePutMapping().setType(MapperService.DEFAULT_MAPPING).setSource(defaultMapping).get(); - XContentBuilder typeMapping = XContentFactory.jsonBuilder().startObject() - .startObject("type").startObject("_all").field("enabled", false).endObject().endObject() - .endObject(); - client().admin().indices().preparePutMapping("test1").setType("type").setSource(typeMapping).get(); - client().admin().indices().preparePutMapping("test1", "test2").setType("type").setSource(typeMapping).get(); - - GetMappingsResponse response = client().admin().indices().prepareGetMappings("test2").get(); - assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_all")); - assertFalse((Boolean) ((LinkedHashMap) response.getMappings().get("test2").get("type").getSourceAsMap().get("_all")).get("enabled")); - assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp")); - assertTrue((Boolean)((LinkedHashMap)response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp")).get("enabled")); - } - - public void testTimestampParsing() throws IOException { - IndexService indexService = createIndex("test", BW_SETTINGS); - XContentBuilder indexMapping = XContentFactory.jsonBuilder(); - boolean enabled = randomBoolean(); - indexMapping.startObject() - .startObject("type") - .startObject("_timestamp") - .field("enabled", enabled) - .endObject() - .endObject() - .endObject(); - DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(indexMapping.string()), true); - assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled)); - assertTrue(documentMapper.timestampFieldMapper().fieldType().stored()); - assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues()); - documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true); - assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled)); - assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues()); - assertTrue(documentMapper.timestampFieldMapper().fieldType().stored()); - } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java index ad9e03327b274..fe05704588296 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java @@ -20,9 +20,7 @@ package org.elasticsearch.index.mapper.ttl; import org.apache.lucene.index.IndexOptions; -import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; @@ -32,48 +30,22 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.internal.TTLFieldMapper; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import java.io.IOException; -import java.util.Collection; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.startsWith; public class TTLMappingTests extends ESSingleNodeTestCase { - - private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build(); - - @Override - protected Collection> getPlugins() { - return pluginList(InternalSettingsPlugin.class); - } - - public void testRejectedOn5x() throws IOException { - String mapping = XContentFactory.jsonBuilder().startObject() - .startObject("type") - .startObject("_ttl") - .field("enabled", true) - .endObject() - .endObject().endObject().string(); - IndexService index = createIndex("test"); - IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, - () -> index.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false)); - assertThat(expected.getMessage(), startsWith("[_ttl] is removed")); - } - public void testSimpleDisabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); BytesReference source = XContentFactory.jsonBuilder() .startObject() .field("field", "value") @@ -88,7 +60,7 @@ public void testEnabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_ttl").field("enabled", "yes").endObject() .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); BytesReference source = XContentFactory.jsonBuilder() .startObject() .field("field", "value") @@ -103,7 +75,7 @@ public void testEnabled() throws Exception { public void testDefaultValues() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); assertThat(docMapper.TTLFieldMapper().enabled(), equalTo(TTLFieldMapper.Defaults.ENABLED_STATE.enabled)); assertThat(docMapper.TTLFieldMapper().fieldType().stored(), equalTo(TTLFieldMapper.Defaults.TTL_FIELD_TYPE.stored())); assertThat(docMapper.TTLFieldMapper().fieldType().indexOptions(), equalTo(TTLFieldMapper.Defaults.TTL_FIELD_TYPE.indexOptions())); @@ -121,7 +93,7 @@ public void testThatEnablingTTLFieldOnMergeWorks() throws Exception { .startObject("properties").field("field").startObject().field("type", "text").endObject().endObject() .endObject().endObject().string(); - MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService(); + MapperService mapperService = createIndex("test").mapperService(); DocumentMapper mapperWithoutTtl = mapperService.merge("type", new CompressedXContent(mappingWithoutTtl), MapperService.MergeReason.MAPPING_UPDATE, false); DocumentMapper mapperWithTtl = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false); @@ -144,7 +116,7 @@ public void testThatChangingTTLKeepsMapperEnabled() throws Exception { .startObject("properties").field("field").startObject().field("type", "text").endObject().endObject() .endObject().endObject().string(); - MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService(); + MapperService mapperService = createIndex("test").mapperService(); DocumentMapper initialMapper = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false); DocumentMapper updatedMapper = mapperService.merge("type", new CompressedXContent(updatedMapping), MapperService.MergeReason.MAPPING_UPDATE, false); @@ -155,7 +127,7 @@ public void testThatChangingTTLKeepsMapperEnabled() throws Exception { public void testThatDisablingTTLReportsConflict() throws Exception { String mappingWithTtl = getMappingWithTtlEnabled().string(); String mappingWithTtlDisabled = getMappingWithTtlDisabled().string(); - MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService(); + MapperService mapperService = createIndex("test").mapperService(); DocumentMapper initialMapper = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false); try { @@ -171,7 +143,7 @@ public void testThatDisablingTTLReportsConflict() throws Exception { public void testThatDisablingTTLReportsConflictOnCluster() throws Exception { String mappingWithTtl = getMappingWithTtlEnabled().string(); String mappingWithTtlDisabled = getMappingWithTtlDisabled().string(); - assertAcked(client().admin().indices().prepareCreate("testindex").setSettings(BW_SETTINGS).addMapping("type", mappingWithTtl)); + assertAcked(client().admin().indices().prepareCreate("testindex").addMapping("type", mappingWithTtl)); GetMappingsResponse mappingsBeforeUpdateResponse = client().admin().indices().prepareGetMappings("testindex").addTypes("type").get(); try { client().admin().indices().preparePutMapping("testindex").setSource(mappingWithTtlDisabled).setType("type").get(); @@ -186,7 +158,7 @@ public void testThatDisablingTTLReportsConflictOnCluster() throws Exception { public void testThatEnablingTTLAfterFirstDisablingWorks() throws Exception { String mappingWithTtl = getMappingWithTtlEnabled().string(); String withTtlDisabled = getMappingWithTtlDisabled().string(); - assertAcked(client().admin().indices().prepareCreate("testindex").setSettings(BW_SETTINGS).addMapping("type", withTtlDisabled)); + assertAcked(client().admin().indices().prepareCreate("testindex").addMapping("type", withTtlDisabled)); GetMappingsResponse mappingsAfterUpdateResponse = client().admin().indices().prepareGetMappings("testindex").addTypes("type").get(); assertThat(mappingsAfterUpdateResponse.getMappings().get("testindex").get("type").sourceAsMap().get("_ttl").toString(), equalTo("{enabled=false}")); client().admin().indices().preparePutMapping("testindex").setSource(mappingWithTtl).setType("type").get(); @@ -195,20 +167,20 @@ public void testThatEnablingTTLAfterFirstDisablingWorks() throws Exception { } public void testNoConflictIfNothingSetAndDisabledLater() throws Exception { - IndexService indexService = createIndex("testindex", BW_SETTINGS, "type"); + IndexService indexService = createIndex("testindex", Settings.builder().build(), "type"); XContentBuilder mappingWithTtlDisabled = getMappingWithTtlDisabled("7d"); indexService.mapperService().merge("type", new CompressedXContent(mappingWithTtlDisabled.string()), MapperService.MergeReason.MAPPING_UPDATE, false); } public void testNoConflictIfNothingSetAndEnabledLater() throws Exception { - IndexService indexService = createIndex("testindex", BW_SETTINGS, "type"); + IndexService indexService = createIndex("testindex", Settings.builder().build(), "type"); XContentBuilder mappingWithTtlEnabled = getMappingWithTtlEnabled("7d"); indexService.mapperService().merge("type", new CompressedXContent(mappingWithTtlEnabled.string()), MapperService.MergeReason.MAPPING_UPDATE, false); } public void testMergeWithOnlyDefaultSet() throws Exception { XContentBuilder mappingWithTtlEnabled = getMappingWithTtlEnabled("7d"); - IndexService indexService = createIndex("testindex", BW_SETTINGS, "type", mappingWithTtlEnabled); + IndexService indexService = createIndex("testindex", Settings.builder().build(), "type", mappingWithTtlEnabled); XContentBuilder mappingWithOnlyDefaultSet = getMappingWithOnlyTtlDefaultSet("6m"); indexService.mapperService().merge("type", new CompressedXContent(mappingWithOnlyDefaultSet.string()), MapperService.MergeReason.MAPPING_UPDATE, false); CompressedXContent mappingAfterMerge = indexService.mapperService().documentMapper("type").mappingSource(); @@ -217,7 +189,7 @@ public void testMergeWithOnlyDefaultSet() throws Exception { public void testMergeWithOnlyDefaultSetTtlDisabled() throws Exception { XContentBuilder mappingWithTtlEnabled = getMappingWithTtlDisabled("7d"); - IndexService indexService = createIndex("testindex", BW_SETTINGS, "type", mappingWithTtlEnabled); + IndexService indexService = createIndex("testindex", Settings.builder().build(), "type", mappingWithTtlEnabled); CompressedXContent mappingAfterCreation = indexService.mapperService().documentMapper("type").mappingSource(); assertThat(mappingAfterCreation, equalTo(new CompressedXContent("{\"type\":{\"_ttl\":{\"enabled\":false},\"properties\":{\"field\":{\"type\":\"text\"}}}}"))); XContentBuilder mappingWithOnlyDefaultSet = getMappingWithOnlyTtlDefaultSet("6m"); @@ -230,7 +202,7 @@ public void testIncludeInObjectNotAllowed() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_ttl").field("enabled", true).endObject() .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); try { docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() diff --git a/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java b/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java index 3041cc8d13c87..1d849d5093274 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java @@ -31,12 +31,12 @@ public class ParseMappingTypeLevelTests extends ESSingleNodeTestCase { public void testTypeLevel() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_all").field("enabled", false).endObject() + .startObject("_timestamp").field("enabled", true).endObject() .endObject().endObject().string(); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); assertThat(mapper.type(), equalTo("type")); - assertThat(mapper.allFieldMapper().enabled(), equalTo(false)); + assertThat(mapper.timestampFieldMapper().enabled(), equalTo(true)); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java index afac48e2b8901..ed20cdb7e3521 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java @@ -139,6 +139,16 @@ public void testDocValuesInvalidMappingOnUpdate() throws Exception { compareMappingOnNodes(mappingsBeforeUpdateResponse); } + // checks if the setting for timestamp and size are kept even if disabled + public void testDisabledSizeTimestampIndexDoNotLooseMappings() throws Exception { + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json"); + prepareCreate(INDEX).addMapping(TYPE, mapping).get(); + GetMappingsResponse mappingsBeforeGreen = client().admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).get(); + ensureGreen(INDEX); + // make sure all nodes have same cluster state + compareMappingOnNodes(mappingsBeforeGreen); + } + protected void testConflict(String mapping, String mappingUpdate, String... errorMessages) throws InterruptedException { assertAcked(prepareCreate(INDEX).setSource(mapping).get()); ensureGreen(INDEX); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java index 86624d938b63b..17cb8a84664fa 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java @@ -259,6 +259,56 @@ public void testReuseMetaFieldBackCompat() throws IOException { } } + public void testTimestampParsing() throws IOException { + IndexService indexService = createIndex("test"); + XContentBuilder indexMapping = XContentFactory.jsonBuilder(); + boolean enabled = randomBoolean(); + indexMapping.startObject() + .startObject("type") + .startObject("_timestamp") + .field("enabled", enabled) + .endObject() + .endObject() + .endObject(); + DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(indexMapping.string()), true); + assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled)); + assertTrue(documentMapper.timestampFieldMapper().fieldType().stored()); + assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues()); + documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true); + assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled)); + assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues()); + assertTrue(documentMapper.timestampFieldMapper().fieldType().stored()); + } + + public void testSizeTimestampIndexParsing() throws IOException { + IndexService indexService = createIndex("test", Settings.builder().build()); + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json"); + DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(mapping), true); + assertThat(documentMapper.mappingSource().string(), equalTo(mapping)); + documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true); + assertThat(documentMapper.mappingSource().string(), equalTo(mapping)); + } + + public void testDefaultApplied() throws IOException { + createIndex("test1", Settings.builder().build()); + createIndex("test2", Settings.builder().build()); + XContentBuilder defaultMapping = XContentFactory.jsonBuilder().startObject() + .startObject(MapperService.DEFAULT_MAPPING).startObject("_timestamp").field("enabled", true).endObject().endObject() + .endObject(); + client().admin().indices().preparePutMapping().setType(MapperService.DEFAULT_MAPPING).setSource(defaultMapping).get(); + XContentBuilder typeMapping = XContentFactory.jsonBuilder().startObject() + .startObject("type").startObject("_all").field("enabled", false).endObject().endObject() + .endObject(); + client().admin().indices().preparePutMapping("test1").setType("type").setSource(typeMapping).get(); + client().admin().indices().preparePutMapping("test1", "test2").setType("type").setSource(typeMapping).get(); + + GetMappingsResponse response = client().admin().indices().prepareGetMappings("test2").get(); + assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_all")); + assertFalse((Boolean) ((LinkedHashMap) response.getMappings().get("test2").get("type").getSourceAsMap().get("_all")).get("enabled")); + assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp")); + assertTrue((Boolean)((LinkedHashMap)response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp")).get("enabled")); + } + public void testRejectFieldDefinedTwice() throws IOException { String mapping1 = XContentFactory.jsonBuilder().startObject() .startObject("type1") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index 4f7064a33bb2b..cdb722ff9ddc8 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -65,6 +65,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; @@ -106,7 +107,8 @@ private IndexRequestBuilder indexDoc(int month, int day, int value) throws Excep @Override public void setupSuiteScopeCluster() throws Exception { - createIndex("idx", "idx_unmapped"); + assertAcked(prepareCreate("idx").addMapping("type", "_timestamp", "enabled=true")); + createIndex("idx_unmapped"); // TODO: would be nice to have more random data here assertAcked(prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer")); List builders = new ArrayList<>(); @@ -1139,6 +1141,13 @@ public void testExceptionOnNegativeInterval() { } } + public void testTimestampField() { // see #11692 + SearchResponse response = client().prepareSearch("idx").addAggregation(dateHistogram("histo").field("_timestamp").dateHistogramInterval(randomFrom(DateHistogramInterval.DAY, DateHistogramInterval.MONTH))).get(); + assertSearchResponse(response); + Histogram histo = response.getAggregations().get("histo"); + assertThat(histo.getBuckets().size(), greaterThan(0)); + } + /** * When DST ends, local time turns back one hour, so between 2am and 4am wall time we should have four buckets: * "2015-10-25T02:00:00.000+02:00", diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java index 99183b11c58fb..188ae22471a4e 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.hamcrest.Matchers; @@ -69,8 +70,10 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; +import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -1337,12 +1340,16 @@ public void testSortOnRareField() throws IOException { } public void testSortMetaField() throws Exception { - createIndex("test"); + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_timestamp").field("enabled", true).endObject() + .endObject().endObject(); + assertAcked(prepareCreate("test") + .addMapping("type", mapping)); ensureGreen(); final int numDocs = randomIntBetween(10, 20); IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs]; for (int i = 0; i < numDocs; ++i) { - indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i)) + indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i)).setTimestamp(Integer.toString(randomInt(1000))) .setSource(); } indexRandom(true, indexReqs); @@ -1361,6 +1368,37 @@ public void testSortMetaField() throws Exception { assertThat(previous, order == SortOrder.ASC ? lessThan(uid) : greaterThan(uid)); previous = uid; } + + /* + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(randomIntBetween(1, numDocs + 5)) + .addSort("_id", order) + .execute().actionGet(); + assertNoFailures(searchResponse); + hits = searchResponse.getHits().hits(); + previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM; + for (int i = 0; i < hits.length; ++i) { + final BytesRef id = new BytesRef(Uid.createUid(hits[i].type(), hits[i].id())); + assertThat(previous, order == SortOrder.ASC ? lessThan(id) : greaterThan(id)); + previous = id; + }*/ + + searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(randomIntBetween(1, numDocs + 5)) + .addSort("_timestamp", order) + .addField("_timestamp") + .execute().actionGet(); + assertNoFailures(searchResponse); + hits = searchResponse.getHits().hits(); + Long previousTs = order == SortOrder.ASC ? 0 : Long.MAX_VALUE; + for (int i = 0; i < hits.length; ++i) { + SearchHitField timestampField = hits[i].getFields().get("_timestamp"); + Long timestamp = timestampField.getValue(); + assertThat(previousTs, order == SortOrder.ASC ? lessThanOrEqualTo(timestamp) : greaterThanOrEqualTo(timestamp)); + previousTs = timestamp; + } } /** diff --git a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java index bd24ab9f43b36..2fa94bc2357c1 100644 --- a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java +++ b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java @@ -19,20 +19,14 @@ package org.elasticsearch.timestamp; -import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.Priority; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; -import java.util.Collection; import java.util.Locale; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -47,17 +41,8 @@ /** */ public class SimpleTimestampIT extends ESIntegTestCase { - - private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build(); - - @Override - protected Collection> nodePlugins() { - return pluginList(InternalSettingsPlugin.class); - } - public void testSimpleTimestamp() throws Exception { client().admin().indices().prepareCreate("test") - .setSettings(BW_SETTINGS) .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("_timestamp").field("enabled", true).endObject().endObject().endObject()) .execute().actionGet(); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); @@ -113,7 +98,7 @@ public void testThatUpdatingMappingShouldNotRemoveTimestampConfiguration() throw String type = "mytype"; XContentBuilder builder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", true).endObject().endObject(); - assertAcked(client().admin().indices().prepareCreate(index).setSettings(BW_SETTINGS).addMapping(type, builder)); + assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); // check mapping again assertTimestampMappingEnabled(index, type, true); @@ -132,7 +117,7 @@ public void testThatTimestampCanBeSwitchedOnAndOff() throws Exception { String type = "mytype"; XContentBuilder builder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", true).endObject().endObject(); - assertAcked(client().admin().indices().prepareCreate(index).setSettings(BW_SETTINGS).addMapping(type, builder)); + assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); // check mapping again assertTimestampMappingEnabled(index, type, true); diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java index 9e08ecde6fa8e..003c08134bc51 100644 --- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java +++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java @@ -19,7 +19,6 @@ package org.elasticsearch.ttl; -import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; @@ -27,19 +26,14 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import java.io.IOException; -import java.util.Collection; -import java.util.Collections; import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -66,11 +60,6 @@ protected int numberOfShards() { return 2; } - @Override - protected Collection> nodePlugins() { - return Collections.singleton(InternalSettingsPlugin.class); - } - @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() @@ -81,7 +70,6 @@ protected Settings nodeSettings(int nodeOrdinal) { public void testSimpleTTL() throws Exception { assertAcked(prepareCreate("test") - .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id) .addMapping("type1", XContentFactory.jsonBuilder() .startObject() .startObject("type1") @@ -221,7 +209,7 @@ public void testThatUpdatingMappingShouldNotRemoveTTLConfiguration() throws Exce String type = "mytype"; XContentBuilder builder = jsonBuilder().startObject().startObject("_ttl").field("enabled", true).endObject().endObject(); - assertAcked(client().admin().indices().prepareCreate(index).setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id).addMapping(type, builder)); + assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); // check mapping again assertTTLMappingEnabled(index, type); @@ -244,7 +232,6 @@ public void testThatUpdatingMappingShouldNotRemoveTTLConfiguration() throws Exce */ public void testNoopUpdate() throws IOException { assertAcked(prepareCreate("test") - .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id) .addMapping("type1", XContentFactory.jsonBuilder() .startObject() .startObject("type1") diff --git a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java deleted file mode 100644 index 3573089fcaa99..0000000000000 --- a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.update; - -import org.elasticsearch.Version; -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.cluster.metadata.IndexMetaData; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.index.engine.DocumentMissingException; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptService; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHitField; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Map; - -import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.lessThanOrEqualTo; - -public class TimestampTTLBWIT extends ESIntegTestCase { - - @Override - protected Collection> nodePlugins() { - return Arrays.asList( - UpdateIT.FieldIncrementScriptPlugin.class, - UpdateIT.ExtractContextInSourceScriptPlugin.class, - UpdateIT.PutFieldValuesScriptPlugin.class, - InternalSettingsPlugin.class - ); - } - - public void testSort() throws Exception { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("enabled", true).endObject() - .endObject().endObject(); - assertAcked(prepareCreate("test") - .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id) - .addMapping("type", mapping)); - ensureGreen(); - final int numDocs = randomIntBetween(10, 20); - IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs]; - for (int i = 0; i < numDocs; ++i) { - indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i)).setTimestamp(Integer.toString(randomInt(1000))) - .setSource(); - } - indexRandom(true, indexReqs); - - SortOrder order = randomFrom(SortOrder.values()); - - SearchResponse searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(randomIntBetween(1, numDocs + 5)) - .addSort("_timestamp", order) - .addField("_timestamp") - .execute().actionGet(); - assertNoFailures(searchResponse); - SearchHit[] hits = searchResponse.getHits().hits(); - Long previousTs = order == SortOrder.ASC ? 0 : Long.MAX_VALUE; - for (int i = 0; i < hits.length; ++i) { - SearchHitField timestampField = hits[i].getFields().get("_timestamp"); - Long timestamp = timestampField.getValue(); - assertThat(previousTs, order == SortOrder.ASC ? lessThanOrEqualTo(timestamp) : greaterThanOrEqualTo(timestamp)); - previousTs = timestamp; - } - } - - public void testUpdate() throws Exception { - assertAcked(prepareCreate("test").addAlias(new Alias("alias")) - .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id) - .addMapping("type1", XContentFactory.jsonBuilder() - .startObject() - .startObject("type1") - .startObject("_timestamp").field("enabled", true).endObject() - .startObject("_ttl").field("enabled", true).endObject() - .endObject() - .endObject())); - - ensureGreen(); - - try { - client().prepareUpdate(indexOrAlias(), "type1", "1") - .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet(); - fail(); - } catch (DocumentMissingException e) { - // all is well - } - - // check TTL is kept after an update without TTL - client().prepareIndex("test", "type1", "2").setSource("field", 1).setTTL(86400000L).setRefreshPolicy(IMMEDIATE).get(); - GetResponse getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet(); - long ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue(); - assertThat(ttl, greaterThan(0L)); - client().prepareUpdate(indexOrAlias(), "type1", "2") - .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet(); - getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet(); - ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue(); - assertThat(ttl, greaterThan(0L)); - - // check TTL update - client().prepareUpdate(indexOrAlias(), "type1", "2") - .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", - Collections.singletonMap("_ctx", Collections.singletonMap("_ttl", 3600000)))).execute().actionGet(); - getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet(); - ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue(); - assertThat(ttl, greaterThan(0L)); - assertThat(ttl, lessThanOrEqualTo(3600000L)); - - // check timestamp update - client().prepareIndex("test", "type1", "3").setSource("field", 1).setRefreshPolicy(IMMEDIATE).get(); - client().prepareUpdate(indexOrAlias(), "type1", "3") - .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", - Collections.singletonMap("_ctx", Collections.singletonMap("_timestamp", "2009-11-15T14:12:12")))).execute() - .actionGet(); - getResponse = client().prepareGet("test", "type1", "3").setFields("_timestamp").execute().actionGet(); - long timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue(); - assertThat(timestamp, equalTo(1258294332000L)); - } - - public void testContextVariables() throws Exception { - assertAcked(prepareCreate("test").addAlias(new Alias("alias")) - .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id) - .addMapping("type1", XContentFactory.jsonBuilder() - .startObject() - .startObject("type1") - .startObject("_timestamp").field("enabled", true).endObject() - .startObject("_ttl").field("enabled", true).endObject() - .endObject() - .endObject()) - .addMapping("subtype1", XContentFactory.jsonBuilder() - .startObject() - .startObject("subtype1") - .startObject("_parent").field("type", "type1").endObject() - .startObject("_timestamp").field("enabled", true).endObject() - .startObject("_ttl").field("enabled", true).endObject() - .endObject() - .endObject()) - ); - ensureGreen(); - - // Index some documents - long timestamp = System.currentTimeMillis(); - client().prepareIndex() - .setIndex("test") - .setType("type1") - .setId("parentId1") - .setTimestamp(String.valueOf(timestamp-1)) - .setSource("field1", 0, "content", "bar") - .execute().actionGet(); - - long ttl = 10000; - client().prepareIndex() - .setIndex("test") - .setType("subtype1") - .setId("id1") - .setParent("parentId1") - .setRouting("routing1") - .setTimestamp(String.valueOf(timestamp)) - .setTTL(ttl) - .setSource("field1", 1, "content", "foo") - .execute().actionGet(); - - // Update the first object and note context variables values - UpdateResponse updateResponse = client().prepareUpdate("test", "subtype1", "id1") - .setRouting("routing1") - .setScript(new Script("", ScriptService.ScriptType.INLINE, "extract_ctx", null)) - .execute().actionGet(); - - assertEquals(2, updateResponse.getVersion()); - - GetResponse getResponse = client().prepareGet("test", "subtype1", "id1").setRouting("routing1").execute().actionGet(); - Map updateContext = (Map) getResponse.getSourceAsMap().get("update_context"); - assertEquals("test", updateContext.get("_index")); - assertEquals("subtype1", updateContext.get("_type")); - assertEquals("id1", updateContext.get("_id")); - assertEquals(1, updateContext.get("_version")); - assertEquals("parentId1", updateContext.get("_parent")); - assertEquals("routing1", updateContext.get("_routing")); - assertThat(((Integer) updateContext.get("_ttl")).longValue(), allOf(greaterThanOrEqualTo(ttl-3000), lessThanOrEqualTo(ttl))); - - // Idem with the second object - updateResponse = client().prepareUpdate("test", "type1", "parentId1") - .setScript(new Script("", ScriptService.ScriptType.INLINE, "extract_ctx", null)) - .execute().actionGet(); - - assertEquals(2, updateResponse.getVersion()); - - getResponse = client().prepareGet("test", "type1", "parentId1").execute().actionGet(); - updateContext = (Map) getResponse.getSourceAsMap().get("update_context"); - assertEquals("test", updateContext.get("_index")); - assertEquals("type1", updateContext.get("_type")); - assertEquals("parentId1", updateContext.get("_id")); - assertEquals(1, updateContext.get("_version")); - assertNull(updateContext.get("_parent")); - assertNull(updateContext.get("_routing")); - assertNull(updateContext.get("_ttl")); - } - - private static String indexOrAlias() { - return randomBoolean() ? "test" : "alias"; - } -} diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java index 55834c181b008..4121543a7a079 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java @@ -62,12 +62,14 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; @@ -355,14 +357,21 @@ protected Collection> nodePlugins() { FieldIncrementScriptPlugin.class, ScriptedUpsertScriptPlugin.class, ExtractContextInSourceScriptPlugin.class, - InternalSettingsPlugin.class + InternalSettingsPlugin.class // uses index.merge.enabled ); } private void createTestIndex() throws Exception { logger.info("--> creating index test"); - assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); + assertAcked(prepareCreate("test").addAlias(new Alias("alias")) + .addMapping("type1", XContentFactory.jsonBuilder() + .startObject() + .startObject("type1") + .startObject("_timestamp").field("enabled", true).endObject() + .startObject("_ttl").field("enabled", true).endObject() + .endObject() + .endObject())); } public void testUpsert() throws Exception { @@ -629,6 +638,34 @@ public void testUpdate() throws Exception { assertThat(getResponse.isExists(), equalTo(false)); } + // check TTL is kept after an update without TTL + client().prepareIndex("test", "type1", "2").setSource("field", 1).setTTL(86400000L).setRefreshPolicy(IMMEDIATE).get(); + GetResponse getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet(); + long ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue(); + assertThat(ttl, greaterThan(0L)); + client().prepareUpdate(indexOrAlias(), "type1", "2") + .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet(); + getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet(); + ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue(); + assertThat(ttl, greaterThan(0L)); + + // check TTL update + client().prepareUpdate(indexOrAlias(), "type1", "2") + .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("_ctx", Collections.singletonMap("_ttl", 3600000)))).execute().actionGet(); + getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet(); + ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue(); + assertThat(ttl, greaterThan(0L)); + assertThat(ttl, lessThanOrEqualTo(3600000L)); + + // check timestamp update + client().prepareIndex("test", "type1", "3").setSource("field", 1).setRefreshPolicy(IMMEDIATE).get(); + client().prepareUpdate(indexOrAlias(), "type1", "3") + .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("_ctx", Collections.singletonMap("_timestamp", "2009-11-15T14:12:12")))).execute() + .actionGet(); + getResponse = client().prepareGet("test", "type1", "3").setFields("_timestamp").execute().actionGet(); + long timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue(); + assertThat(timestamp, equalTo(1258294332000L)); + // check fields parameter client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet(); updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1") @@ -645,7 +682,7 @@ public void testUpdate() throws Exception { client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet(); updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("field2", 2).endObject()).execute().actionGet(); for (int i = 0; i < 5; i++) { - GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet(); + getResponse = client().prepareGet("test", "type1", "1").execute().actionGet(); assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("1")); assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2")); } @@ -653,7 +690,7 @@ public void testUpdate() throws Exception { // change existing field updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("field", 3).endObject()).execute().actionGet(); for (int i = 0; i < 5; i++) { - GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet(); + getResponse = client().prepareGet("test", "type1", "1").execute().actionGet(); assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("3")); assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2")); } @@ -671,7 +708,7 @@ public void testUpdate() throws Exception { client().prepareIndex("test", "type1", "1").setSource("map", testMap).execute().actionGet(); updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("map", testMap3).endObject()).execute().actionGet(); for (int i = 0; i < 5; i++) { - GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet(); + getResponse = client().prepareGet("test", "type1", "1").execute().actionGet(); Map map1 = (Map) getResponse.getSourceAsMap().get("map"); assertThat(map1.size(), equalTo(3)); assertThat(map1.containsKey("map1"), equalTo(true)); @@ -723,12 +760,16 @@ public void testContextVariables() throws Exception { .addMapping("type1", XContentFactory.jsonBuilder() .startObject() .startObject("type1") + .startObject("_timestamp").field("enabled", true).endObject() + .startObject("_ttl").field("enabled", true).endObject() .endObject() .endObject()) .addMapping("subtype1", XContentFactory.jsonBuilder() .startObject() .startObject("subtype1") .startObject("_parent").field("type", "type1").endObject() + .startObject("_timestamp").field("enabled", true).endObject() + .startObject("_ttl").field("enabled", true).endObject() .endObject() .endObject()) ); @@ -772,6 +813,7 @@ public void testContextVariables() throws Exception { assertEquals(1, updateContext.get("_version")); assertEquals("parentId1", updateContext.get("_parent")); assertEquals("routing1", updateContext.get("_routing")); + assertThat(((Integer) updateContext.get("_ttl")).longValue(), allOf(greaterThanOrEqualTo(ttl-3000), lessThanOrEqualTo(ttl))); // Idem with the second object updateResponse = client().prepareUpdate("test", "type1", "parentId1") @@ -862,6 +904,13 @@ public void run() { public void testStressUpdateDeleteConcurrency() throws Exception { //We create an index with merging disabled so that deletes don't get merged away assertAcked(prepareCreate("test") + .addMapping("type1", XContentFactory.jsonBuilder() + .startObject() + .startObject("type1") + .startObject("_timestamp").field("enabled", true).endObject() + .startObject("_ttl").field("enabled", true).endObject() + .endObject() + .endObject()) .setSettings(Settings.builder().put(MergePolicyConfig.INDEX_MERGE_ENABLED, false))); ensureGreen(); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java index 28894c5c1cc01..d834f0fcf6d88 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.Priority; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -44,6 +45,7 @@ import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; @@ -86,6 +88,8 @@ public void testStoredFields() throws Exception { client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet(); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") + // _timestamp is randomly enabled via templates but we don't want it here to test stored fields behaviour + .startObject("_timestamp").field("enabled", false).endObject() .startObject("properties") .startObject("field1").field("type", "text").field("store", true).endObject() .startObject("field2").field("type", "text").field("store", false).endObject() @@ -694,7 +698,7 @@ public void testScriptFields() throws Exception { public void testLoadMetadata() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") - .addMapping("my-type1", "_parent", "type=parent")); + .addMapping("my-type1", "_timestamp", "enabled=true", "_ttl", "enabled=true", "_parent", "type=parent")); indexRandom(true, client().prepareIndex("test", "my-type1", "1") @@ -713,6 +717,12 @@ public void testLoadMetadata() throws Exception { assertThat(fields.get("field1"), nullValue()); assertThat(fields.get("_routing").isMetadataField(), equalTo(true)); assertThat(fields.get("_routing").getValue().toString(), equalTo("1")); + assertThat(fields.get("_timestamp").isMetadataField(), equalTo(true)); + assertThat(fields.get("_timestamp").getValue().toString(), equalTo("205097")); + assertThat(fields.get("_ttl").isMetadataField(), equalTo(true)); + // TODO: _ttl should return the original value, but it does not work today because + // it would use now() instead of the value of _timestamp to rebase + // assertThat(fields.get("_ttl").getValue().toString(), equalTo("10000000205097")); assertThat(fields.get("_parent").isMetadataField(), equalTo(true)); assertThat(fields.get("_parent").getValue().toString(), equalTo("parent_1")); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc index 688d8cbdc5bbc..304d31978cf0f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc @@ -255,7 +255,7 @@ Supports also regular expressions with flag X for more readability (accepts whit Compares two numeric values, eg: .... - - lt: { foo: 10000 } # the `foo` value is less than 10,000 + - lt: { _ttl: 10000 } # the `_ttl` value is less than 10,000 .... === `lte` and `gte` @@ -263,7 +263,7 @@ Compares two numeric values, eg: Compares two numeric values, eg: .... - - lte: { foo: 10000 } # the `foo` value is less than or equal to 10,000 + - lte: { _ttl: 10000 } # the `_ttl` value is less than or equal to 10,000 .... === `length` diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml new file mode 100644 index 0000000000000..83173817b6e7d --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml @@ -0,0 +1,80 @@ +--- +"Timestamp": + + - do: + indices.create: + index: test_1 + body: + mappings: + test: + _timestamp: + enabled: 1 + - do: + cluster.health: + wait_for_status: yellow + +# blank timestamp + - do: + create: + index: test_1 + type: test + id: 1 + body: { foo: bar } + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _timestamp + + - is_true: _timestamp + +# milliseconds since epoch + + - do: + delete: + index: test_1 + type: test + id: 1 + - do: + create: + index: test_1 + type: test + id: 1 + body: { foo: bar } + timestamp: 1372011280000 + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _timestamp + + - match: { _timestamp: 1372011280000 } + +# date format + + - do: + delete: + index: test_1 + type: test + id: 1 + - do: + create: + index: test_1 + type: test + id: 1 + body: { foo: bar } + timestamp: 2013-06-23T18:14:40 + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _timestamp + + - match: { _timestamp: 1372011280000 } + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml new file mode 100644 index 0000000000000..3f9b476366936 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml @@ -0,0 +1,100 @@ +--- +"TTL": + + - do: + indices.create: + index: test_1 + body: + mappings: + test: + _ttl: + enabled: 1 + default: 10s + - do: + cluster.health: + wait_for_status: yellow + +# blank ttl + - do: + create: + index: test_1 + type: test + id: 1 + body: { foo: bar } + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _ttl + + - lte: { _ttl: 10000} + - gt: { _ttl: 0} + +# milliseconds + + - do: + delete: + index: test_1 + type: test + id: 1 + - do: + create: + index: test_1 + type: test + id: 1 + body: { foo: bar } + ttl: 100000ms + - do: + get: + index: test_1 + type: test + id: 1 + fields: _ttl + + - lte: { _ttl: 100000} + - gt: { _ttl: 10000} + +# duration + + - do: + delete: + index: test_1 + type: test + id: 1 + - do: + create: + index: test_1 + type: test + id: 1 + body: { foo: bar } + ttl: 20s + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _ttl + + - lte: { _ttl: 20000} + - gt: { _ttl: 10000} + +# with timestamp + + - do: + delete: + index: test_1 + type: test + id: 1 + - do: + catch: /already_expired_exception/ + create: + index: test_1 + type: test + id: 1 + body: { foo: bar } + ttl: 20s + timestamp: 2013-06-23T18:14:40 + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml new file mode 100644 index 0000000000000..938a358b103e6 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml @@ -0,0 +1,70 @@ +--- +"Timestamp": + + - do: + indices.create: + index: test_1 + body: + mappings: + test: + _timestamp: + enabled: 1 + - do: + cluster.health: + wait_for_status: yellow + +# blank timestamp + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _timestamp + + - is_true: _timestamp + +# milliseconds since epoch + + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + timestamp: 1372011280000 + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _timestamp + + - match: { _timestamp: 1372011280000 } + +# date format + + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + timestamp: 2013-06-23T18:14:40 + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _timestamp + + - match: { _timestamp: 1372011280000 } + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml new file mode 100644 index 0000000000000..2ec00eaf2f46c --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml @@ -0,0 +1,85 @@ +--- +"TTL": + + - do: + indices.create: + index: test_1 + body: + mappings: + test: + _ttl: + enabled: 1 + default: 10s + - do: + cluster.health: + wait_for_status: yellow + +# blank ttl + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _ttl + + - lte: { _ttl: 10000} + - gt: { _ttl: 0} + +# milliseconds + + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + ttl: 100000ms + - do: + get: + index: test_1 + type: test + id: 1 + fields: _ttl + + - lte: { _ttl: 100000} + - gt: { _ttl: 10000} + +# duration + + - do: + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + ttl: 20s + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _ttl + + - lte: { _ttl: 20000} + - gt: { _ttl: 10000} + +# with timestamp + + - do: + catch: /already_expired_exception/ + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + ttl: 20s + timestamp: 2013-06-23T18:14:40 + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml new file mode 100644 index 0000000000000..864c219c44847 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml @@ -0,0 +1,76 @@ +--- +"Timestamp": + + - do: + indices.create: + index: test_1 + body: + mappings: + test: + _timestamp: + enabled: 1 + - do: + cluster.health: + wait_for_status: yellow + +# blank timestamp + - do: + update: + index: test_1 + type: test + id: 1 + body: + doc: { foo: baz } + upsert: { foo: bar } + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _timestamp + + - is_true: _timestamp + +# milliseconds since epoch + + - do: + update: + index: test_1 + type: test + id: 1 + body: + doc: { foo: baz } + upsert: { foo: bar } + timestamp: 1372011280000 + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _timestamp + + - match: { _timestamp: 1372011280000 } + +# date format + + - do: + update: + index: test_1 + type: test + id: 1 + body: + doc: { foo: baz } + upsert: { foo: bar } + timestamp: 2013-06-23T18:14:40 + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _timestamp + + - match: { _timestamp: 1372011280000 } + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml new file mode 100644 index 0000000000000..09179ebd5ac76 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml @@ -0,0 +1,92 @@ +--- +"TTL": + + - do: + indices.create: + index: test_1 + body: + mappings: + test: + _ttl: + enabled: 1 + default: 10s + - do: + cluster.health: + wait_for_status: yellow + +# blank ttl + - do: + update: + index: test_1 + type: test + id: 1 + body: + doc: { foo: baz } + upsert: { foo: bar } + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _ttl + + - lte: { _ttl: 10000} + - gt: { _ttl: 0} + +# milliseconds + + - do: + update: + index: test_1 + type: test + id: 1 + body: + doc: { foo: baz } + upsert: { foo: bar } + ttl: 100000ms + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _ttl + + - lte: { _ttl: 100000} + - gt: { _ttl: 10000} + +# seconds + + - do: + update: + index: test_1 + type: test + id: 1 + body: + doc: { foo: baz } + upsert: { foo: bar } + detect_noop: false + ttl: 20s + + - do: + get: + index: test_1 + type: test + id: 1 + fields: _ttl + + - lte: { _ttl: 20000} + - gt: { _ttl: 10000} + +# with timestamp + + - do: + catch: /already_expired_exception/ + index: + index: test_1 + type: test + id: 1 + body: { foo: bar } + ttl: 20s + timestamp: 2013-06-23T18:14:40 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml index 0194453a99cbc..985048ad44d2d 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml @@ -12,6 +12,11 @@ mappings: test: _parent: { type: "foo" } + _timestamp: + enabled: 1 + _ttl: + enabled: 1 + default: 10s - do: cluster.health: @@ -23,13 +28,15 @@ type: test id: 1 parent: 5 - fields: [ _parent, _routing ] + fields: [ _parent, _routing, _timestamp, _ttl ] body: doc: { foo: baz } upsert: { foo: bar } - match: { get._parent: "5" } - match: { get._routing: "5" } + - is_true: get._timestamp + - is_true: get._ttl - do: get: @@ -37,6 +44,6 @@ type: test id: 1 parent: 5 - fields: [ _parent, _routing ] + fields: [ _parent, _routing, _timestamp, _ttl ] diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 8df2fd911c42b..095b1339800f8 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -383,7 +383,13 @@ public void randomIndexTemplate() throws IOException { XContentBuilder mappings = null; if (frequently() && randomDynamicTemplates()) { - mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_").endObject().endObject(); + mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_"); + if (randomBoolean()) { + mappings.startObject(TimestampFieldMapper.NAME) + .field("enabled", randomBoolean()); + mappings.endObject(); + } + mappings.endObject().endObject(); } for (String setting : randomSettingsBuilder.internalMap().keySet()) {