diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java index 33d194a7892e9..ec1024c07f54a 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java @@ -21,6 +21,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexOptions; +import org.elasticsearch.Version; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; @@ -100,6 +101,9 @@ public TTLFieldMapper build(BuilderContext context) { public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha4)) { + throw new IllegalArgumentException("[_ttl] is removed in 5.0. As a replacement, you should use time based indexes or cron a delete-by-query with a range query on a timestamp field."); + } Builder builder = new Builder(); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); @@ -165,6 +169,9 @@ private TTLFieldMapper(Settings indexSettings) { private TTLFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, long defaultTTL, Settings indexSettings) { super(NAME, fieldType, Defaults.TTL_FIELD_TYPE, indexSettings); + if (enabled.enabled && Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha4)) { + throw new IllegalArgumentException("[_ttl] is removed in 5.0. As a replacement, you should use time based indexes or cron a delete-by-query with a range query on a timestamp field."); + } this.enabledState = enabled; this.defaultTTL = defaultTTL; } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java index 09f1144db28a1..24a86b11392e8 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java @@ -22,6 +22,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.index.IndexOptions; +import org.elasticsearch.Version; import org.elasticsearch.action.TimestampParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.joda.FormatDateTimeFormatter; @@ -126,6 +127,9 @@ public TimestampFieldMapper build(BuilderContext context) { public static class TypeParser implements MetadataFieldMapper.TypeParser { @Override public MetadataFieldMapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha4)) { + throw new IllegalArgumentException("[_timestamp] is removed in 5.0. As a replacement, you can use an ingest pipeline to add a field with the current timestamp to your documents."); + } Builder builder = new Builder(parserContext.mapperService().fullName(NAME), parserContext.mapperService().getIndexSettings().getSettings()); boolean defaultSet = false; Boolean ignoreMissing = null; @@ -200,6 +204,9 @@ private TimestampFieldMapper(Settings indexSettings, MappedFieldType existing) { private TimestampFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, EnabledAttributeMapper enabledState, String defaultTimestamp, Boolean ignoreMissing, Settings indexSettings) { super(NAME, fieldType, defaultFieldType, indexSettings); + if (enabledState.enabled && Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha4)) { + throw new IllegalArgumentException("[_timestamp] is removed in 5.0. As a replacement, you can use an ingest pipeline to add a field with the current timestamp to your documents."); + } this.enabledState = enabledState; this.defaultTimestamp = defaultTimestamp; this.ignoreMissing = ignoreMissing; diff --git a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java index 2a16625d03726..5ac1bf40af6cc 100644 --- a/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java +++ b/core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java @@ -1035,13 +1035,13 @@ public void testAddAliasWithFilterNoMapping() throws Exception { } public void testAliasFilterWithNowInRangeFilterAndQuery() throws Exception { - assertAcked(prepareCreate("my-index").addMapping("my-type", "_timestamp", "enabled=true")); - assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", rangeQuery("_timestamp").from("now-1d").to("now"))); - assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter2", rangeQuery("_timestamp").from("now-1d").to("now"))); + assertAcked(prepareCreate("my-index").addMapping("my-type", "timestamp", "type=date")); + assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", rangeQuery("timestamp").from("2016-12-01").to("2016-12-31"))); + assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter2", rangeQuery("timestamp").from("2016-01-01").to("2016-12-31"))); final int numDocs = scaledRandomIntBetween(5, 52); for (int i = 1; i <= numDocs; i++) { - client().prepareIndex("my-index", "my-type").setCreate(true).setSource("{}").get(); + client().prepareIndex("my-index", "my-type").setCreate(true).setSource("timestamp", "2016-12-12").get(); if (i % 2 == 0) { refresh(); SearchResponse response = client().prepareSearch("filter1").get(); diff --git a/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java b/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java index 72db338ab809d..4773aafbf3f9f 100644 --- a/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java +++ b/core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java @@ -30,6 +30,7 @@ import org.elasticsearch.test.ESIntegTestCase.Scope; import java.io.IOException; +import java.util.Map; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @@ -111,16 +112,18 @@ public void testCustomDefaultMapping() throws Exception { internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false)); createIndex("test"); - assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("_timestamp", "enabled=true")); + assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("timestamp", "type=date")); MappingMetaData defaultMapping = client().admin().cluster().prepareState().get().getState().getMetaData().getIndices().get("test").getMappings().get("_default_"); - assertThat(defaultMapping.getSourceAsMap().get("_timestamp"), notNullValue()); + Map properties = (Map) defaultMapping.getSourceAsMap().get("properties"); + assertThat(properties.get("timestamp"), notNullValue()); - assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("_timestamp", "enabled=true")); + assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("timestamp", "type=date")); assertAcked(client().admin().indices().preparePutMapping("test").setType("type1").setSource("foo", "enabled=true")); MappingMetaData type1Mapping = client().admin().cluster().prepareState().get().getState().getMetaData().getIndices().get("test").getMappings().get("type1"); - assertThat(type1Mapping.getSourceAsMap().get("_timestamp"), notNullValue()); + properties = (Map) type1Mapping.getSourceAsMap().get("properties"); + assertThat(properties.get("timestamp"), notNullValue()); } public void testAliasFilterValidation() throws Exception { diff --git a/core/src/test/java/org/elasticsearch/get/GetActionIT.java b/core/src/test/java/org/elasticsearch/get/GetActionIT.java index 8c4b699cbff54..b73f563fbb3a0 100644 --- a/core/src/test/java/org/elasticsearch/get/GetActionIT.java +++ b/core/src/test/java/org/elasticsearch/get/GetActionIT.java @@ -20,7 +20,6 @@ package org.elasticsearch.get; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.admin.indices.alias.Alias; import org.elasticsearch.action.admin.indices.flush.FlushResponse; @@ -30,7 +29,6 @@ import org.elasticsearch.action.get.MultiGetRequest; import org.elasticsearch.action.get.MultiGetRequestBuilder; import org.elasticsearch.action.get.MultiGetResponse; -import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; @@ -40,15 +38,11 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; -import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.InternalSettingsPlugin; import java.io.IOException; -import java.util.Collection; import java.util.Collections; import java.util.HashSet; -import java.util.Map; import java.util.Set; import static java.util.Collections.singleton; @@ -57,7 +51,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; @@ -536,7 +529,7 @@ public void testMultiGetWithVersion() throws Exception { public void testGetFieldsMetaData() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") - .addMapping("my-type1", "_timestamp", "enabled=true", "_ttl", "enabled=true", "_parent", "type=parent") + .addMapping("my-type1", "_parent", "type=parent") .addAlias(new Alias("alias")) .setSettings(Settings.builder().put("index.refresh_interval", -1))); @@ -557,12 +550,6 @@ public void testGetFieldsMetaData() throws Exception { assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value")); assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true)); assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1")); - assertThat(getResponse.getField("_timestamp").isMetadataField(), equalTo(true)); - assertThat(getResponse.getField("_timestamp").getValue().toString(), equalTo("205097")); - assertThat(getResponse.getField("_ttl").isMetadataField(), equalTo(true)); - // TODO: _ttl should return the original value, but it does not work today because - // it would use now() instead of the value of _timestamp to rebase - // assertThat(getResponse.getField("_ttl").getValue().toString(), equalTo("10000000205097")); assertThat(getResponse.getField("_parent").isMetadataField(), equalTo(true)); assertThat(getResponse.getField("_parent").getValue().toString(), equalTo("parent_1")); @@ -577,12 +564,6 @@ public void testGetFieldsMetaData() throws Exception { assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value")); assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true)); assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1")); - assertThat(getResponse.getField("_timestamp").isMetadataField(), equalTo(true)); - assertThat(getResponse.getField("_timestamp").getValue().toString(), equalTo("205097")); - assertThat(getResponse.getField("_ttl").isMetadataField(), equalTo(true)); - // TODO: _ttl should return the original value, but it does not work today because - // it would use now() instead of the value of _timestamp to rebase - //assertThat(getResponse.getField("_ttl").getValue().toString(), equalTo("10000000000000")); assertThat(getResponse.getField("_parent").isMetadataField(), equalTo(true)); assertThat(getResponse.getField("_parent").getValue().toString(), equalTo("parent_1")); } @@ -779,16 +760,10 @@ public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException { " },\n" + " \"mappings\": {\n" + " \"parentdoc\": {\n" + - " \"_ttl\": {\n" + - " \"enabled\": true\n" + - " }\n" + " },\n" + " \"doc\": {\n" + " \"_parent\": {\n" + " \"type\": \"parentdoc\"\n" + - " },\n" + - " \"_ttl\": {\n" + - " \"enabled\": true\n" + " }\n" + " }\n" + " }\n" + @@ -798,7 +773,7 @@ public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException { client().prepareIndex("test", "doc").setId("1").setSource("{}").setParent("1").setTTL(TimeValue.timeValueHours(1).getMillis()).get(); - String[] fieldsList = {"_ttl", "_parent"}; + String[] fieldsList = {"_parent"}; // before refresh - document is only in translog assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1"); refresh(); @@ -814,14 +789,6 @@ public void testUngeneratedFieldsNotPartOfSourceStored() throws IOException { " \"settings\": {\n" + " \"index.translog.flush_threshold_size\": \"1pb\",\n" + " \"refresh_interval\": \"-1\"\n" + - " },\n" + - " \"mappings\": {\n" + - " \"parentdoc\": {},\n" + - " \"doc\": {\n" + - " \"_timestamp\": {\n" + - " \"enabled\": true\n" + - " }\n" + - " }\n" + " }\n" + "}"; @@ -831,7 +798,7 @@ public void testUngeneratedFieldsNotPartOfSourceStored() throws IOException { " \"text\": \"some text.\"\n" + "}\n"; client().prepareIndex("test", "doc").setId("1").setSource(doc).setRouting("1").get(); - String[] fieldsList = {"_timestamp", "_routing"}; + String[] fieldsList = {"_routing"}; // before refresh - document is only in translog assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1"); refresh(); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java index 966146655b883..165b49d3145c1 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java @@ -40,7 +40,6 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.internal.AllFieldMapper; -import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; @@ -50,9 +49,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath; @@ -430,23 +427,6 @@ public void testRootObjectMapperPropertiesDoNotCauseException() throws IOExcepti parser.parse("test", new CompressedXContent(mapping)); } - // issue https://github.com/elastic/elasticsearch/issues/5864 - public void testMetadataMappersStillWorking() throws MapperParsingException, IOException { - String mapping = "{"; - Map rootTypes = new HashMap<>(); - //just pick some example from DocumentMapperParser.rootTypeParsers - rootTypes.put(TimestampFieldMapper.NAME, "{\"enabled\" : true}"); - rootTypes.put("include_in_all", "true"); - rootTypes.put("dynamic_date_formats", "[\"yyyy-MM-dd\", \"dd-MM-yyyy\"]"); - rootTypes.put("numeric_detection", "true"); - rootTypes.put("dynamic_templates", "[]"); - for (String key : rootTypes.keySet()) { - mapping += "\"" + key+ "\"" + ":" + rootTypes.get(key) + ",\n"; - } - mapping += "\"properties\":{}}" ; - createIndex("test").mapperService().documentMapperParser().parse("test", new CompressedXContent(mapping)); - } - public void testDocValuesNotAllowed() throws IOException { String mapping = jsonBuilder().startObject().startObject("type") .startObject("_all") diff --git a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java index 14d86290db41b..78da5abb746b9 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java @@ -22,6 +22,7 @@ import org.apache.lucene.index.IndexOptions; import org.elasticsearch.Version; import org.elasticsearch.action.TimestampParsingException; +import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; @@ -31,25 +32,31 @@ import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.joda.Joda; +import org.elasticsearch.common.lucene.uid.Versions; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.internal.TimestampFieldMapper; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.test.VersionUtils; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collection; +import java.util.LinkedHashMap; +import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.elasticsearch.test.VersionUtils.randomVersion; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; @@ -58,19 +65,35 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.startsWith; /** */ public class TimestampMappingTests extends ESSingleNodeTestCase { + private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build(); + @Override protected Collection> getPlugins() { return pluginList(InternalSettingsPlugin.class); } + public void testRejectedOn5x() throws IOException { + String mapping = XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("_timestamp") + .field("enabled", true) + .endObject() + .endObject().endObject().string(); + IndexService index = createIndex("test"); + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, + () -> index.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false)); + assertThat(expected.getMessage(), startsWith("[_timestamp] is removed")); + } + public void testSimpleDisabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); BytesReference source = XContentFactory.jsonBuilder() .startObject() .field("field", "value") @@ -85,7 +108,7 @@ public void testEnabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", "yes").endObject() .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); BytesReference source = XContentFactory.jsonBuilder() .startObject() .field("field", "value") @@ -99,10 +122,7 @@ public void testEnabled() throws Exception { } public void testDefaultValues() throws Exception { - Version version; - do { - version = randomVersion(random()); - } while (version.before(Version.V_2_0_0_beta1)); + Version version = VersionUtils.randomVersionBetween(random(), Version.V_2_0_0_beta1, Version.V_5_0_0_alpha3); for (String mapping : Arrays.asList( XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(), XContentFactory.jsonBuilder().startObject().startObject("type").startObject("_timestamp").endObject().endObject().endObject().string())) { @@ -120,7 +140,7 @@ public void testThatDisablingDuringMergeIsWorking() throws Exception { String enabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true).endObject() .endObject().endObject().string(); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService(); DocumentMapper enabledMapper = mapperService.merge("type", new CompressedXContent(enabledMapping), MapperService.MergeReason.MAPPING_UPDATE, false); String disabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type") @@ -146,7 +166,7 @@ public void testTimestampMissingDefaultToEpochValue() throws Exception { .field("foo", "bar") .endObject(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); + DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData(); MappingMetaData mappingMetaData = new MappingMetaData(docMapper); @@ -172,7 +192,7 @@ public void testTimestampMissingNowDefaultValue() throws Exception { .endObject(); MetaData metaData = MetaData.builder().build(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); + DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); MappingMetaData mappingMetaData = new MappingMetaData(docMapper); @@ -195,7 +215,7 @@ public void testPathMissingWithForcedNullDefaultShouldFail() throws Exception { .endObject() .endObject().endObject(); try { - createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); + createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set to null"); } catch (TimestampParsingException e) { assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null")); @@ -212,7 +232,7 @@ public void testTimestampMissingWithForcedNullDefaultShouldFail() throws Excepti .endObject().endObject(); try { - createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); + createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set to null"); } catch (TimestampParsingException e) { assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set to null")); @@ -230,7 +250,7 @@ public void testTimestampDefaultAndIgnore() throws Exception { .endObject().endObject(); try { - createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); + createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); fail("we should reject the mapping with a TimestampParsingException: default timestamp can not be set with ignore_missing set to false"); } catch (TimestampParsingException e) { assertThat(e.getDetailedMessage(), containsString("default timestamp can not be set with ignore_missing set to false")); @@ -250,7 +270,7 @@ public void testTimestampMissingShouldNotFail() throws Exception { .endObject(); MetaData metaData = MetaData.builder().build(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); + DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping.string())); MappingMetaData mappingMetaData = new MappingMetaData(docMapper); @@ -323,7 +343,7 @@ public void testParsingNotDefaultTwiceDoesNotChangeMapping() throws Exception { .field("enabled", true) .field("default", "1970-01-01") .endObject().endObject().endObject().string(); - DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + DocumentMapperParser parser = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser(); DocumentMapper docMapper = parser.parse("type", new CompressedXContent(mapping)); docMapper = parser.parse("type", docMapper.mappingSource()); @@ -360,7 +380,7 @@ public void testIncludeInObjectNotAllowed() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true).field("default", "1970").field("format", "YYYY").endObject() .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); try { docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() @@ -375,7 +395,7 @@ public void testThatEpochCanBeIgnoredWithCustomFormat() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_timestamp").field("enabled", true).field("format", "yyyyMMddHH").endObject() .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); MetaData metaData = client().admin().cluster().prepareState().get().getState().getMetaData(); XContentBuilder doc = XContentFactory.jsonBuilder().startObject().endObject(); @@ -392,7 +412,7 @@ public void testThatIndicesAfter2_0DontSupportUnixTimestampsInAnyDateFormat() th .endObject().endObject().string(); BytesReference source = XContentFactory.jsonBuilder().startObject().field("field", "value").endObject().bytes(); // test with 2.x - DocumentMapper currentMapper = createIndex("new-index").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper currentMapper = createIndex("new-index", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); // this works with 2.x IndexRequest request = new IndexRequest("new-index", "type", "1").source(source).timestamp("1970-01-01"); @@ -407,4 +427,54 @@ public void testThatIndicesAfter2_0DontSupportUnixTimestampsInAnyDateFormat() th assertThat(e.getMessage(), containsString("failed to parse timestamp [1234567890]")); } } + + public void testSizeTimestampIndexParsing() throws IOException { + IndexService indexService = createIndex("test", BW_SETTINGS); + String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json"); + DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(mapping), true); + assertThat(documentMapper.mappingSource().string(), equalTo(mapping)); + documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true); + assertThat(documentMapper.mappingSource().string(), equalTo(mapping)); + } + + public void testDefaultApplied() throws IOException { + createIndex("test1", BW_SETTINGS); + createIndex("test2", BW_SETTINGS); + XContentBuilder defaultMapping = XContentFactory.jsonBuilder().startObject() + .startObject(MapperService.DEFAULT_MAPPING).startObject("_timestamp").field("enabled", true).endObject().endObject() + .endObject(); + client().admin().indices().preparePutMapping().setType(MapperService.DEFAULT_MAPPING).setSource(defaultMapping).get(); + XContentBuilder typeMapping = XContentFactory.jsonBuilder().startObject() + .startObject("type").startObject("_all").field("enabled", false).endObject().endObject() + .endObject(); + client().admin().indices().preparePutMapping("test1").setType("type").setSource(typeMapping).get(); + client().admin().indices().preparePutMapping("test1", "test2").setType("type").setSource(typeMapping).get(); + + GetMappingsResponse response = client().admin().indices().prepareGetMappings("test2").get(); + assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_all")); + assertFalse((Boolean) ((LinkedHashMap) response.getMappings().get("test2").get("type").getSourceAsMap().get("_all")).get("enabled")); + assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp")); + assertTrue((Boolean)((LinkedHashMap)response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp")).get("enabled")); + } + + public void testTimestampParsing() throws IOException { + IndexService indexService = createIndex("test", BW_SETTINGS); + XContentBuilder indexMapping = XContentFactory.jsonBuilder(); + boolean enabled = randomBoolean(); + indexMapping.startObject() + .startObject("type") + .startObject("_timestamp") + .field("enabled", enabled) + .endObject() + .endObject() + .endObject(); + DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(indexMapping.string()), true); + assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled)); + assertTrue(documentMapper.timestampFieldMapper().fieldType().stored()); + assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues()); + documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true); + assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled)); + assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues()); + assertTrue(documentMapper.timestampFieldMapper().fieldType().stored()); + } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java index fe05704588296..ad9e03327b274 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java @@ -20,7 +20,9 @@ package org.elasticsearch.index.mapper.ttl; import org.apache.lucene.index.IndexOptions; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; @@ -30,22 +32,48 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.index.mapper.internal.TTLFieldMapper; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; import java.io.IOException; +import java.util.Collection; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.startsWith; public class TTLMappingTests extends ESSingleNodeTestCase { + + private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build(); + + @Override + protected Collection> getPlugins() { + return pluginList(InternalSettingsPlugin.class); + } + + public void testRejectedOn5x() throws IOException { + String mapping = XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("_ttl") + .field("enabled", true) + .endObject() + .endObject().endObject().string(); + IndexService index = createIndex("test"); + IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, + () -> index.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE, false)); + assertThat(expected.getMessage(), startsWith("[_ttl] is removed")); + } + public void testSimpleDisabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); BytesReference source = XContentFactory.jsonBuilder() .startObject() .field("field", "value") @@ -60,7 +88,7 @@ public void testEnabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_ttl").field("enabled", "yes").endObject() .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); BytesReference source = XContentFactory.jsonBuilder() .startObject() .field("field", "value") @@ -75,7 +103,7 @@ public void testEnabled() throws Exception { public void testDefaultValues() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); assertThat(docMapper.TTLFieldMapper().enabled(), equalTo(TTLFieldMapper.Defaults.ENABLED_STATE.enabled)); assertThat(docMapper.TTLFieldMapper().fieldType().stored(), equalTo(TTLFieldMapper.Defaults.TTL_FIELD_TYPE.stored())); assertThat(docMapper.TTLFieldMapper().fieldType().indexOptions(), equalTo(TTLFieldMapper.Defaults.TTL_FIELD_TYPE.indexOptions())); @@ -93,7 +121,7 @@ public void testThatEnablingTTLFieldOnMergeWorks() throws Exception { .startObject("properties").field("field").startObject().field("type", "text").endObject().endObject() .endObject().endObject().string(); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService(); DocumentMapper mapperWithoutTtl = mapperService.merge("type", new CompressedXContent(mappingWithoutTtl), MapperService.MergeReason.MAPPING_UPDATE, false); DocumentMapper mapperWithTtl = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false); @@ -116,7 +144,7 @@ public void testThatChangingTTLKeepsMapperEnabled() throws Exception { .startObject("properties").field("field").startObject().field("type", "text").endObject().endObject() .endObject().endObject().string(); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService(); DocumentMapper initialMapper = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false); DocumentMapper updatedMapper = mapperService.merge("type", new CompressedXContent(updatedMapping), MapperService.MergeReason.MAPPING_UPDATE, false); @@ -127,7 +155,7 @@ public void testThatChangingTTLKeepsMapperEnabled() throws Exception { public void testThatDisablingTTLReportsConflict() throws Exception { String mappingWithTtl = getMappingWithTtlEnabled().string(); String mappingWithTtlDisabled = getMappingWithTtlDisabled().string(); - MapperService mapperService = createIndex("test").mapperService(); + MapperService mapperService = createIndex("test", BW_SETTINGS).mapperService(); DocumentMapper initialMapper = mapperService.merge("type", new CompressedXContent(mappingWithTtl), MapperService.MergeReason.MAPPING_UPDATE, false); try { @@ -143,7 +171,7 @@ public void testThatDisablingTTLReportsConflict() throws Exception { public void testThatDisablingTTLReportsConflictOnCluster() throws Exception { String mappingWithTtl = getMappingWithTtlEnabled().string(); String mappingWithTtlDisabled = getMappingWithTtlDisabled().string(); - assertAcked(client().admin().indices().prepareCreate("testindex").addMapping("type", mappingWithTtl)); + assertAcked(client().admin().indices().prepareCreate("testindex").setSettings(BW_SETTINGS).addMapping("type", mappingWithTtl)); GetMappingsResponse mappingsBeforeUpdateResponse = client().admin().indices().prepareGetMappings("testindex").addTypes("type").get(); try { client().admin().indices().preparePutMapping("testindex").setSource(mappingWithTtlDisabled).setType("type").get(); @@ -158,7 +186,7 @@ public void testThatDisablingTTLReportsConflictOnCluster() throws Exception { public void testThatEnablingTTLAfterFirstDisablingWorks() throws Exception { String mappingWithTtl = getMappingWithTtlEnabled().string(); String withTtlDisabled = getMappingWithTtlDisabled().string(); - assertAcked(client().admin().indices().prepareCreate("testindex").addMapping("type", withTtlDisabled)); + assertAcked(client().admin().indices().prepareCreate("testindex").setSettings(BW_SETTINGS).addMapping("type", withTtlDisabled)); GetMappingsResponse mappingsAfterUpdateResponse = client().admin().indices().prepareGetMappings("testindex").addTypes("type").get(); assertThat(mappingsAfterUpdateResponse.getMappings().get("testindex").get("type").sourceAsMap().get("_ttl").toString(), equalTo("{enabled=false}")); client().admin().indices().preparePutMapping("testindex").setSource(mappingWithTtl).setType("type").get(); @@ -167,20 +195,20 @@ public void testThatEnablingTTLAfterFirstDisablingWorks() throws Exception { } public void testNoConflictIfNothingSetAndDisabledLater() throws Exception { - IndexService indexService = createIndex("testindex", Settings.builder().build(), "type"); + IndexService indexService = createIndex("testindex", BW_SETTINGS, "type"); XContentBuilder mappingWithTtlDisabled = getMappingWithTtlDisabled("7d"); indexService.mapperService().merge("type", new CompressedXContent(mappingWithTtlDisabled.string()), MapperService.MergeReason.MAPPING_UPDATE, false); } public void testNoConflictIfNothingSetAndEnabledLater() throws Exception { - IndexService indexService = createIndex("testindex", Settings.builder().build(), "type"); + IndexService indexService = createIndex("testindex", BW_SETTINGS, "type"); XContentBuilder mappingWithTtlEnabled = getMappingWithTtlEnabled("7d"); indexService.mapperService().merge("type", new CompressedXContent(mappingWithTtlEnabled.string()), MapperService.MergeReason.MAPPING_UPDATE, false); } public void testMergeWithOnlyDefaultSet() throws Exception { XContentBuilder mappingWithTtlEnabled = getMappingWithTtlEnabled("7d"); - IndexService indexService = createIndex("testindex", Settings.builder().build(), "type", mappingWithTtlEnabled); + IndexService indexService = createIndex("testindex", BW_SETTINGS, "type", mappingWithTtlEnabled); XContentBuilder mappingWithOnlyDefaultSet = getMappingWithOnlyTtlDefaultSet("6m"); indexService.mapperService().merge("type", new CompressedXContent(mappingWithOnlyDefaultSet.string()), MapperService.MergeReason.MAPPING_UPDATE, false); CompressedXContent mappingAfterMerge = indexService.mapperService().documentMapper("type").mappingSource(); @@ -189,7 +217,7 @@ public void testMergeWithOnlyDefaultSet() throws Exception { public void testMergeWithOnlyDefaultSetTtlDisabled() throws Exception { XContentBuilder mappingWithTtlEnabled = getMappingWithTtlDisabled("7d"); - IndexService indexService = createIndex("testindex", Settings.builder().build(), "type", mappingWithTtlEnabled); + IndexService indexService = createIndex("testindex", BW_SETTINGS, "type", mappingWithTtlEnabled); CompressedXContent mappingAfterCreation = indexService.mapperService().documentMapper("type").mappingSource(); assertThat(mappingAfterCreation, equalTo(new CompressedXContent("{\"type\":{\"_ttl\":{\"enabled\":false},\"properties\":{\"field\":{\"type\":\"text\"}}}}"))); XContentBuilder mappingWithOnlyDefaultSet = getMappingWithOnlyTtlDefaultSet("6m"); @@ -202,7 +230,7 @@ public void testIncludeInObjectNotAllowed() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_ttl").field("enabled", true).endObject() .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + DocumentMapper docMapper = createIndex("test", BW_SETTINGS).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); try { docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() diff --git a/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java b/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java index 1d849d5093274..3041cc8d13c87 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/typelevels/ParseMappingTypeLevelTests.java @@ -31,12 +31,12 @@ public class ParseMappingTypeLevelTests extends ESSingleNodeTestCase { public void testTypeLevel() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("enabled", true).endObject() + .startObject("_all").field("enabled", false).endObject() .endObject().endObject().string(); DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); assertThat(mapper.type(), equalTo("type")); - assertThat(mapper.timestampFieldMapper().enabled(), equalTo(true)); + assertThat(mapper.allFieldMapper().enabled(), equalTo(false)); } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java index ed20cdb7e3521..afac48e2b8901 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterIT.java @@ -139,16 +139,6 @@ public void testDocValuesInvalidMappingOnUpdate() throws Exception { compareMappingOnNodes(mappingsBeforeUpdateResponse); } - // checks if the setting for timestamp and size are kept even if disabled - public void testDisabledSizeTimestampIndexDoNotLooseMappings() throws Exception { - String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json"); - prepareCreate(INDEX).addMapping(TYPE, mapping).get(); - GetMappingsResponse mappingsBeforeGreen = client().admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).get(); - ensureGreen(INDEX); - // make sure all nodes have same cluster state - compareMappingOnNodes(mappingsBeforeGreen); - } - protected void testConflict(String mapping, String mappingUpdate, String... errorMessages) throws InterruptedException { assertAcked(prepareCreate(INDEX).setSource(mapping).get()); ensureGreen(INDEX); diff --git a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java index 17cb8a84664fa..86624d938b63b 100644 --- a/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java +++ b/core/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java @@ -259,56 +259,6 @@ public void testReuseMetaFieldBackCompat() throws IOException { } } - public void testTimestampParsing() throws IOException { - IndexService indexService = createIndex("test"); - XContentBuilder indexMapping = XContentFactory.jsonBuilder(); - boolean enabled = randomBoolean(); - indexMapping.startObject() - .startObject("type") - .startObject("_timestamp") - .field("enabled", enabled) - .endObject() - .endObject() - .endObject(); - DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(indexMapping.string()), true); - assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled)); - assertTrue(documentMapper.timestampFieldMapper().fieldType().stored()); - assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues()); - documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true); - assertThat(documentMapper.timestampFieldMapper().enabled(), equalTo(enabled)); - assertTrue(documentMapper.timestampFieldMapper().fieldType().hasDocValues()); - assertTrue(documentMapper.timestampFieldMapper().fieldType().stored()); - } - - public void testSizeTimestampIndexParsing() throws IOException { - IndexService indexService = createIndex("test", Settings.builder().build()); - String mapping = copyToStringFromClasspath("/org/elasticsearch/index/mapper/update/default_mapping_with_disabled_root_types.json"); - DocumentMapper documentMapper = indexService.mapperService().parse("type", new CompressedXContent(mapping), true); - assertThat(documentMapper.mappingSource().string(), equalTo(mapping)); - documentMapper = indexService.mapperService().parse("type", new CompressedXContent(documentMapper.mappingSource().string()), true); - assertThat(documentMapper.mappingSource().string(), equalTo(mapping)); - } - - public void testDefaultApplied() throws IOException { - createIndex("test1", Settings.builder().build()); - createIndex("test2", Settings.builder().build()); - XContentBuilder defaultMapping = XContentFactory.jsonBuilder().startObject() - .startObject(MapperService.DEFAULT_MAPPING).startObject("_timestamp").field("enabled", true).endObject().endObject() - .endObject(); - client().admin().indices().preparePutMapping().setType(MapperService.DEFAULT_MAPPING).setSource(defaultMapping).get(); - XContentBuilder typeMapping = XContentFactory.jsonBuilder().startObject() - .startObject("type").startObject("_all").field("enabled", false).endObject().endObject() - .endObject(); - client().admin().indices().preparePutMapping("test1").setType("type").setSource(typeMapping).get(); - client().admin().indices().preparePutMapping("test1", "test2").setType("type").setSource(typeMapping).get(); - - GetMappingsResponse response = client().admin().indices().prepareGetMappings("test2").get(); - assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_all")); - assertFalse((Boolean) ((LinkedHashMap) response.getMappings().get("test2").get("type").getSourceAsMap().get("_all")).get("enabled")); - assertNotNull(response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp")); - assertTrue((Boolean)((LinkedHashMap)response.getMappings().get("test2").get("type").getSourceAsMap().get("_timestamp")).get("enabled")); - } - public void testRejectFieldDefinedTwice() throws IOException { String mapping1 = XContentFactory.jsonBuilder().startObject() .startObject("type1") diff --git a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java index cdb722ff9ddc8..4f7064a33bb2b 100644 --- a/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java +++ b/core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateHistogramIT.java @@ -65,7 +65,6 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.IsNull.notNullValue; @@ -107,8 +106,7 @@ private IndexRequestBuilder indexDoc(int month, int day, int value) throws Excep @Override public void setupSuiteScopeCluster() throws Exception { - assertAcked(prepareCreate("idx").addMapping("type", "_timestamp", "enabled=true")); - createIndex("idx_unmapped"); + createIndex("idx", "idx_unmapped"); // TODO: would be nice to have more random data here assertAcked(prepareCreate("empty_bucket_idx").addMapping("type", "value", "type=integer")); List builders = new ArrayList<>(); @@ -1141,13 +1139,6 @@ public void testExceptionOnNegativeInterval() { } } - public void testTimestampField() { // see #11692 - SearchResponse response = client().prepareSearch("idx").addAggregation(dateHistogram("histo").field("_timestamp").dateHistogramInterval(randomFrom(DateHistogramInterval.DAY, DateHistogramInterval.MONTH))).get(); - assertSearchResponse(response); - Histogram histo = response.getAggregations().get("histo"); - assertThat(histo.getBuckets().size(), greaterThan(0)); - } - /** * When DST ends, local time turns back one hour, so between 2am and 4am wall time we should have four buckets: * "2015-10-25T02:00:00.000+02:00", diff --git a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java index 188ae22471a4e..99183b11c58fb 100644 --- a/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java +++ b/core/src/test/java/org/elasticsearch/search/sort/FieldSortIT.java @@ -36,7 +36,6 @@ import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHitField; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.hamcrest.Matchers; @@ -70,10 +69,8 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.lessThan; -import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.nullValue; @@ -1340,16 +1337,12 @@ public void testSortOnRareField() throws IOException { } public void testSortMetaField() throws Exception { - XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_timestamp").field("enabled", true).endObject() - .endObject().endObject(); - assertAcked(prepareCreate("test") - .addMapping("type", mapping)); + createIndex("test"); ensureGreen(); final int numDocs = randomIntBetween(10, 20); IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs]; for (int i = 0; i < numDocs; ++i) { - indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i)).setTimestamp(Integer.toString(randomInt(1000))) + indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i)) .setSource(); } indexRandom(true, indexReqs); @@ -1368,37 +1361,6 @@ public void testSortMetaField() throws Exception { assertThat(previous, order == SortOrder.ASC ? lessThan(uid) : greaterThan(uid)); previous = uid; } - - /* - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(randomIntBetween(1, numDocs + 5)) - .addSort("_id", order) - .execute().actionGet(); - assertNoFailures(searchResponse); - hits = searchResponse.getHits().hits(); - previous = order == SortOrder.ASC ? new BytesRef() : UnicodeUtil.BIG_TERM; - for (int i = 0; i < hits.length; ++i) { - final BytesRef id = new BytesRef(Uid.createUid(hits[i].type(), hits[i].id())); - assertThat(previous, order == SortOrder.ASC ? lessThan(id) : greaterThan(id)); - previous = id; - }*/ - - searchResponse = client().prepareSearch() - .setQuery(matchAllQuery()) - .setSize(randomIntBetween(1, numDocs + 5)) - .addSort("_timestamp", order) - .addField("_timestamp") - .execute().actionGet(); - assertNoFailures(searchResponse); - hits = searchResponse.getHits().hits(); - Long previousTs = order == SortOrder.ASC ? 0 : Long.MAX_VALUE; - for (int i = 0; i < hits.length; ++i) { - SearchHitField timestampField = hits[i].getFields().get("_timestamp"); - Long timestamp = timestampField.getValue(); - assertThat(previousTs, order == SortOrder.ASC ? lessThanOrEqualTo(timestamp) : greaterThanOrEqualTo(timestamp)); - previousTs = timestamp; - } } /** diff --git a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java index 2fa94bc2357c1..bd24ab9f43b36 100644 --- a/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java +++ b/core/src/test/java/org/elasticsearch/timestamp/SimpleTimestampIT.java @@ -19,14 +19,20 @@ package org.elasticsearch.timestamp; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.metadata.MappingMetaData; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; +import java.util.Collection; import java.util.Locale; import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; @@ -41,8 +47,17 @@ /** */ public class SimpleTimestampIT extends ESIntegTestCase { + + private static final Settings BW_SETTINGS = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0).build(); + + @Override + protected Collection> nodePlugins() { + return pluginList(InternalSettingsPlugin.class); + } + public void testSimpleTimestamp() throws Exception { client().admin().indices().prepareCreate("test") + .setSettings(BW_SETTINGS) .addMapping("type1", jsonBuilder().startObject().startObject("type1").startObject("_timestamp").field("enabled", true).endObject().endObject().endObject()) .execute().actionGet(); client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); @@ -98,7 +113,7 @@ public void testThatUpdatingMappingShouldNotRemoveTimestampConfiguration() throw String type = "mytype"; XContentBuilder builder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", true).endObject().endObject(); - assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); + assertAcked(client().admin().indices().prepareCreate(index).setSettings(BW_SETTINGS).addMapping(type, builder)); // check mapping again assertTimestampMappingEnabled(index, type, true); @@ -117,7 +132,7 @@ public void testThatTimestampCanBeSwitchedOnAndOff() throws Exception { String type = "mytype"; XContentBuilder builder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", true).endObject().endObject(); - assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); + assertAcked(client().admin().indices().prepareCreate(index).setSettings(BW_SETTINGS).addMapping(type, builder)); // check mapping again assertTimestampMappingEnabled(index, type, true); diff --git a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java index 003c08134bc51..9e08ecde6fa8e 100644 --- a/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java +++ b/core/src/test/java/org/elasticsearch/ttl/SimpleTTLIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.ttl; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse; @@ -26,14 +27,19 @@ import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.update.UpdateRequestBuilder; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; import java.io.IOException; +import java.util.Collection; +import java.util.Collections; import java.util.Locale; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -60,6 +66,11 @@ protected int numberOfShards() { return 2; } + @Override + protected Collection> nodePlugins() { + return Collections.singleton(InternalSettingsPlugin.class); + } + @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() @@ -70,6 +81,7 @@ protected Settings nodeSettings(int nodeOrdinal) { public void testSimpleTTL() throws Exception { assertAcked(prepareCreate("test") + .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id) .addMapping("type1", XContentFactory.jsonBuilder() .startObject() .startObject("type1") @@ -209,7 +221,7 @@ public void testThatUpdatingMappingShouldNotRemoveTTLConfiguration() throws Exce String type = "mytype"; XContentBuilder builder = jsonBuilder().startObject().startObject("_ttl").field("enabled", true).endObject().endObject(); - assertAcked(client().admin().indices().prepareCreate(index).addMapping(type, builder)); + assertAcked(client().admin().indices().prepareCreate(index).setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id).addMapping(type, builder)); // check mapping again assertTTLMappingEnabled(index, type); @@ -232,6 +244,7 @@ public void testThatUpdatingMappingShouldNotRemoveTTLConfiguration() throws Exce */ public void testNoopUpdate() throws IOException { assertAcked(prepareCreate("test") + .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id) .addMapping("type1", XContentFactory.jsonBuilder() .startObject() .startObject("type1") diff --git a/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java new file mode 100644 index 0000000000000..3573089fcaa99 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/update/TimestampTTLBWIT.java @@ -0,0 +1,237 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.update; + +import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.alias.Alias; +import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.index.IndexRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.engine.DocumentMissingException; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHitField; +import org.elasticsearch.search.sort.SortOrder; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.InternalSettingsPlugin; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Map; + +import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; + +public class TimestampTTLBWIT extends ESIntegTestCase { + + @Override + protected Collection> nodePlugins() { + return Arrays.asList( + UpdateIT.FieldIncrementScriptPlugin.class, + UpdateIT.ExtractContextInSourceScriptPlugin.class, + UpdateIT.PutFieldValuesScriptPlugin.class, + InternalSettingsPlugin.class + ); + } + + public void testSort() throws Exception { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("_timestamp").field("enabled", true).endObject() + .endObject().endObject(); + assertAcked(prepareCreate("test") + .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id) + .addMapping("type", mapping)); + ensureGreen(); + final int numDocs = randomIntBetween(10, 20); + IndexRequestBuilder[] indexReqs = new IndexRequestBuilder[numDocs]; + for (int i = 0; i < numDocs; ++i) { + indexReqs[i] = client().prepareIndex("test", "type", Integer.toString(i)).setTimestamp(Integer.toString(randomInt(1000))) + .setSource(); + } + indexRandom(true, indexReqs); + + SortOrder order = randomFrom(SortOrder.values()); + + SearchResponse searchResponse = client().prepareSearch() + .setQuery(matchAllQuery()) + .setSize(randomIntBetween(1, numDocs + 5)) + .addSort("_timestamp", order) + .addField("_timestamp") + .execute().actionGet(); + assertNoFailures(searchResponse); + SearchHit[] hits = searchResponse.getHits().hits(); + Long previousTs = order == SortOrder.ASC ? 0 : Long.MAX_VALUE; + for (int i = 0; i < hits.length; ++i) { + SearchHitField timestampField = hits[i].getFields().get("_timestamp"); + Long timestamp = timestampField.getValue(); + assertThat(previousTs, order == SortOrder.ASC ? lessThanOrEqualTo(timestamp) : greaterThanOrEqualTo(timestamp)); + previousTs = timestamp; + } + } + + public void testUpdate() throws Exception { + assertAcked(prepareCreate("test").addAlias(new Alias("alias")) + .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id) + .addMapping("type1", XContentFactory.jsonBuilder() + .startObject() + .startObject("type1") + .startObject("_timestamp").field("enabled", true).endObject() + .startObject("_ttl").field("enabled", true).endObject() + .endObject() + .endObject())); + + ensureGreen(); + + try { + client().prepareUpdate(indexOrAlias(), "type1", "1") + .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet(); + fail(); + } catch (DocumentMissingException e) { + // all is well + } + + // check TTL is kept after an update without TTL + client().prepareIndex("test", "type1", "2").setSource("field", 1).setTTL(86400000L).setRefreshPolicy(IMMEDIATE).get(); + GetResponse getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet(); + long ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue(); + assertThat(ttl, greaterThan(0L)); + client().prepareUpdate(indexOrAlias(), "type1", "2") + .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet(); + getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet(); + ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue(); + assertThat(ttl, greaterThan(0L)); + + // check TTL update + client().prepareUpdate(indexOrAlias(), "type1", "2") + .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", + Collections.singletonMap("_ctx", Collections.singletonMap("_ttl", 3600000)))).execute().actionGet(); + getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet(); + ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue(); + assertThat(ttl, greaterThan(0L)); + assertThat(ttl, lessThanOrEqualTo(3600000L)); + + // check timestamp update + client().prepareIndex("test", "type1", "3").setSource("field", 1).setRefreshPolicy(IMMEDIATE).get(); + client().prepareUpdate(indexOrAlias(), "type1", "3") + .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", + Collections.singletonMap("_ctx", Collections.singletonMap("_timestamp", "2009-11-15T14:12:12")))).execute() + .actionGet(); + getResponse = client().prepareGet("test", "type1", "3").setFields("_timestamp").execute().actionGet(); + long timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue(); + assertThat(timestamp, equalTo(1258294332000L)); + } + + public void testContextVariables() throws Exception { + assertAcked(prepareCreate("test").addAlias(new Alias("alias")) + .setSettings(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_3_0.id) + .addMapping("type1", XContentFactory.jsonBuilder() + .startObject() + .startObject("type1") + .startObject("_timestamp").field("enabled", true).endObject() + .startObject("_ttl").field("enabled", true).endObject() + .endObject() + .endObject()) + .addMapping("subtype1", XContentFactory.jsonBuilder() + .startObject() + .startObject("subtype1") + .startObject("_parent").field("type", "type1").endObject() + .startObject("_timestamp").field("enabled", true).endObject() + .startObject("_ttl").field("enabled", true).endObject() + .endObject() + .endObject()) + ); + ensureGreen(); + + // Index some documents + long timestamp = System.currentTimeMillis(); + client().prepareIndex() + .setIndex("test") + .setType("type1") + .setId("parentId1") + .setTimestamp(String.valueOf(timestamp-1)) + .setSource("field1", 0, "content", "bar") + .execute().actionGet(); + + long ttl = 10000; + client().prepareIndex() + .setIndex("test") + .setType("subtype1") + .setId("id1") + .setParent("parentId1") + .setRouting("routing1") + .setTimestamp(String.valueOf(timestamp)) + .setTTL(ttl) + .setSource("field1", 1, "content", "foo") + .execute().actionGet(); + + // Update the first object and note context variables values + UpdateResponse updateResponse = client().prepareUpdate("test", "subtype1", "id1") + .setRouting("routing1") + .setScript(new Script("", ScriptService.ScriptType.INLINE, "extract_ctx", null)) + .execute().actionGet(); + + assertEquals(2, updateResponse.getVersion()); + + GetResponse getResponse = client().prepareGet("test", "subtype1", "id1").setRouting("routing1").execute().actionGet(); + Map updateContext = (Map) getResponse.getSourceAsMap().get("update_context"); + assertEquals("test", updateContext.get("_index")); + assertEquals("subtype1", updateContext.get("_type")); + assertEquals("id1", updateContext.get("_id")); + assertEquals(1, updateContext.get("_version")); + assertEquals("parentId1", updateContext.get("_parent")); + assertEquals("routing1", updateContext.get("_routing")); + assertThat(((Integer) updateContext.get("_ttl")).longValue(), allOf(greaterThanOrEqualTo(ttl-3000), lessThanOrEqualTo(ttl))); + + // Idem with the second object + updateResponse = client().prepareUpdate("test", "type1", "parentId1") + .setScript(new Script("", ScriptService.ScriptType.INLINE, "extract_ctx", null)) + .execute().actionGet(); + + assertEquals(2, updateResponse.getVersion()); + + getResponse = client().prepareGet("test", "type1", "parentId1").execute().actionGet(); + updateContext = (Map) getResponse.getSourceAsMap().get("update_context"); + assertEquals("test", updateContext.get("_index")); + assertEquals("type1", updateContext.get("_type")); + assertEquals("parentId1", updateContext.get("_id")); + assertEquals(1, updateContext.get("_version")); + assertNull(updateContext.get("_parent")); + assertNull(updateContext.get("_routing")); + assertNull(updateContext.get("_ttl")); + } + + private static String indexOrAlias() { + return randomBoolean() ? "test" : "alias"; + } +} diff --git a/core/src/test/java/org/elasticsearch/update/UpdateIT.java b/core/src/test/java/org/elasticsearch/update/UpdateIT.java index 4121543a7a079..55834c181b008 100644 --- a/core/src/test/java/org/elasticsearch/update/UpdateIT.java +++ b/core/src/test/java/org/elasticsearch/update/UpdateIT.java @@ -62,14 +62,12 @@ import java.util.concurrent.Semaphore; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.hamcrest.Matchers.notNullValue; @@ -357,21 +355,14 @@ protected Collection> nodePlugins() { FieldIncrementScriptPlugin.class, ScriptedUpsertScriptPlugin.class, ExtractContextInSourceScriptPlugin.class, - InternalSettingsPlugin.class // uses index.merge.enabled + InternalSettingsPlugin.class ); } private void createTestIndex() throws Exception { logger.info("--> creating index test"); - assertAcked(prepareCreate("test").addAlias(new Alias("alias")) - .addMapping("type1", XContentFactory.jsonBuilder() - .startObject() - .startObject("type1") - .startObject("_timestamp").field("enabled", true).endObject() - .startObject("_ttl").field("enabled", true).endObject() - .endObject() - .endObject())); + assertAcked(prepareCreate("test").addAlias(new Alias("alias"))); } public void testUpsert() throws Exception { @@ -638,34 +629,6 @@ public void testUpdate() throws Exception { assertThat(getResponse.isExists(), equalTo(false)); } - // check TTL is kept after an update without TTL - client().prepareIndex("test", "type1", "2").setSource("field", 1).setTTL(86400000L).setRefreshPolicy(IMMEDIATE).get(); - GetResponse getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet(); - long ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue(); - assertThat(ttl, greaterThan(0L)); - client().prepareUpdate(indexOrAlias(), "type1", "2") - .setScript(new Script("field", ScriptService.ScriptType.INLINE, "field_inc", null)).execute().actionGet(); - getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet(); - ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue(); - assertThat(ttl, greaterThan(0L)); - - // check TTL update - client().prepareUpdate(indexOrAlias(), "type1", "2") - .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("_ctx", Collections.singletonMap("_ttl", 3600000)))).execute().actionGet(); - getResponse = client().prepareGet("test", "type1", "2").setFields("_ttl").execute().actionGet(); - ttl = ((Number) getResponse.getField("_ttl").getValue()).longValue(); - assertThat(ttl, greaterThan(0L)); - assertThat(ttl, lessThanOrEqualTo(3600000L)); - - // check timestamp update - client().prepareIndex("test", "type1", "3").setSource("field", 1).setRefreshPolicy(IMMEDIATE).get(); - client().prepareUpdate(indexOrAlias(), "type1", "3") - .setScript(new Script("", ScriptService.ScriptType.INLINE, "put_values", Collections.singletonMap("_ctx", Collections.singletonMap("_timestamp", "2009-11-15T14:12:12")))).execute() - .actionGet(); - getResponse = client().prepareGet("test", "type1", "3").setFields("_timestamp").execute().actionGet(); - long timestamp = ((Number) getResponse.getField("_timestamp").getValue()).longValue(); - assertThat(timestamp, equalTo(1258294332000L)); - // check fields parameter client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet(); updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1") @@ -682,7 +645,7 @@ public void testUpdate() throws Exception { client().prepareIndex("test", "type1", "1").setSource("field", 1).execute().actionGet(); updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("field2", 2).endObject()).execute().actionGet(); for (int i = 0; i < 5; i++) { - getResponse = client().prepareGet("test", "type1", "1").execute().actionGet(); + GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet(); assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("1")); assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2")); } @@ -690,7 +653,7 @@ public void testUpdate() throws Exception { // change existing field updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("field", 3).endObject()).execute().actionGet(); for (int i = 0; i < 5; i++) { - getResponse = client().prepareGet("test", "type1", "1").execute().actionGet(); + GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet(); assertThat(getResponse.getSourceAsMap().get("field").toString(), equalTo("3")); assertThat(getResponse.getSourceAsMap().get("field2").toString(), equalTo("2")); } @@ -708,7 +671,7 @@ public void testUpdate() throws Exception { client().prepareIndex("test", "type1", "1").setSource("map", testMap).execute().actionGet(); updateResponse = client().prepareUpdate(indexOrAlias(), "type1", "1").setDoc(XContentFactory.jsonBuilder().startObject().field("map", testMap3).endObject()).execute().actionGet(); for (int i = 0; i < 5; i++) { - getResponse = client().prepareGet("test", "type1", "1").execute().actionGet(); + GetResponse getResponse = client().prepareGet("test", "type1", "1").execute().actionGet(); Map map1 = (Map) getResponse.getSourceAsMap().get("map"); assertThat(map1.size(), equalTo(3)); assertThat(map1.containsKey("map1"), equalTo(true)); @@ -760,16 +723,12 @@ public void testContextVariables() throws Exception { .addMapping("type1", XContentFactory.jsonBuilder() .startObject() .startObject("type1") - .startObject("_timestamp").field("enabled", true).endObject() - .startObject("_ttl").field("enabled", true).endObject() .endObject() .endObject()) .addMapping("subtype1", XContentFactory.jsonBuilder() .startObject() .startObject("subtype1") .startObject("_parent").field("type", "type1").endObject() - .startObject("_timestamp").field("enabled", true).endObject() - .startObject("_ttl").field("enabled", true).endObject() .endObject() .endObject()) ); @@ -813,7 +772,6 @@ public void testContextVariables() throws Exception { assertEquals(1, updateContext.get("_version")); assertEquals("parentId1", updateContext.get("_parent")); assertEquals("routing1", updateContext.get("_routing")); - assertThat(((Integer) updateContext.get("_ttl")).longValue(), allOf(greaterThanOrEqualTo(ttl-3000), lessThanOrEqualTo(ttl))); // Idem with the second object updateResponse = client().prepareUpdate("test", "type1", "parentId1") @@ -904,13 +862,6 @@ public void run() { public void testStressUpdateDeleteConcurrency() throws Exception { //We create an index with merging disabled so that deletes don't get merged away assertAcked(prepareCreate("test") - .addMapping("type1", XContentFactory.jsonBuilder() - .startObject() - .startObject("type1") - .startObject("_timestamp").field("enabled", true).endObject() - .startObject("_ttl").field("enabled", true).endObject() - .endObject() - .endObject()) .setSettings(Settings.builder().put(MergePolicyConfig.INDEX_MERGE_ENABLED, false))); ensureGreen(); diff --git a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java index d834f0fcf6d88..28894c5c1cc01 100644 --- a/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java +++ b/modules/lang-groovy/src/test/java/org/elasticsearch/messy/tests/SearchFieldsTests.java @@ -22,7 +22,6 @@ import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.common.Priority; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -45,7 +44,6 @@ import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; @@ -88,8 +86,6 @@ public void testStoredFields() throws Exception { client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet(); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") - // _timestamp is randomly enabled via templates but we don't want it here to test stored fields behaviour - .startObject("_timestamp").field("enabled", false).endObject() .startObject("properties") .startObject("field1").field("type", "text").field("store", true).endObject() .startObject("field2").field("type", "text").field("store", false).endObject() @@ -698,7 +694,7 @@ public void testScriptFields() throws Exception { public void testLoadMetadata() throws Exception { assertAcked(prepareCreate("test") .addMapping("parent") - .addMapping("my-type1", "_timestamp", "enabled=true", "_ttl", "enabled=true", "_parent", "type=parent")); + .addMapping("my-type1", "_parent", "type=parent")); indexRandom(true, client().prepareIndex("test", "my-type1", "1") @@ -717,12 +713,6 @@ public void testLoadMetadata() throws Exception { assertThat(fields.get("field1"), nullValue()); assertThat(fields.get("_routing").isMetadataField(), equalTo(true)); assertThat(fields.get("_routing").getValue().toString(), equalTo("1")); - assertThat(fields.get("_timestamp").isMetadataField(), equalTo(true)); - assertThat(fields.get("_timestamp").getValue().toString(), equalTo("205097")); - assertThat(fields.get("_ttl").isMetadataField(), equalTo(true)); - // TODO: _ttl should return the original value, but it does not work today because - // it would use now() instead of the value of _timestamp to rebase - // assertThat(fields.get("_ttl").getValue().toString(), equalTo("10000000205097")); assertThat(fields.get("_parent").isMetadataField(), equalTo(true)); assertThat(fields.get("_parent").getValue().toString(), equalTo("parent_1")); } diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc index 304d31978cf0f..688d8cbdc5bbc 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc @@ -255,7 +255,7 @@ Supports also regular expressions with flag X for more readability (accepts whit Compares two numeric values, eg: .... - - lt: { _ttl: 10000 } # the `_ttl` value is less than 10,000 + - lt: { foo: 10000 } # the `foo` value is less than 10,000 .... === `lte` and `gte` @@ -263,7 +263,7 @@ Compares two numeric values, eg: Compares two numeric values, eg: .... - - lte: { _ttl: 10000 } # the `_ttl` value is less than or equal to 10,000 + - lte: { foo: 10000 } # the `foo` value is less than or equal to 10,000 .... === `length` diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml deleted file mode 100644 index 83173817b6e7d..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/70_timestamp.yaml +++ /dev/null @@ -1,80 +0,0 @@ ---- -"Timestamp": - - - do: - indices.create: - index: test_1 - body: - mappings: - test: - _timestamp: - enabled: 1 - - do: - cluster.health: - wait_for_status: yellow - -# blank timestamp - - do: - create: - index: test_1 - type: test - id: 1 - body: { foo: bar } - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _timestamp - - - is_true: _timestamp - -# milliseconds since epoch - - - do: - delete: - index: test_1 - type: test - id: 1 - - do: - create: - index: test_1 - type: test - id: 1 - body: { foo: bar } - timestamp: 1372011280000 - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _timestamp - - - match: { _timestamp: 1372011280000 } - -# date format - - - do: - delete: - index: test_1 - type: test - id: 1 - - do: - create: - index: test_1 - type: test - id: 1 - body: { foo: bar } - timestamp: 2013-06-23T18:14:40 - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _timestamp - - - match: { _timestamp: 1372011280000 } - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml deleted file mode 100644 index 3f9b476366936..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/create/75_ttl.yaml +++ /dev/null @@ -1,100 +0,0 @@ ---- -"TTL": - - - do: - indices.create: - index: test_1 - body: - mappings: - test: - _ttl: - enabled: 1 - default: 10s - - do: - cluster.health: - wait_for_status: yellow - -# blank ttl - - do: - create: - index: test_1 - type: test - id: 1 - body: { foo: bar } - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _ttl - - - lte: { _ttl: 10000} - - gt: { _ttl: 0} - -# milliseconds - - - do: - delete: - index: test_1 - type: test - id: 1 - - do: - create: - index: test_1 - type: test - id: 1 - body: { foo: bar } - ttl: 100000ms - - do: - get: - index: test_1 - type: test - id: 1 - fields: _ttl - - - lte: { _ttl: 100000} - - gt: { _ttl: 10000} - -# duration - - - do: - delete: - index: test_1 - type: test - id: 1 - - do: - create: - index: test_1 - type: test - id: 1 - body: { foo: bar } - ttl: 20s - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _ttl - - - lte: { _ttl: 20000} - - gt: { _ttl: 10000} - -# with timestamp - - - do: - delete: - index: test_1 - type: test - id: 1 - - do: - catch: /already_expired_exception/ - create: - index: test_1 - type: test - id: 1 - body: { foo: bar } - ttl: 20s - timestamp: 2013-06-23T18:14:40 - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml deleted file mode 100644 index 938a358b103e6..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/70_timestamp.yaml +++ /dev/null @@ -1,70 +0,0 @@ ---- -"Timestamp": - - - do: - indices.create: - index: test_1 - body: - mappings: - test: - _timestamp: - enabled: 1 - - do: - cluster.health: - wait_for_status: yellow - -# blank timestamp - - do: - index: - index: test_1 - type: test - id: 1 - body: { foo: bar } - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _timestamp - - - is_true: _timestamp - -# milliseconds since epoch - - - do: - index: - index: test_1 - type: test - id: 1 - body: { foo: bar } - timestamp: 1372011280000 - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _timestamp - - - match: { _timestamp: 1372011280000 } - -# date format - - - do: - index: - index: test_1 - type: test - id: 1 - body: { foo: bar } - timestamp: 2013-06-23T18:14:40 - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _timestamp - - - match: { _timestamp: 1372011280000 } - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml deleted file mode 100644 index 2ec00eaf2f46c..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/75_ttl.yaml +++ /dev/null @@ -1,85 +0,0 @@ ---- -"TTL": - - - do: - indices.create: - index: test_1 - body: - mappings: - test: - _ttl: - enabled: 1 - default: 10s - - do: - cluster.health: - wait_for_status: yellow - -# blank ttl - - do: - index: - index: test_1 - type: test - id: 1 - body: { foo: bar } - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _ttl - - - lte: { _ttl: 10000} - - gt: { _ttl: 0} - -# milliseconds - - - do: - index: - index: test_1 - type: test - id: 1 - body: { foo: bar } - ttl: 100000ms - - do: - get: - index: test_1 - type: test - id: 1 - fields: _ttl - - - lte: { _ttl: 100000} - - gt: { _ttl: 10000} - -# duration - - - do: - index: - index: test_1 - type: test - id: 1 - body: { foo: bar } - ttl: 20s - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _ttl - - - lte: { _ttl: 20000} - - gt: { _ttl: 10000} - -# with timestamp - - - do: - catch: /already_expired_exception/ - index: - index: test_1 - type: test - id: 1 - body: { foo: bar } - ttl: 20s - timestamp: 2013-06-23T18:14:40 - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml deleted file mode 100644 index 864c219c44847..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/70_timestamp.yaml +++ /dev/null @@ -1,76 +0,0 @@ ---- -"Timestamp": - - - do: - indices.create: - index: test_1 - body: - mappings: - test: - _timestamp: - enabled: 1 - - do: - cluster.health: - wait_for_status: yellow - -# blank timestamp - - do: - update: - index: test_1 - type: test - id: 1 - body: - doc: { foo: baz } - upsert: { foo: bar } - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _timestamp - - - is_true: _timestamp - -# milliseconds since epoch - - - do: - update: - index: test_1 - type: test - id: 1 - body: - doc: { foo: baz } - upsert: { foo: bar } - timestamp: 1372011280000 - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _timestamp - - - match: { _timestamp: 1372011280000 } - -# date format - - - do: - update: - index: test_1 - type: test - id: 1 - body: - doc: { foo: baz } - upsert: { foo: bar } - timestamp: 2013-06-23T18:14:40 - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _timestamp - - - match: { _timestamp: 1372011280000 } - diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml deleted file mode 100644 index 09179ebd5ac76..0000000000000 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/75_ttl.yaml +++ /dev/null @@ -1,92 +0,0 @@ ---- -"TTL": - - - do: - indices.create: - index: test_1 - body: - mappings: - test: - _ttl: - enabled: 1 - default: 10s - - do: - cluster.health: - wait_for_status: yellow - -# blank ttl - - do: - update: - index: test_1 - type: test - id: 1 - body: - doc: { foo: baz } - upsert: { foo: bar } - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _ttl - - - lte: { _ttl: 10000} - - gt: { _ttl: 0} - -# milliseconds - - - do: - update: - index: test_1 - type: test - id: 1 - body: - doc: { foo: baz } - upsert: { foo: bar } - ttl: 100000ms - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _ttl - - - lte: { _ttl: 100000} - - gt: { _ttl: 10000} - -# seconds - - - do: - update: - index: test_1 - type: test - id: 1 - body: - doc: { foo: baz } - upsert: { foo: bar } - detect_noop: false - ttl: 20s - - - do: - get: - index: test_1 - type: test - id: 1 - fields: _ttl - - - lte: { _ttl: 20000} - - gt: { _ttl: 10000} - -# with timestamp - - - do: - catch: /already_expired_exception/ - index: - index: test_1 - type: test - id: 1 - body: { foo: bar } - ttl: 20s - timestamp: 2013-06-23T18:14:40 diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml b/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml index 985048ad44d2d..0194453a99cbc 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/update/85_fields_meta.yaml @@ -12,11 +12,6 @@ mappings: test: _parent: { type: "foo" } - _timestamp: - enabled: 1 - _ttl: - enabled: 1 - default: 10s - do: cluster.health: @@ -28,15 +23,13 @@ type: test id: 1 parent: 5 - fields: [ _parent, _routing, _timestamp, _ttl ] + fields: [ _parent, _routing ] body: doc: { foo: baz } upsert: { foo: bar } - match: { get._parent: "5" } - match: { get._routing: "5" } - - is_true: get._timestamp - - is_true: get._ttl - do: get: @@ -44,6 +37,6 @@ type: test id: 1 parent: 5 - fields: [ _parent, _routing, _timestamp, _ttl ] + fields: [ _parent, _routing ] diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index 095b1339800f8..8df2fd911c42b 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -383,13 +383,7 @@ public void randomIndexTemplate() throws IOException { XContentBuilder mappings = null; if (frequently() && randomDynamicTemplates()) { - mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_"); - if (randomBoolean()) { - mappings.startObject(TimestampFieldMapper.NAME) - .field("enabled", randomBoolean()); - mappings.endObject(); - } - mappings.endObject().endObject(); + mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_").endObject().endObject(); } for (String setting : randomSettingsBuilder.internalMap().keySet()) {