Skip to content

Commit

Permalink
Remove _timestamp and _ttl on 5.x indices. elastic#18980
Browse files Browse the repository at this point in the history 
This removes the ability to use `_timestamp` and `_ttl` on indices created on
or after 5.0.

Closes elastic#18280
  • Loading branch information
@jpountz
jpountz committed Jun 21, 2016
1 parent 2f2ea0c commit 969e953
Show file tree
Hide file tree
Showing 27 changed files with 440 additions and 795 deletions.
7 changes: 7 additions & 0 deletions core/src/main/java/org/elasticsearch/index/mapper/internal/TTLFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexOptions;
import org.elasticsearch.Version;
import org.elasticsearch.common.lucene.Lucene;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
Expand Down Expand Up @@ -100,6 +101,9 @@ public TTLFieldMapper build(BuilderContext context) {
public static class TypeParser implements MetadataFieldMapper.TypeParser {
@Override
public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha4)) {
throw new IllegalArgumentException("[_ttl] is removed in 5.0. As a replacement, you should use time based indexes or cron a delete-by-query with a range query on a timestamp field.");
}
Builder builder = new Builder();
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
Map.Entry<String, Object> entry = iterator.next();
Expand Down Expand Up @@ -165,6 +169,9 @@ private TTLFieldMapper(Settings indexSettings) {
private TTLFieldMapper(MappedFieldType fieldType, EnabledAttributeMapper enabled, long defaultTTL,
Settings indexSettings) {
super(NAME, fieldType, Defaults.TTL_FIELD_TYPE, indexSettings);
if (enabled.enabled && Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha4)) {
throw new IllegalArgumentException("[_ttl] is removed in 5.0. As a replacement, you should use time based indexes or cron a delete-by-query with a range query on a timestamp field.");
}
this.enabledState = enabled;
this.defaultTTL = defaultTTL;
}
Expand Down
7 changes: 7 additions & 0 deletions core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.IndexOptions;
import org.elasticsearch.Version;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.joda.FormatDateTimeFormatter;
Expand Down Expand Up @@ -126,6 +127,9 @@ public TimestampFieldMapper build(BuilderContext context) {
public static class TypeParser implements MetadataFieldMapper.TypeParser {
@Override
public MetadataFieldMapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
if (parserContext.indexVersionCreated().onOrAfter(Version.V_5_0_0_alpha4)) {
throw new IllegalArgumentException("[_timestamp] is removed in 5.0. As a replacement, you can use an ingest pipeline to add a field with the current timestamp to your documents.");
}
Builder builder = new Builder(parserContext.mapperService().fullName(NAME), parserContext.mapperService().getIndexSettings().getSettings());
boolean defaultSet = false;
Boolean ignoreMissing = null;
Expand Down Expand Up @@ -200,6 +204,9 @@ private TimestampFieldMapper(Settings indexSettings, MappedFieldType existing) {
private TimestampFieldMapper(MappedFieldType fieldType, MappedFieldType defaultFieldType, EnabledAttributeMapper enabledState,
String defaultTimestamp, Boolean ignoreMissing, Settings indexSettings) {
super(NAME, fieldType, defaultFieldType, indexSettings);
if (enabledState.enabled && Version.indexCreated(indexSettings).onOrAfter(Version.V_5_0_0_alpha4)) {
throw new IllegalArgumentException("[_timestamp] is removed in 5.0. As a replacement, you can use an ingest pipeline to add a field with the current timestamp to your documents.");
}
this.enabledState = enabledState;
this.defaultTimestamp = defaultTimestamp;
this.ignoreMissing = ignoreMissing;
Expand Down
8 changes: 4 additions & 4 deletions core/src/test/java/org/elasticsearch/aliases/IndexAliasesIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1035,13 +1035,13 @@ public void testAddAliasWithFilterNoMapping() throws Exception {
}

public void testAliasFilterWithNowInRangeFilterAndQuery() throws Exception {
assertAcked(prepareCreate("my-index").addMapping("my-type", "_timestamp", "enabled=true"));
assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", rangeQuery("_timestamp").from("now-1d").to("now")));
assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter2", rangeQuery("_timestamp").from("now-1d").to("now")));
assertAcked(prepareCreate("my-index").addMapping("my-type", "timestamp", "type=date"));
assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter1", rangeQuery("timestamp").from("2016-12-01").to("2016-12-31")));
assertAcked(admin().indices().prepareAliases().addAlias("my-index", "filter2", rangeQuery("timestamp").from("2016-01-01").to("2016-12-31")));

final int numDocs = scaledRandomIntBetween(5, 52);
for (int i = 1; i <= numDocs; i++) {
client().prepareIndex("my-index", "my-type").setCreate(true).setSource("{}").get();
client().prepareIndex("my-index", "my-type").setCreate(true).setSource("timestamp", "2016-12-12").get();
if (i % 2 == 0) {
refresh();
SearchResponse response = client().prepareSearch("filter1").get();
Expand Down
11 changes: 7 additions & 4 deletions core/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.elasticsearch.test.ESIntegTestCase.Scope;

import java.io.IOException;
import java.util.Map;

import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.hamcrest.Matchers.equalTo;
Expand Down Expand Up @@ -111,16 +112,18 @@ public void testCustomDefaultMapping() throws Exception {
internalCluster().startNode(settingsBuilder().put(Node.NODE_DATA_SETTING.getKey(), true).put(Node.NODE_MASTER_SETTING.getKey(), false));

createIndex("test");
assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("_timestamp", "enabled=true"));
assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("timestamp", "type=date"));

MappingMetaData defaultMapping = client().admin().cluster().prepareState().get().getState().getMetaData().getIndices().get("test").getMappings().get("_default_");
assertThat(defaultMapping.getSourceAsMap().get("_timestamp"), notNullValue());
Map<?,?> properties = (Map<?, ?>) defaultMapping.getSourceAsMap().get("properties");
assertThat(properties.get("timestamp"), notNullValue());

assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("_timestamp", "enabled=true"));
assertAcked(client().admin().indices().preparePutMapping("test").setType("_default_").setSource("timestamp", "type=date"));

assertAcked(client().admin().indices().preparePutMapping("test").setType("type1").setSource("foo", "enabled=true"));
MappingMetaData type1Mapping = client().admin().cluster().prepareState().get().getState().getMetaData().getIndices().get("test").getMappings().get("type1");
assertThat(type1Mapping.getSourceAsMap().get("_timestamp"), notNullValue());
properties = (Map<?, ?>) type1Mapping.getSourceAsMap().get("properties");
assertThat(properties.get("timestamp"), notNullValue());
}

public void testAliasFilterValidation() throws Exception {
Expand Down
39 changes: 3 additions & 36 deletions core/src/test/java/org/elasticsearch/get/GetActionIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
package org.elasticsearch.get;

import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.flush.FlushResponse;
Expand All @@ -30,7 +29,6 @@
import org.elasticsearch.action.get.MultiGetRequest;
import org.elasticsearch.action.get.MultiGetRequestBuilder;
import org.elasticsearch.action.get.MultiGetResponse;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
Expand All @@ -40,15 +38,11 @@
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.engine.VersionConflictEngineException;
import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.InternalSettingsPlugin;

import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import static java.util.Collections.singleton;
Expand All @@ -57,7 +51,6 @@
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasKey;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
Expand Down Expand Up @@ -536,7 +529,7 @@ public void testMultiGetWithVersion() throws Exception {
public void testGetFieldsMetaData() throws Exception {
assertAcked(prepareCreate("test")
.addMapping("parent")
.addMapping("my-type1", "_timestamp", "enabled=true", "_ttl", "enabled=true", "_parent", "type=parent")
.addMapping("my-type1", "_parent", "type=parent")
.addAlias(new Alias("alias"))
.setSettings(Settings.builder().put("index.refresh_interval", -1)));

Expand All @@ -557,12 +550,6 @@ public void testGetFieldsMetaData() throws Exception {
assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value"));
assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true));
assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1"));
assertThat(getResponse.getField("_timestamp").isMetadataField(), equalTo(true));
assertThat(getResponse.getField("_timestamp").getValue().toString(), equalTo("205097"));
assertThat(getResponse.getField("_ttl").isMetadataField(), equalTo(true));
// TODO: _ttl should return the original value, but it does not work today because
// it would use now() instead of the value of _timestamp to rebase
// assertThat(getResponse.getField("_ttl").getValue().toString(), equalTo("10000000205097"));
assertThat(getResponse.getField("_parent").isMetadataField(), equalTo(true));
assertThat(getResponse.getField("_parent").getValue().toString(), equalTo("parent_1"));

Expand All @@ -577,12 +564,6 @@ public void testGetFieldsMetaData() throws Exception {
assertThat(getResponse.getField("field1").getValue().toString(), equalTo("value"));
assertThat(getResponse.getField("_routing").isMetadataField(), equalTo(true));
assertThat(getResponse.getField("_routing").getValue().toString(), equalTo("1"));
assertThat(getResponse.getField("_timestamp").isMetadataField(), equalTo(true));
assertThat(getResponse.getField("_timestamp").getValue().toString(), equalTo("205097"));
assertThat(getResponse.getField("_ttl").isMetadataField(), equalTo(true));
// TODO: _ttl should return the original value, but it does not work today because
// it would use now() instead of the value of _timestamp to rebase
//assertThat(getResponse.getField("_ttl").getValue().toString(), equalTo("10000000000000"));
assertThat(getResponse.getField("_parent").isMetadataField(), equalTo(true));
assertThat(getResponse.getField("_parent").getValue().toString(), equalTo("parent_1"));
}
Expand Down Expand Up @@ -779,16 +760,10 @@ public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException {
" },\n" +
" \"mappings\": {\n" +
" \"parentdoc\": {\n" +
" \"_ttl\": {\n" +
" \"enabled\": true\n" +
" }\n" +
" },\n" +
" \"doc\": {\n" +
" \"_parent\": {\n" +
" \"type\": \"parentdoc\"\n" +
" },\n" +
" \"_ttl\": {\n" +
" \"enabled\": true\n" +
" }\n" +
" }\n" +
" }\n" +
Expand All @@ -798,7 +773,7 @@ public void testUngeneratedFieldsThatAreAlwaysStored() throws IOException {

client().prepareIndex("test", "doc").setId("1").setSource("{}").setParent("1").setTTL(TimeValue.timeValueHours(1).getMillis()).get();

String[] fieldsList = {"_ttl", "_parent"};
String[] fieldsList = {"_parent"};
// before refresh - document is only in translog
assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1");
refresh();
Expand All @@ -814,14 +789,6 @@ public void testUngeneratedFieldsNotPartOfSourceStored() throws IOException {
" \"settings\": {\n" +
" \"index.translog.flush_threshold_size\": \"1pb\",\n" +
" \"refresh_interval\": \"-1\"\n" +
" },\n" +
" \"mappings\": {\n" +
" \"parentdoc\": {},\n" +
" \"doc\": {\n" +
" \"_timestamp\": {\n" +
" \"enabled\": true\n" +
" }\n" +
" }\n" +
" }\n" +
"}";

Expand All @@ -831,7 +798,7 @@ public void testUngeneratedFieldsNotPartOfSourceStored() throws IOException {
" \"text\": \"some text.\"\n" +
"}\n";
client().prepareIndex("test", "doc").setId("1").setSource(doc).setRouting("1").get();
String[] fieldsList = {"_timestamp", "_routing"};
String[] fieldsList = {"_routing"};
// before refresh - document is only in translog
assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", fieldsList, "1");
refresh();
Expand Down
20 changes: 0 additions & 20 deletions core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParseContext.Document;
import org.elasticsearch.index.mapper.internal.AllFieldMapper;
import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.test.InternalSettingsPlugin;
Expand All @@ -50,9 +49,7 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.StreamsUtils.copyToBytesFromClasspath;
Expand Down Expand Up @@ -430,23 +427,6 @@ public void testRootObjectMapperPropertiesDoNotCauseException() throws IOExcepti
parser.parse("test", new CompressedXContent(mapping));
}

// issue https://github.com/elastic/elasticsearch/issues/5864
public void testMetadataMappersStillWorking() throws MapperParsingException, IOException {
String mapping = "{";
Map<String, String> rootTypes = new HashMap<>();
//just pick some example from DocumentMapperParser.rootTypeParsers
rootTypes.put(TimestampFieldMapper.NAME, "{\"enabled\" : true}");
rootTypes.put("include_in_all", "true");
rootTypes.put("dynamic_date_formats", "[\"yyyy-MM-dd\", \"dd-MM-yyyy\"]");
rootTypes.put("numeric_detection", "true");
rootTypes.put("dynamic_templates", "[]");
for (String key : rootTypes.keySet()) {
mapping += "\"" + key+ "\"" + ":" + rootTypes.get(key) + ",\n";
}
mapping += "\"properties\":{}}" ;
createIndex("test").mapperService().documentMapperParser().parse("test", new CompressedXContent(mapping));
}

public void testDocValuesNotAllowed() throws IOException {
String mapping = jsonBuilder().startObject().startObject("type")
.startObject("_all")
Expand Down
Loading

0 comments on commit 969e953

Please sign in to comment.