From 9d708e20a00b7661c893b99b73aa736cbccb7ee5 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 26 Feb 2015 14:02:24 -0800 Subject: [PATCH] Mappings: Lock down _size field This also changes the stored setting for _size to true (for indexes created in 2.x). see #8143 closes #9913 --- docs/reference/migration/migrate_2_0.asciidoc | 1 + .../mapper/internal/SizeFieldMapper.java | 8 +++--- .../org/elasticsearch/get/GetActionTests.java | 5 ++-- .../index/mapper/size/SizeMappingTests.java | 26 +++++++++---------- .../mapper/update/UpdateMappingTests.java | 1 - .../percolator/PercolatorTests.java | 2 +- .../search/fields/SearchFieldsTests.java | 3 ++- 7 files changed, 24 insertions(+), 22 deletions(-) diff --git a/docs/reference/migration/migrate_2_0.asciidoc b/docs/reference/migration/migrate_2_0.asciidoc index 132b6638ca051..aec7699e3f914 100644 --- a/docs/reference/migration/migrate_2_0.asciidoc +++ b/docs/reference/migration/migrate_2_0.asciidoc @@ -257,6 +257,7 @@ to provide special features. They now have limited configuration options. * `_routing` configuration is limited to requiring the field. * `_boost` has been removed. * `_field_names` configuration is limited to disabling the field. +* `_size` configuration is limited to enabling the field. === Codecs diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java index acf5015eea60f..5bb2173d6aa3d 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/SizeFieldMapper.java @@ -21,6 +21,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; @@ -55,6 +56,7 @@ public static class Defaults extends IntegerFieldMapper.Defaults { public static final FieldType SIZE_FIELD_TYPE = new FieldType(IntegerFieldMapper.Defaults.FIELD_TYPE); static { + SIZE_FIELD_TYPE.setStored(true); SIZE_FIELD_TYPE.freeze(); } } @@ -90,7 +92,7 @@ public Mapper.Builder parse(String name, Map node, ParserContext if (fieldName.equals("enabled")) { builder.enabled(nodeBooleanValue(fieldNode) ? EnabledAttributeMapper.ENABLED : EnabledAttributeMapper.DISABLED); iterator.remove(); - } else if (fieldName.equals("store")) { + } else if (fieldName.equals("store") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) { builder.store(parseStore(fieldName, fieldNode.toString())); iterator.remove(); } @@ -162,14 +164,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws boolean includeDefaults = params.paramAsBoolean("include_defaults", false); // all are defaults, no need to write it at all - if (!includeDefaults && enabledState == Defaults.ENABLED_STATE && fieldType().stored() == Defaults.SIZE_FIELD_TYPE.stored()) { + if (!includeDefaults && enabledState == Defaults.ENABLED_STATE && (writePre2xSettings == false || fieldType().stored() == false)) { return builder; } builder.startObject(contentType()); if (includeDefaults || enabledState != Defaults.ENABLED_STATE) { builder.field("enabled", enabledState.enabled); } - if (includeDefaults || fieldType().stored() != Defaults.SIZE_FIELD_TYPE.stored()) { + if (writePre2xSettings && (includeDefaults || fieldType().stored() == true)) { builder.field("store", fieldType().stored()); } builder.endObject(); diff --git a/src/test/java/org/elasticsearch/get/GetActionTests.java b/src/test/java/org/elasticsearch/get/GetActionTests.java index 8741bfc1690ce..f9d566a024046 100644 --- a/src/test/java/org/elasticsearch/get/GetActionTests.java +++ b/src/test/java/org/elasticsearch/get/GetActionTests.java @@ -1067,8 +1067,8 @@ void indexSingleDocumentWithUngeneratedFieldsThatArePartOf_source(boolean stored @Test public void testUngeneratedFieldsNotPartOfSourceUnstored() throws IOException { indexSingleDocumentWithUngeneratedFieldsThatAreNeverPartOf_source(false, randomBoolean()); - String[] fieldsList = {"_timestamp", "_size"}; - String[] alwaysStoredFieldsList = {"_routing"}; + String[] fieldsList = {"_timestamp"}; + String[] alwaysStoredFieldsList = {"_routing", "_size"}; // before refresh - document is only in translog assertGetFieldsAlwaysNull(indexOrAlias(), "doc", "1", fieldsList, "1"); assertGetFieldsAlwaysWorks(indexOrAlias(), "doc", "1", alwaysStoredFieldsList, "1"); @@ -1111,7 +1111,6 @@ void indexSingleDocumentWithUngeneratedFieldsThatAreNeverPartOf_source(boolean s " \"enabled\": true\n" + " },\n" + " \"_size\": {\n" + - " \"store\": \"" + storedString + "\",\n" + " \"enabled\": true\n" + " }\n" + " }\n" + diff --git a/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java b/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java index 7e69ee38c6024..07af4a4ef45cf 100644 --- a/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/size/SizeMappingTests.java @@ -19,7 +19,11 @@ package org.elasticsearch.index.mapper.size; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentMapperParser; @@ -31,8 +35,7 @@ import static org.hamcrest.Matchers.*; public class SizeMappingTests extends ElasticsearchSingleNodeTest { - - @Test + public void testSizeEnabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_size").field("enabled", true).endObject() @@ -46,16 +49,16 @@ public void testSizeEnabled() throws Exception { .bytes(); ParsedDocument doc = docMapper.parse(SourceToParse.source(source).type("type").id("1")); - assertThat(doc.rootDoc().getField("_size").fieldType().stored(), equalTo(false)); + assertThat(doc.rootDoc().getField("_size").fieldType().stored(), equalTo(true)); assertThat(doc.rootDoc().getField("_size").tokenStream(docMapper.mappers().indexAnalyzer(), null), notNullValue()); } - - @Test - public void testSizeEnabledAndStored() throws Exception { + + public void testSizeEnabledAndStoredBackcompat() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_size").field("enabled", true).field("store", "yes").endObject() .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Settings indexSettings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); + DocumentMapper docMapper = createIndex("test", indexSettings).mapperService().documentMapperParser().parse(mapping); BytesReference source = XContentFactory.jsonBuilder() .startObject() @@ -67,8 +70,7 @@ public void testSizeEnabledAndStored() throws Exception { assertThat(doc.rootDoc().getField("_size").fieldType().stored(), equalTo(true)); assertThat(doc.rootDoc().getField("_size").tokenStream(docMapper.mappers().indexAnalyzer(), null), notNullValue()); } - - @Test + public void testSizeDisabled() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_size").field("enabled", false).endObject() @@ -84,8 +86,7 @@ public void testSizeDisabled() throws Exception { assertThat(doc.rootDoc().getField("_size"), nullValue()); } - - @Test + public void testSizeNotSet() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .endObject().endObject().string(); @@ -100,8 +101,7 @@ public void testSizeNotSet() throws Exception { assertThat(doc.rootDoc().getField("_size"), nullValue()); } - - @Test + public void testThatDisablingWorksWhenMerging() throws Exception { String enabledMapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_size").field("enabled", true).endObject() diff --git a/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java b/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java index 1182d42c362d2..a59050667a063 100644 --- a/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingTests.java @@ -166,7 +166,6 @@ public void testSizeParsing() throws IOException { .startObject("type") .startObject("_size") .field("enabled", enabled) - .field("store", true) .endObject() .endObject() .endObject(); diff --git a/src/test/java/org/elasticsearch/percolator/PercolatorTests.java b/src/test/java/org/elasticsearch/percolator/PercolatorTests.java index deb2044773d0e..c136df36055aa 100644 --- a/src/test/java/org/elasticsearch/percolator/PercolatorTests.java +++ b/src/test/java/org/elasticsearch/percolator/PercolatorTests.java @@ -560,7 +560,7 @@ public void dynamicAddingRemovingQueries() throws Exception { @Test public void percolateWithSizeField() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") - .startObject("_size").field("enabled", true).field("store", "yes").endObject() + .startObject("_size").field("enabled", true).endObject() .startObject("properties").startObject("field1").field("type", "string").endObject().endObject() .endObject().endObject().string(); diff --git a/src/test/java/org/elasticsearch/search/fields/SearchFieldsTests.java b/src/test/java/org/elasticsearch/search/fields/SearchFieldsTests.java index 3e78d4029317e..b69ddf0b6e700 100644 --- a/src/test/java/org/elasticsearch/search/fields/SearchFieldsTests.java +++ b/src/test/java/org/elasticsearch/search/fields/SearchFieldsTests.java @@ -72,8 +72,9 @@ public void testStoredFields() throws Exception { client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForYellowStatus().execute().actionGet(); String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") - // _timestamp is randomly enabled via templates but we don't want it here to test stored fields behaviour + // _timestamp and _size are randomly enabled via templates but we don't want it here to test stored fields behaviour .startObject("_timestamp").field("enabled", false).endObject() + .startObject("_size").field("enabled", false).endObject() .startObject("properties") .startObject("field1").field("type", "string").field("store", "yes").endObject() .startObject("field2").field("type", "string").field("store", "no").endObject()