diff --git a/docs/reference/migration/migrate_2_0.asciidoc b/docs/reference/migration/migrate_2_0.asciidoc index 6f1868664c749..d540b72b7e40a 100644 --- a/docs/reference/migration/migrate_2_0.asciidoc +++ b/docs/reference/migration/migrate_2_0.asciidoc @@ -237,6 +237,12 @@ curl -XGET 'localhost:9200/index/type/_search' } --------------- +==== Meta fields have limited confiugration +Meta fields (those beginning with underscore) are fields used by elasticsearch +to provide special features. They now have limited configuration options. + +* `_id` configuration can no longer be changed. If you need to sort, use `_uid` instead. + === Codecs It is no longer possible to specify per-field postings and doc values formats diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 885544cc2ee2e..02bb00d95359e 100755 --- a/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -196,7 +196,6 @@ public MapperService(Index index, @IndexSettings Settings indexSettings, Environ defaultPercolatorMappingSource = "{\n" + //" \"" + PercolatorService.TYPE_NAME + "\":{\n" + " \"" + "_default_" + "\":{\n" + - " \"_id\" : {\"index\": \"not_analyzed\"}," + " \"properties\" : {\n" + " \"query\" : {\n" + " \"type\" : \"object\",\n" + diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java index 06eb4419a7b72..32e25a23cafd2 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java @@ -121,13 +121,16 @@ public IdFieldMapper build(BuilderContext context) { public static class TypeParser implements Mapper.TypeParser { @Override public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + if (parserContext.indexVersionCreated().onOrAfter(Version.V_2_0_0)) { + throw new MapperParsingException(NAME + " is not configurable"); + } IdFieldMapper.Builder builder = id(); parseField(builder, builder.name, node, parserContext); for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry entry = iterator.next(); String fieldName = Strings.toUnderscoreCase(entry.getKey()); Object fieldNode = entry.getValue(); - if (fieldName.equals("path") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) { + if (fieldName.equals("path")) { builder.path(fieldNode.toString()); iterator.remove(); } @@ -151,7 +154,8 @@ protected IdFieldMapper(String name, String indexName, float boost, FieldType fi private static FieldType idFieldType(Settings indexSettings) { FieldType fieldType = new FieldType(Defaults.FIELD_TYPE); - if (indexSettings.getAsBoolean("index.mapping._id.indexed", true) == false) { + boolean pre2x = Version.indexCreated(indexSettings).before(Version.V_2_0_0); + if (pre2x && indexSettings.getAsBoolean("index.mapping._id.indexed", true) == false) { fieldType.setTokenized(false); } return fieldType; @@ -345,6 +349,9 @@ protected String contentType() { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + if (writePre2xSettings == false) { + return builder; + } boolean includeDefaults = params.paramAsBoolean("include_defaults", false); // if all are defaults, no sense to write it at all @@ -361,7 +368,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (includeDefaults || fieldType.indexOptions() != Defaults.FIELD_TYPE.indexOptions()) { builder.field("index", indexTokenizeOptionToString(fieldType.indexOptions() != IndexOptions.NONE, fieldType.tokenized())); } - if (writePre2xSettings && (includeDefaults || path != Defaults.PATH)) { + if (includeDefaults || path != Defaults.PATH) { builder.field("path", path); } diff --git a/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java b/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java index 5f0065dfd5efe..200a39be20738 100644 --- a/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java +++ b/src/main/java/org/elasticsearch/index/percolator/QueriesLoaderCollector.java @@ -32,7 +32,8 @@ import org.elasticsearch.index.fieldvisitor.JustSourceFieldsVisitor; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.index.mapper.internal.IdFieldMapper; +import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.index.mapper.internal.UidFieldMapper; import java.io.IOException; import java.util.Map; @@ -44,17 +45,17 @@ final class QueriesLoaderCollector extends SimpleCollector { private final Map queries = Maps.newHashMap(); private final JustSourceFieldsVisitor fieldsVisitor = new JustSourceFieldsVisitor(); private final PercolatorQueriesRegistry percolator; - private final IndexFieldData idFieldData; + private final IndexFieldData uidFieldData; private final ESLogger logger; - private SortedBinaryDocValues idValues; + private SortedBinaryDocValues uidValues; private LeafReader reader; QueriesLoaderCollector(PercolatorQueriesRegistry percolator, ESLogger logger, MapperService mapperService, IndexFieldDataService indexFieldDataService) { this.percolator = percolator; this.logger = logger; - final FieldMapper idMapper = mapperService.smartNameFieldMapper(IdFieldMapper.NAME); - this.idFieldData = indexFieldDataService.getForField(idMapper); + final FieldMapper uidMapper = mapperService.smartNameFieldMapper(UidFieldMapper.NAME); + this.uidFieldData = indexFieldDataService.getForField(uidMapper); } public Map queries() { @@ -65,10 +66,11 @@ public Map queries() { public void collect(int doc) throws IOException { // the _source is the query - idValues.setDocument(doc); - if (idValues.count() > 0) { - assert idValues.count() == 1; - BytesRef id = idValues.valueAt(0); + uidValues.setDocument(doc); + if (uidValues.count() > 0) { + assert uidValues.count() == 1; + final BytesRef uid = uidValues.valueAt(0); + final BytesRef id = Uid.splitUidIntoTypeAndId(uid)[1]; fieldsVisitor.reset(); reader.document(doc, fieldsVisitor); @@ -90,7 +92,7 @@ public void collect(int doc) throws IOException { @Override protected void doSetNextReader(LeafReaderContext context) throws IOException { reader = context.reader(); - idValues = idFieldData.load(context).getBytesValues(); + uidValues = uidFieldData.load(context).getBytesValues(); } @Override diff --git a/src/main/java/org/elasticsearch/percolator/PercolatorService.java b/src/main/java/org/elasticsearch/percolator/PercolatorService.java index fdf8dbfa8f942..ac29b5d754fa7 100644 --- a/src/main/java/org/elasticsearch/percolator/PercolatorService.java +++ b/src/main/java/org/elasticsearch/percolator/PercolatorService.java @@ -19,7 +19,6 @@ package org.elasticsearch.percolator; import com.carrotsearch.hppc.ByteObjectOpenHashMap; - import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.memory.ExtendedMemoryIndex; @@ -68,7 +67,9 @@ import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.Uid; import org.elasticsearch.index.mapper.internal.IdFieldMapper; +import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.index.percolator.stats.ShardPercolateService; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.search.nested.NonNestedDocsFilter; @@ -99,8 +100,6 @@ import static org.elasticsearch.percolator.QueryCollector.match; import static org.elasticsearch.percolator.QueryCollector.matchAndScore; -/** - */ public class PercolatorService extends AbstractComponent { public final static float NO_SCORE = Float.NEGATIVE_INFINITY; @@ -741,18 +740,18 @@ public PercolateShardResponse doPercolate(PercolateShardRequest request, Percola hls = new ArrayList<>(topDocs.scoreDocs.length); } - final FieldMapper idMapper = context.mapperService().smartNameFieldMapper(IdFieldMapper.NAME); - final IndexFieldData idFieldData = context.fieldData().getForField(idMapper); + final FieldMapper uidMapper = context.mapperService().smartNameFieldMapper(UidFieldMapper.NAME); + final IndexFieldData uidFieldData = context.fieldData().getForField(uidMapper); int i = 0; for (ScoreDoc scoreDoc : topDocs.scoreDocs) { int segmentIdx = ReaderUtil.subIndex(scoreDoc.doc, percolatorSearcher.reader().leaves()); LeafReaderContext atomicReaderContext = percolatorSearcher.reader().leaves().get(segmentIdx); - SortedBinaryDocValues values = idFieldData.load(atomicReaderContext).getBytesValues(); + SortedBinaryDocValues values = uidFieldData.load(atomicReaderContext).getBytesValues(); final int localDocId = scoreDoc.doc - atomicReaderContext.docBase; values.setDocument(localDocId); final int numValues = values.count(); assert numValues == 1; - BytesRef bytes = values.valueAt(0); + BytesRef bytes = Uid.splitUidIntoTypeAndId(values.valueAt(0))[1]; matches.add(BytesRef.deepCopyOf(bytes)); if (hls != null) { Query query = context.percolateQueries().get(bytes); diff --git a/src/main/java/org/elasticsearch/percolator/QueryCollector.java b/src/main/java/org/elasticsearch/percolator/QueryCollector.java index 763d94f6e96ab..204f19db20215 100644 --- a/src/main/java/org/elasticsearch/percolator/QueryCollector.java +++ b/src/main/java/org/elasticsearch/percolator/QueryCollector.java @@ -19,7 +19,6 @@ package org.elasticsearch.percolator; import com.carrotsearch.hppc.FloatArrayList; - import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; @@ -34,7 +33,8 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.internal.IdFieldMapper; +import org.elasticsearch.index.mapper.Uid; +import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.index.query.ParsedQuery; import org.elasticsearch.index.search.nested.NonNestedDocsFilter; import org.elasticsearch.search.aggregations.Aggregator; @@ -54,7 +54,7 @@ */ abstract class QueryCollector extends SimpleCollector { - final IndexFieldData idFieldData; + final IndexFieldData uidFieldData; final IndexSearcher searcher; final ConcurrentMap queries; final ESLogger logger; @@ -72,8 +72,8 @@ abstract class QueryCollector extends SimpleCollector { this.logger = logger; this.queries = context.percolateQueries(); this.searcher = context.docSearcher(); - final FieldMapper idMapper = context.mapperService().smartNameFieldMapper(IdFieldMapper.NAME); - this.idFieldData = context.fieldData().getForField(idMapper); + final FieldMapper uidMapper = context.mapperService().smartNameFieldMapper(UidFieldMapper.NAME); + this.uidFieldData = context.fieldData().getForField(uidMapper); this.isNestedDoc = isNestedDoc; List aggregatorCollectors = new ArrayList<>(); @@ -111,7 +111,7 @@ public boolean needsScores() { @Override public void doSetNextReader(LeafReaderContext context) throws IOException { // we use the UID because id might not be indexed - values = idFieldData.load(context).getBytesValues(); + values = uidFieldData.load(context).getBytesValues(); aggregatorLeafCollector = aggregatorCollector.getLeafCollector(context); } @@ -139,7 +139,7 @@ protected final Query getQuery(int doc) { return null; } assert numValues == 1; - current = values.valueAt(0); + current = Uid.splitUidIntoTypeAndId(values.valueAt(0))[1]; return queries.get(current); } diff --git a/src/test/java/org/elasticsearch/count/query/CountQueryTests.java b/src/test/java/org/elasticsearch/count/query/CountQueryTests.java index c9e0c78636505..8d5646b8368bd 100644 --- a/src/test/java/org/elasticsearch/count/query/CountQueryTests.java +++ b/src/test/java/org/elasticsearch/count/query/CountQueryTests.java @@ -20,9 +20,13 @@ package org.elasticsearch.count.query; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.action.count.CountResponse; +import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.*; import org.elasticsearch.index.query.CommonTermsQueryBuilder.Operator; @@ -256,10 +260,11 @@ public void idsFilterTestsIdNotIndexed() throws Exception { } private void idsFilterTests(String index) throws Exception { - assertAcked(prepareCreate("test") - .addMapping("type1", jsonBuilder().startObject().startObject("type1") - .startObject("_id").field("index", index).endObject() - .endObject().endObject())); + Settings indexSettings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); + assertAcked(prepareCreate("test").setSettings(indexSettings) + .addMapping("type1", jsonBuilder().startObject().startObject("type1") + .startObject("_id").field("index", index).endObject() + .endObject().endObject())); indexRandom(true, client().prepareIndex("test", "type1", "1").setSource("field1", "value1"), client().prepareIndex("test", "type1", "2").setSource("field1", "value2"), diff --git a/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java b/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java index a90ca1b07d3ab..d57a46f93b466 100644 --- a/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java +++ b/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java @@ -71,11 +71,12 @@ public void simpleIdTests() throws Exception { assertThat(doc.rootDoc().get(IdFieldMapper.NAME), nullValue()); } - public void testIdIndexed() throws Exception { + public void testIdIndexedBackcompat() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_id").field("index", "not_analyzed").endObject() .endObject().endObject().string(); - DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping); + Settings indexSettings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); + DocumentMapper docMapper = createIndex("test", indexSettings).mapperService().documentMapperParser().parse(mapping); ParsedDocument doc = docMapper.parse("type", "1", XContentFactory.jsonBuilder() .startObject() @@ -95,7 +96,7 @@ public void testIdIndexed() throws Exception { assertThat(doc.rootDoc().get(IdFieldMapper.NAME), notNullValue()); } - public void testIdPath() throws Exception { + public void testIdPathBackcompat() throws Exception { String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") .startObject("_id").field("path", "my_path").endObject() .endObject().endObject().string(); diff --git a/src/test/java/org/elasticsearch/index/mapper/simple/test-mapping.json b/src/test/java/org/elasticsearch/index/mapper/simple/test-mapping.json index aadbbe408681c..f0e97d4bf3f7d 100644 --- a/src/test/java/org/elasticsearch/index/mapper/simple/test-mapping.json +++ b/src/test/java/org/elasticsearch/index/mapper/simple/test-mapping.json @@ -6,8 +6,6 @@ date_formats:["yyyy-MM-dd", "dd-MM-yyyy"], dynamic:false, enabled:true, - _id:{ - }, _source:{ }, _type:{ diff --git a/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenTests.java b/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenTests.java index 4c7b1d29ca3e2..f4dca96f239f6 100644 --- a/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenTests.java +++ b/src/test/java/org/elasticsearch/search/aggregations/bucket/ChildrenTests.java @@ -55,8 +55,8 @@ public class ChildrenTests extends ElasticsearchIntegrationTest { public void setupSuiteScopeCluster() throws Exception { assertAcked( prepareCreate("test") - .addMapping("article", "_id", "index=not_analyzed") - .addMapping("comment", "_parent", "type=article", "_id", "index=not_analyzed") + .addMapping("article") + .addMapping("comment", "_parent", "type=article") ); List requests = new ArrayList<>(); @@ -70,6 +70,7 @@ public void setupSuiteScopeCluster() throws Exception { for (int i = 0; i < numParentDocs; i++) { String id = Integer.toString(i); + // TODO: this array is always of length 1, and testChildrenAggs fails if this is changed String[] categories = new String[randomIntBetween(1,1)]; for (int j = 0; j < categories.length; j++) { String category = categories[j] = uniqueCategories[catIndex++ % uniqueCategories.length]; @@ -165,7 +166,7 @@ public void testParentWithMultipleBuckets() throws Exception { .setQuery(matchQuery("randomized", false)) .addAggregation( terms("category").field("category").size(0).subAggregation( - children("to_comment").childType("comment").subAggregation(topHits("top_comments").addSort("_id", SortOrder.ASC)) + children("to_comment").childType("comment").subAggregation(topHits("top_comments").addSort("_uid", SortOrder.ASC)) ) ).get(); assertSearchResponse(searchResponse); @@ -192,10 +193,8 @@ public void testParentWithMultipleBuckets() throws Exception { assertThat(childrenBucket.getDocCount(), equalTo(2l)); TopHits topHits = childrenBucket.getAggregations().get("top_comments"); assertThat(topHits.getHits().totalHits(), equalTo(2l)); - assertThat(topHits.getHits().getAt(0).sortValues()[0].toString(), equalTo("a")); assertThat(topHits.getHits().getAt(0).getId(), equalTo("a")); assertThat(topHits.getHits().getAt(0).getType(), equalTo("comment")); - assertThat(topHits.getHits().getAt(1).sortValues()[0].toString(), equalTo("c")); assertThat(topHits.getHits().getAt(1).getId(), equalTo("c")); assertThat(topHits.getHits().getAt(1).getType(), equalTo("comment")); diff --git a/src/test/java/org/elasticsearch/search/query/SearchQueryTests.java b/src/test/java/org/elasticsearch/search/query/SearchQueryTests.java index 2e928995b84fe..b27679e6d9134 100644 --- a/src/test/java/org/elasticsearch/search/query/SearchQueryTests.java +++ b/src/test/java/org/elasticsearch/search/query/SearchQueryTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.query.*; @@ -629,7 +630,8 @@ public void idsFilterTestsIdNotIndexed() throws Exception { } private void idsFilterTests(String index) throws Exception { - assertAcked(client().admin().indices().prepareCreate("test") + Settings indexSettings = ImmutableSettings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build(); + assertAcked(client().admin().indices().prepareCreate("test").setSettings(indexSettings) .addMapping("type1", jsonBuilder().startObject().startObject("type1") .startObject("_id").field("index", index).endObject() .endObject().endObject())); diff --git a/src/test/java/org/elasticsearch/search/sort/SimpleSortTests.java b/src/test/java/org/elasticsearch/search/sort/SimpleSortTests.java index 1af9f623bef7b..52f4bd4f2af85 100644 --- a/src/test/java/org/elasticsearch/search/sort/SimpleSortTests.java +++ b/src/test/java/org/elasticsearch/search/sort/SimpleSortTests.java @@ -1562,7 +1562,6 @@ public void testSortMetaField() throws Exception { final boolean timestampDocValues = maybeDocValues(); assertAcked(prepareCreate("test") .addMapping("type", XContentFactory.jsonBuilder().startObject().startObject("type") - .startObject("_id").field("index", !idDocValues || randomBoolean() ? "not_analyzed" : "no").startObject("fielddata").field("format", idDocValues ? "doc_values" : null).endObject().endObject() .startObject("_timestamp").field("enabled", true).field("store", true).field("index", !timestampDocValues || randomBoolean() ? "not_analyzed" : "no").startObject("fielddata").field("format", timestampDocValues ? "doc_values" : null).endObject().endObject() .endObject().endObject())); ensureGreen(); @@ -1588,7 +1587,8 @@ public void testSortMetaField() throws Exception { previous = uid; } - /*searchResponse = client().prepareSearch() + /* + searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .setSize(randomIntBetween(1, numDocs + 5)) .addSort("_id", order) diff --git a/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java b/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java index 672f0d97c7c26..68a68f47e39e0 100644 --- a/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java +++ b/src/test/java/org/elasticsearch/test/ElasticsearchIntegrationTest.java @@ -312,11 +312,6 @@ private void randomIndexTemplate() throws IOException { XContentBuilder mappings = null; if (frequently() && randomDynamicTemplates()) { mappings = XContentFactory.jsonBuilder().startObject().startObject("_default_"); - if (randomBoolean()) { - mappings.startObject(IdFieldMapper.NAME) - .field("index", randomFrom("not_analyzed", "no")) - .endObject(); - } if (randomBoolean()) { mappings.startObject(TypeFieldMapper.NAME) .field("index", randomFrom("no", "not_analyzed"))