From d4be8c10cabccd93e6bc81a077070aebcb04eb4b Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Mon, 11 Aug 2014 12:11:31 +0200 Subject: [PATCH] Added doc values support to _parent field. On indices created on or after 1.4.0 will store the _parent field also as doc values. Also added `index._parent.doc_values` option which controls whether doc values are used for parent/child field data, if set to false parent/child field data will be created on the fly based on _parent field inverted index. The `index._parent.doc_values` defaults to true. Closes #6107 Closes #6511 --- .../common/collect/ImmutableOpenSet.java | 129 ++++++++++++++++++ .../PerFieldMappingPostingFormatCodec.java | 19 ++- .../plain/ParentChildIndexFieldData.java | 38 ++++++ .../index/mapper/MapperService.java | 2 + .../mapper/internal/ParentFieldMapper.java | 85 ++++++++++-- .../search/child/ChildSearchBenchmark.java | 2 + .../search/child/AbstractChildTests.java | 9 +- .../ChildrenConstantScoreQueryTests.java | 23 +++- .../search/child/ChildrenQueryTests.java | 13 +- .../child/ParentConstantScoreQueryTests.java | 13 +- .../index/search/child/ParentQueryTests.java | 13 +- .../child/SimpleChildQuerySearchTests.java | 17 ++- .../elasticsearch/test/TestSearchContext.java | 4 + 13 files changed, 350 insertions(+), 17 deletions(-) create mode 100644 src/main/java/org/elasticsearch/common/collect/ImmutableOpenSet.java diff --git a/src/main/java/org/elasticsearch/common/collect/ImmutableOpenSet.java b/src/main/java/org/elasticsearch/common/collect/ImmutableOpenSet.java new file mode 100644 index 0000000000000..43a88419b42e8 --- /dev/null +++ b/src/main/java/org/elasticsearch/common/collect/ImmutableOpenSet.java @@ -0,0 +1,129 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.collect; + +import com.carrotsearch.hppc.ObjectOpenHashSet; +import com.carrotsearch.hppc.cursors.ObjectCursor; + +import java.util.Iterator; + +/** + * Immutable set based on the hppc {@link ObjectOpenHashSet}. + * + * For a copy on write this immutable set is more efficient because of the {@link #builder(ImmutableOpenSet)} method + * that uses cloning on array backing the wrapped set. + */ +public final class ImmutableOpenSet implements Iterable> { + + @SuppressWarnings("unchecked") + private final static ImmutableOpenSet EMPTY = new ImmutableOpenSet(new ObjectOpenHashSet()); + + private final ObjectOpenHashSet set; + + private ImmutableOpenSet(ObjectOpenHashSet set) { + this.set = set; + } + + /** + * {@inheritDoc} + */ + @Override + public Iterator> iterator() { + return set.iterator(); + } + + /** + * @see ObjectOpenHashSet#contains(Object) + */ + public boolean contains(KType e) { + return set.contains(e); + } + + /** + * @return an empty immutable set + */ + @SuppressWarnings("unchecked") + public static ImmutableOpenSet of() { + return EMPTY; + } + + /** + * @return a builder to create an immutable set + */ + public static Builder builder() { + return new Builder<>(); + } + + /** + * @return a builder to create an immutable set and preset with the specified initial capacity. + */ + public static Builder builder(int initialCapacity) { + return new Builder<>(initialCapacity); + } + + /** + * @return a builder to create an immutable set based on existing immutable set. + */ + public static Builder builder(ImmutableOpenSet set) { + return new Builder<>(set); + } + + public final static class Builder { + + private ObjectOpenHashSet set; + + Builder() { + //noinspection unchecked + this(EMPTY); + } + + Builder(int initialCapacity) { + this.set = new ObjectOpenHashSet<>(initialCapacity); + } + + Builder(ImmutableOpenSet set) { + this.set = set.set.clone(); + } + + /** + * @see ObjectOpenHashSet#add(Object) + */ + public boolean add(KType k) { + return set.add(k); + } + + /** + * @see ObjectOpenHashSet#remove(Object) + */ + public boolean remove(KType key) { + return set.remove(key); + } + + /** + * @return an immutable set with the elements added via this builder + */ + public ImmutableOpenSet build() { + ObjectOpenHashSet set = this.set; + this.set = null; + return new ImmutableOpenSet<>(set); + } + + } +} diff --git a/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java b/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java index 88350ee875ecc..550644aaa0d53 100644 --- a/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -25,8 +25,10 @@ import org.elasticsearch.common.logging.ESLogger; import org.elasticsearch.index.codec.docvaluesformat.DocValuesFormatProvider; import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider; +import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.FieldMappers; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.internal.ParentFieldMapper; /** * {@link PerFieldMappingPostingFormatCodec This postings format} is the default @@ -63,7 +65,22 @@ public PostingsFormat getPostingsFormatForField(String field) { @Override public DocValuesFormat getDocValuesFormatForField(String field) { - final FieldMappers indexName = mapperService.indexName(field); + FieldMappers indexName = mapperService.indexName(field); + if (indexName == null) { + // The _parent field produces doc values fields that start with the prefix: _parent + // Since those fields don't match with the index field name of the _parent field itself, this would print + // a warning. Catching this on this point seems like the cleanest solution to me. + if (field.startsWith(ParentFieldMapper.NAME)) { + int indexOf = field.indexOf('#'); + if (indexOf != -1) { + String type = field.substring(indexOf + 1); + DocumentMapper documentMapper = mapperService.documentMapper(type); + if (documentMapper != null) { + indexName = documentMapper.mappers().indexName(ParentFieldMapper.NAME); + } + } + } + } if (indexName == null) { logger.warn("no index mapper found for field: [{}] returning default doc values format", field); return defaultDocValuesFormat; diff --git a/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java b/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java index 5e52b5a438f75..a8904d07c68c8 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java +++ b/src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java @@ -60,9 +60,14 @@ */ public class ParentChildIndexFieldData extends AbstractIndexFieldData implements IndexParentChildFieldData, DocumentTypeListener { + // From 2.0 and up this option should be removed and parent/child only work with doc values + public final static String PARENT_DOC_VALUES = "index._parent.doc_values"; + private final NavigableSet parentTypes; private final CircuitBreakerService breakerService; + private final boolean docValues; + // If child type (a type with _parent field) is added or removed, we want to make sure modifications don't happen // while loading. private final Object lock = new Object(); @@ -77,6 +82,7 @@ public ParentChildIndexFieldData(Index index, @IndexSettings Settings indexSetti beforeCreate(documentMapper); } mapperService.addTypeListener(this); + this.docValues = indexSettings.getAsBoolean(PARENT_DOC_VALUES, true); } @Override @@ -84,6 +90,38 @@ public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, Mu return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); } + @Override + public AtomicParentChildFieldData load(AtomicReaderContext context) { + if (docValues) { + final NavigableSet parentTypes; + synchronized (lock) { + parentTypes = ImmutableSortedSet.copyOf(BytesRef.getUTF8SortedAsUnicodeComparator(), this.parentTypes); + } + + AtomicReader reader = context.reader(); + ObjectObjectOpenHashMap typeDvBuilders = ObjectObjectOpenHashMap.newInstance(); + for (BytesRef parentType : parentTypes) { + String type = parentType.utf8ToString(); + String field = ParentFieldMapper.NAME + "#" + type; + FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field); + if (fieldInfo != null && fieldInfo.hasDocValues()) { + typeDvBuilders.put(type, new SortedSetDVBytesAtomicFieldData(reader, field)); + } + } + if (!typeDvBuilders.isEmpty()) { + // An index has either doc values fields for p/c or none at all + assert typeDvBuilders.size() == parentTypes.size(); + ImmutableOpenMap.Builder typeToAtomicFieldData = ImmutableOpenMap.builder(typeDvBuilders.size()); + for (ObjectObjectCursor cursor : typeDvBuilders) { + typeToAtomicFieldData.put(cursor.key, cursor.value); + } + return new ParentChildAtomicFieldData(typeToAtomicFieldData.build()); + } + // If we got here then this is a < 1.4.0 index, which doesn't have doc values field for _parent mapping + } + return super.load(context); + } + @Override public ParentChildAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception { AtomicReader reader = context.reader(); diff --git a/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/src/main/java/org/elasticsearch/index/mapper/MapperService.java index 64c3da4764961..c79c525ee525b 100755 --- a/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -362,6 +362,7 @@ private DocumentMapper merge(DocumentMapper mapper) { mapper.traverse(objectMappersAgg); addObjectMappers(objectMappersAgg.mappers.toArray(new ObjectMapper[objectMappersAgg.mappers.size()])); mapper.addObjectMapperListener(objectMapperListener, false); + mapper.parentFieldMapper().setMappingService(this); for (DocumentTypeListener typeListener : typeListeners) { typeListener.beforeCreate(mapper); @@ -405,6 +406,7 @@ public void remove(String type) { return; } docMapper.close(); + removeTypeListener(docMapper.parentFieldMapper()); mappers = newMapBuilder(mappers).remove(type).map(); removeObjectAndFieldMappers(docMapper); for (DocumentTypeListener typeListener : typeListeners) { diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java index 10cf726256c06..6ade756dbddab 100644 --- a/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java +++ b/src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java @@ -20,6 +20,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.Term; import org.apache.lucene.queries.TermFilter; @@ -28,8 +29,10 @@ import org.apache.lucene.search.Filter; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Version; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.collect.ImmutableOpenSet; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.Queries; @@ -40,6 +43,7 @@ import org.elasticsearch.index.fielddata.FieldDataType; import org.elasticsearch.index.mapper.*; import org.elasticsearch.index.mapper.core.AbstractFieldMapper; +import org.elasticsearch.index.mapper.core.TypeParsers; import org.elasticsearch.index.query.QueryParseContext; import java.io.IOException; @@ -52,7 +56,7 @@ /** * */ -public class ParentFieldMapper extends AbstractFieldMapper implements InternalMapper, RootMapper { +public class ParentFieldMapper extends AbstractFieldMapper implements InternalMapper, RootMapper, DocumentTypeListener { public static final String NAME = "_parent"; @@ -64,6 +68,7 @@ public static class Defaults extends AbstractFieldMapper.Defaults { public static final FieldType FIELD_TYPE = new FieldType(AbstractFieldMapper.Defaults.FIELD_TYPE); static { + // TODO: index and store can be false now. Should we do this in 2.0? FIELD_TYPE.setIndexed(true); FIELD_TYPE.setTokenized(false); FIELD_TYPE.setStored(true); @@ -105,7 +110,8 @@ public ParentFieldMapper build(BuilderContext context) { if (type == null) { throw new MapperParsingException("Parent mapping must contain the parent type"); } - return new ParentFieldMapper(name, indexName, type, postingsFormat, FIELD_DATA_SETTINGS, context.indexSettings()); + boolean docValues = context.indexCreatedVersion().onOrAfter(Version.V_1_4_0); + return new ParentFieldMapper(name, indexName, type, postingsFormat, FIELD_DATA_SETTINGS, context.indexSettings(), docValues); } } @@ -129,16 +135,20 @@ public Mapper.Builder parse(String name, Map node, ParserContext private final String type; private final BytesRef typeAsBytes; + private final boolean storeDocValues; + private MapperService mapperService; + private volatile ImmutableOpenSet parentTypes; - protected ParentFieldMapper(String name, String indexName, String type, PostingsFormatProvider postingsFormat, @Nullable Settings fieldDataSettings, Settings indexSettings) { + protected ParentFieldMapper(String name, String indexName, String type, PostingsFormatProvider postingsFormat, @Nullable Settings fieldDataSettings, Settings indexSettings, boolean storeDocValues) { super(new Names(name, indexName, indexName, name), Defaults.BOOST, new FieldType(Defaults.FIELD_TYPE), null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, postingsFormat, null, null, null, fieldDataSettings, indexSettings); this.type = type; this.typeAsBytes = type == null ? null : new BytesRef(type); + this.storeDocValues = storeDocValues; } public ParentFieldMapper() { - this(Defaults.NAME, Defaults.NAME, null, null, null, null); + this(Defaults.NAME, Defaults.NAME, null, null, null, null, false); } public String type() { @@ -157,7 +167,7 @@ public FieldDataType defaultFieldDataType() { @Override public boolean hasDocValues() { - return false; + return storeDocValues; } @Override @@ -177,19 +187,29 @@ public boolean includeInObject() { @Override protected void parseCreateField(ParseContext context, List fields) throws IOException { if (!active()) { + // If this is a parent document referred by a child doc that has doc values enabled in the _parent field + // we need to save the id in the _parent doc values field + for (DocumentMapper documentMapper : mapperService.docMappers(false)) { + ParentFieldMapper parentFieldMapper = documentMapper.parentFieldMapper(); + if (parentFieldMapper.active() && parentFieldMapper.hasDocValues() && parentFieldMapper.parentTypes.contains(context.type())) { + fields.add(createParentIdField(context.type(), context.id())); + break; + } + } return; } + String parentId = null; if (context.parser().currentName() != null && context.parser().currentName().equals(Defaults.NAME)) { // we are in the parsing of _parent phase - String parentId = context.parser().text(); + parentId = context.parser().text(); context.sourceToParse().parent(parentId); fields.add(new Field(names.indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType)); } else { // otherwise, we are running it post processing of the xcontent String parsedParentId = context.doc().get(Defaults.NAME); if (context.sourceToParse().parent() != null) { - String parentId = context.sourceToParse().parent(); + parentId = context.sourceToParse().parent(); if (parsedParentId == null) { if (parentId == null) { throw new MapperParsingException("No parent id provided, not within the document, and not externally"); @@ -201,7 +221,21 @@ protected void parseCreateField(ParseContext context, List fields) throws } } } - // we have parent mapping, yet no value was set, ignore it... + if (hasDocValues() && parentId != null) { + // Doc values is enabled for the _parent field, so we add an extra field + fields.add(createParentIdField(type(), parentId)); + } + + // A document can be both child and parent: + DocumentMapper documentMapper = mapperService.documentMapper(context.type()); + if (documentMapper.parentFieldMapper().active() && documentMapper.parentFieldMapper().hasDocValues() && parentTypes.contains(context.type())) { + fields.add(createParentIdField(context.type(), context.id())); + } + } + + public static SortedSetDocValuesField createParentIdField(String parentType, String id) { + String fieldName = ParentFieldMapper.NAME + "#" + parentType; + return new SortedSetDocValuesField(fieldName, new BytesRef(id)); } @Override @@ -333,9 +367,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (!active()) { return builder; } + boolean includeDefaults = params.paramAsBoolean("include_defaults", false); builder.startObject(CONTENT_TYPE); builder.field("type", type); + builder.field(TypeParsers.DOC_VALUES, storeDocValues); builder.endObject(); return builder; } @@ -347,7 +383,7 @@ public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappi return; } - if (active() != other.active() || !type.equals(other.type)) { + if (active() != other.active() || !type.equals(other.type) || hasDocValues() != other.hasDocValues()) { mergeContext.addConflict("The _parent field can't be added or updated"); } } @@ -359,4 +395,35 @@ public boolean active() { return type != null; } + @Override + public void beforeCreate(DocumentMapper mapper) { + ParentFieldMapper fieldMapper = mapper.parentFieldMapper(); + if (fieldMapper.active()) { + ImmutableOpenSet.Builder builder = ImmutableOpenSet.builder(parentTypes); + builder.add(fieldMapper.type()); + this.parentTypes = builder.build(); + } + } + + @Override + public void afterRemove(DocumentMapper mapper) { + ParentFieldMapper fieldMapper = mapper.parentFieldMapper(); + if (fieldMapper.active()) { + ImmutableOpenSet.Builder builder = ImmutableOpenSet.builder(parentTypes); + builder.remove(fieldMapper.type()); + this.parentTypes = builder.build(); + } + } + + public void setMappingService(MapperService mapperService) { + ImmutableOpenSet.Builder parentTypesBuilder = ImmutableOpenSet.builder(); + for (DocumentMapper documentMapper : mapperService.docMappers(false)) { + if (documentMapper.parentFieldMapper().active()) { + parentTypesBuilder.add(documentMapper.parentFieldMapper().type()); + } + } + mapperService.addTypeListener(this); + this.mapperService = mapperService; + this.parentTypes = parentTypesBuilder.build(); + } } diff --git a/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java b/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java index b8155147c886d..ba1becb6bde3f 100644 --- a/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java +++ b/src/test/java/org/elasticsearch/benchmark/search/child/ChildSearchBenchmark.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.SizeValue; import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; import org.elasticsearch.indices.IndexAlreadyExistsException; import org.elasticsearch.node.Node; @@ -46,6 +47,7 @@ public class ChildSearchBenchmark { public static void main(String[] args) throws Exception { Settings settings = settingsBuilder() .put("index.refresh_interval", "-1") + .put(ParentChildIndexFieldData.PARENT_DOC_VALUES, true) .put("gateway.type", "local") .put(SETTING_NUMBER_OF_SHARDS, 1) .put(SETTING_NUMBER_OF_REPLICAS, 0) diff --git a/src/test/java/org/elasticsearch/index/search/child/AbstractChildTests.java b/src/test/java/org/elasticsearch/index/search/child/AbstractChildTests.java index f7b03d18ac711..692e8fc72582c 100644 --- a/src/test/java/org/elasticsearch/index/search/child/AbstractChildTests.java +++ b/src/test/java/org/elasticsearch/index/search/child/AbstractChildTests.java @@ -27,11 +27,14 @@ import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; import org.elasticsearch.common.compress.CompressedString; +import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.internal.UidFieldMapper; import org.elasticsearch.index.service.IndexService; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.test.ElasticsearchSingleNodeLuceneTestCase; +import org.elasticsearch.test.TestSearchContext; import org.hamcrest.Description; import org.hamcrest.StringDescription; import org.junit.Ignore; @@ -45,12 +48,16 @@ public abstract class AbstractChildTests extends ElasticsearchSingleNodeLuceneTestCase { static SearchContext createSearchContext(String indexName, String parentType, String childType) throws IOException { - IndexService indexService = createIndex(indexName); + IndexService indexService = createIndex(indexName, ImmutableSettings.builder().put(ParentChildIndexFieldData.PARENT_DOC_VALUES, random().nextBoolean()).build()); MapperService mapperService = indexService.mapperService(); mapperService.merge(childType, new CompressedString(PutMappingRequest.buildFromSimplifiedDef(childType, "_parent", "type=" + parentType).string()), true); return createSearchContext(indexService); } + static boolean useParentDocValues() { + return ((TestSearchContext) SearchContext.current()).getIndexService().settingsService().getSettings().getAsBoolean(ParentChildIndexFieldData.PARENT_DOC_VALUES, true); + } + static void assertBitSet(FixedBitSet actual, FixedBitSet expected, IndexSearcher searcher) throws IOException { if (!actual.equals(expected)) { Description description = new StringDescription(); diff --git a/src/test/java/org/elasticsearch/index/search/child/ChildrenConstantScoreQueryTests.java b/src/test/java/org/elasticsearch/index/search/child/ChildrenConstantScoreQueryTests.java index 0ad34a076d5d6..81e2d3836bd45 100644 --- a/src/test/java/org/elasticsearch/index/search/child/ChildrenConstantScoreQueryTests.java +++ b/src/test/java/org/elasticsearch/index/search/child/ChildrenConstantScoreQueryTests.java @@ -85,10 +85,15 @@ public void testSimple() throws Exception { Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory); + boolean docValues = useParentDocValues(); for (int parent = 1; parent <= 5; parent++) { + String parentId = Integer.toString(parent); Document document = new Document(); - document.add(new StringField(UidFieldMapper.NAME, Uid.createUid("parent", Integer.toString(parent)), Field.Store.NO)); + document.add(new StringField(UidFieldMapper.NAME, Uid.createUid("parent", parentId), Field.Store.NO)); document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO)); + if (docValues) { + document.add(ParentFieldMapper.createParentIdField("parent", parentId)); + } indexWriter.addDocument(document); for (int child = 1; child <= 3; child++) { @@ -97,6 +102,9 @@ public void testSimple() throws Exception { document.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO)); document.add(new StringField(ParentFieldMapper.NAME, Uid.createUid("parent", Integer.toString(parent)), Field.Store.NO)); document.add(new StringField("field1", "value" + child, Field.Store.NO)); + if (docValues) { + document.add(ParentFieldMapper.createParentIdField("parent", parentId)); + } indexWriter.addDocument(document); } } @@ -140,6 +148,7 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) childValues[i] = Integer.toString(i); } + boolean docValues = useParentDocValues(); IntOpenHashSet filteredOrDeletedDocs = new IntOpenHashSet(); int childDocId = 0; int numParentDocs = scaledRandomIntBetween(1, numUniqueChildValues); @@ -159,6 +168,9 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) filteredOrDeletedDocs.add(parentDocId); document.add(new StringField("filter", "me", Field.Store.NO)); } + if (docValues) { + document.add(ParentFieldMapper.createParentIdField("parent", parent)); + } indexWriter.addDocument(document); final int numChildDocs = scaledRandomIntBetween(0, 100); @@ -174,6 +186,9 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) if (markChildAsDeleted) { document.add(new StringField("delete", "me", Field.Store.NO)); } + if (docValues) { + document.add(ParentFieldMapper.createParentIdField("parent", parent)); + } indexWriter.addDocument(document); if (!markChildAsDeleted) { @@ -233,13 +248,17 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) do { parentId = random().nextInt(numParentDocs); } while (filteredOrDeletedDocs.contains(parentId)); + String parent = Integer.toString(parentId); - String parentUid = Uid.createUid("parent", Integer.toString(parentId)); + String parentUid = Uid.createUid("parent", parent); indexWriter.deleteDocuments(new Term(UidFieldMapper.NAME, parentUid)); Document document = new Document(); document.add(new StringField(UidFieldMapper.NAME, parentUid, Field.Store.YES)); document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO)); + if (docValues) { + document.add(ParentFieldMapper.createParentIdField("parent", parent)); + } indexWriter.addDocument(document); } diff --git a/src/test/java/org/elasticsearch/index/search/child/ChildrenQueryTests.java b/src/test/java/org/elasticsearch/index/search/child/ChildrenQueryTests.java index 624e005b1f98b..61fdbea3b7808 100644 --- a/src/test/java/org/elasticsearch/index/search/child/ChildrenQueryTests.java +++ b/src/test/java/org/elasticsearch/index/search/child/ChildrenQueryTests.java @@ -102,6 +102,7 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) IntOpenHashSet filteredOrDeletedDocs = new IntOpenHashSet(); int childDocId = 0; + boolean docValues = useParentDocValues(); int numParentDocs = scaledRandomIntBetween(1, numUniqueChildValues); ObjectObjectOpenHashMap> childValueToParentIds = new ObjectObjectOpenHashMap<>(); for (int parentDocId = 0; parentDocId < numParentDocs; parentDocId++) { @@ -119,6 +120,9 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) filteredOrDeletedDocs.add(parentDocId); document.add(new StringField("filter", "me", Field.Store.NO)); } + if (docValues) { + document.add(ParentFieldMapper.createParentIdField("parent", parent)); + } indexWriter.addDocument(document); int numChildDocs = scaledRandomIntBetween(0, 100); @@ -134,6 +138,9 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) if (markChildAsDeleted) { document.add(new StringField("delete", "me", Field.Store.NO)); } + if (docValues) { + document.add(ParentFieldMapper.createParentIdField("parent", parent)); + } indexWriter.addDocument(document); if (!markChildAsDeleted) { @@ -197,13 +204,17 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) do { parentId = random().nextInt(numParentDocs); } while (filteredOrDeletedDocs.contains(parentId)); + String parent = Integer.toString(parentId); - String parentUid = Uid.createUid("parent", Integer.toString(parentId)); + String parentUid = Uid.createUid("parent", parent); indexWriter.deleteDocuments(new Term(UidFieldMapper.NAME, parentUid)); Document document = new Document(); document.add(new StringField(UidFieldMapper.NAME, parentUid, Field.Store.YES)); document.add(new StringField(TypeFieldMapper.NAME, "parent", Field.Store.NO)); + if (docValues) { + document.add(ParentFieldMapper.createParentIdField("parent", parent)); + } indexWriter.addDocument(document); } diff --git a/src/test/java/org/elasticsearch/index/search/child/ParentConstantScoreQueryTests.java b/src/test/java/org/elasticsearch/index/search/child/ParentConstantScoreQueryTests.java index c2c2fbd638da3..106851d6fc1dd 100644 --- a/src/test/java/org/elasticsearch/index/search/child/ParentConstantScoreQueryTests.java +++ b/src/test/java/org/elasticsearch/index/search/child/ParentConstantScoreQueryTests.java @@ -95,6 +95,7 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) } int childDocId = 0; + boolean docValues = useParentDocValues(); int numParentDocs = scaledRandomIntBetween(1, numUniqueParentValues); ObjectObjectOpenHashMap> parentValueToChildDocIds = new ObjectObjectOpenHashMap<>(); IntIntOpenHashMap childIdToParentId = new IntIntOpenHashMap(); @@ -109,6 +110,9 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) if (markParentAsDeleted) { document.add(new StringField("delete", "me", Field.Store.NO)); } + if (docValues) { + document.add(ParentFieldMapper.createParentIdField("parent", parent)); + } indexWriter.addDocument(document); int numChildDocs = scaledRandomIntBetween(0, 100); @@ -131,6 +135,9 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) if (filterMe) { document.add(new StringField("filter", "me", Field.Store.NO)); } + if (docValues) { + document.add(ParentFieldMapper.createParentIdField("parent", parent)); + } indexWriter.addDocument(document); if (!markParentAsDeleted) { @@ -190,12 +197,16 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) int childId = childIds[random().nextInt(childIds.length)]; String childUid = Uid.createUid("child", Integer.toString(childId)); indexWriter.deleteDocuments(new Term(UidFieldMapper.NAME, childUid)); + String parent = Integer.toString(childIdToParentId.get(childId)); Document document = new Document(); document.add(new StringField(UidFieldMapper.NAME, childUid, Field.Store.YES)); document.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO)); - String parentUid = Uid.createUid("parent", Integer.toString(childIdToParentId.get(childId))); + String parentUid = Uid.createUid("parent", parent); document.add(new StringField(ParentFieldMapper.NAME, parentUid, Field.Store.NO)); + if (docValues) { + document.add(ParentFieldMapper.createParentIdField("parent", parent)); + } indexWriter.addDocument(document); } diff --git a/src/test/java/org/elasticsearch/index/search/child/ParentQueryTests.java b/src/test/java/org/elasticsearch/index/search/child/ParentQueryTests.java index 476859bd6b761..b9cf82d724b5a 100644 --- a/src/test/java/org/elasticsearch/index/search/child/ParentQueryTests.java +++ b/src/test/java/org/elasticsearch/index/search/child/ParentQueryTests.java @@ -94,6 +94,7 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) } int childDocId = 0; + boolean docValues = useParentDocValues(); int numParentDocs = scaledRandomIntBetween(1, numUniqueParentValues); ObjectObjectOpenHashMap> parentValueToChildIds = new ObjectObjectOpenHashMap<>(); IntIntOpenHashMap childIdToParentId = new IntIntOpenHashMap(); @@ -108,6 +109,9 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) if (markParentAsDeleted) { document.add(new StringField("delete", "me", Field.Store.NO)); } + if (docValues) { + document.add(ParentFieldMapper.createParentIdField("parent", parent)); + } indexWriter.addDocument(document); int numChildDocs = scaledRandomIntBetween(0, 100); @@ -129,6 +133,9 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) if (filterMe) { document.add(new StringField("filter", "me", Field.Store.NO)); } + if (docValues) { + document.add(ParentFieldMapper.createParentIdField("parent", parent)); + } indexWriter.addDocument(document); if (!markParentAsDeleted) { @@ -189,12 +196,16 @@ LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r)) int childId = childIds[random().nextInt(childIds.length)]; String childUid = Uid.createUid("child", Integer.toString(childId)); indexWriter.deleteDocuments(new Term(UidFieldMapper.NAME, childUid)); + String parent = Integer.toString(childIdToParentId.get(childId)); Document document = new Document(); document.add(new StringField(UidFieldMapper.NAME, childUid, Field.Store.YES)); document.add(new StringField(TypeFieldMapper.NAME, "child", Field.Store.NO)); - String parentUid = Uid.createUid("parent", Integer.toString(childIdToParentId.get(childId))); + String parentUid = Uid.createUid("parent", parent); document.add(new StringField(ParentFieldMapper.NAME, parentUid, Field.Store.NO)); + if (docValues) { + document.add(ParentFieldMapper.createParentIdField("parent", parent)); + } indexWriter.addDocument(document); } diff --git a/src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java b/src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java index 5923fb8e58024..9e0657ff7247f 100644 --- a/src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java +++ b/src/test/java/org/elasticsearch/search/child/SimpleChildQuerySearchTests.java @@ -20,6 +20,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchIllegalArgumentException; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse; import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; @@ -34,9 +35,11 @@ import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.lucene.search.function.CombineFunction; import org.elasticsearch.common.settings.ImmutableSettings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.fielddata.FieldDataType; +import org.elasticsearch.index.fielddata.plain.ParentChildIndexFieldData; import org.elasticsearch.index.mapper.FieldMapper.Loading; import org.elasticsearch.index.mapper.MergeMappingException; import org.elasticsearch.index.query.*; @@ -72,6 +75,16 @@ */ public class SimpleChildQuerySearchTests extends ElasticsearchIntegrationTest { + @Override + public Settings indexSettings() { + return ImmutableSettings.builder() + .put(super.indexSettings()) + // Before 1.4.0 we don't store parent ids in doc values, after 1.4.0 we do. + .put(IndexMetaData.SETTING_VERSION_CREATED, randomBoolean() ? Version.V_1_3_0 : Version.V_1_4_0) + .put(ParentChildIndexFieldData.PARENT_DOC_VALUES, randomBoolean()) + .build(); + } + @Test public void multiLevelChild() throws Exception { assertAcked(prepareCreate("test") @@ -265,9 +278,11 @@ public void simpleChildQuery() throws Exception { @Test public void testClearIdCacheBug() throws Exception { + boolean pcDv = randomBoolean(); // enforce lazy loading to make sure that p/c stats are not counted as part of field data assertAcked(prepareCreate("test") .setSettings(ImmutableSettings.builder().put(indexSettings()) + .put(ParentChildIndexFieldData.PARENT_DOC_VALUES, pcDv) .put("index.refresh_interval", -1)) // Disable automatic refresh, so that the _parent doesn't get warmed .addMapping("parent", XContentFactory.jsonBuilder().startObject().startObject("parent") .startObject("properties") @@ -318,7 +333,7 @@ public void testClearIdCacheBug() throws Exception { indicesStatsResponse = client().admin().indices() .prepareStats("test").setFieldData(true).get(); // automatic warm-up has populated the cache since it found a parent field mapper - assertThat(indicesStatsResponse.getTotal().getIdCache().getMemorySizeInBytes(), greaterThan(0l)); + assertThat(indicesStatsResponse.getTotal().getIdCache().getMemorySizeInBytes(), pcDv ? equalTo(0l) : greaterThan(0l)); // Even though p/c is field data based the stats stay zero, because _parent field data field is kept // track of under id cache stats memory wise for bwc assertThat(indicesStatsResponse.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0l)); diff --git a/src/test/java/org/elasticsearch/test/TestSearchContext.java b/src/test/java/org/elasticsearch/test/TestSearchContext.java index f33aa8e1f8af5..dcca463011c32 100644 --- a/src/test/java/org/elasticsearch/test/TestSearchContext.java +++ b/src/test/java/org/elasticsearch/test/TestSearchContext.java @@ -614,4 +614,8 @@ public boolean useSlowScroll() { public SearchContext useSlowScroll(boolean useSlowScroll) { return null; } + + public IndexService getIndexService() { + return indexService; + } }