Added doc values support to _parent field.

On indices created on or after 1.4.0 will store the _parent field also as doc values.
Also added `index._parent.doc_values` option which controls whether doc values are used for parent/child field data, if set to false parent/child field data will be created on the fly based on _parent field inverted index.
The `index._parent.doc_values` defaults to true.

Closes elastic#6107
Closes elastic#6511

src/main/java/org/elasticsearch/common/collect/ImmutableOpenSet.java

@@ -0,0 +1,129 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.common.collect;
+
+import com.carrotsearch.hppc.ObjectOpenHashSet;
+import com.carrotsearch.hppc.cursors.ObjectCursor;
+
+import java.util.Iterator;
+
+/**
+ * Immutable set based on the hppc {@link ObjectOpenHashSet}.
+ *
+ * For a copy on write this immutable set is more efficient because of the {@link #builder(ImmutableOpenSet)} method
+ * that uses cloning on array backing the wrapped set.
+ */
+public final class ImmutableOpenSet<KType> implements Iterable<ObjectCursor<KType>> {
+
+    @SuppressWarnings("unchecked")
+    private final static ImmutableOpenSet EMPTY = new ImmutableOpenSet(new ObjectOpenHashSet());
+
+    private final ObjectOpenHashSet<KType> set;
+
+    private ImmutableOpenSet(ObjectOpenHashSet<KType> set) {
+        this.set = set;
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    @Override
+    public Iterator<ObjectCursor<KType>> iterator() {
+        return set.iterator();
+    }
+
+    /**
+     * @see ObjectOpenHashSet#contains(Object)
+     */
+    public boolean contains(KType e) {
+        return set.contains(e);
+    }
+
+    /**
+     * @return an empty immutable set
+     */
+    @SuppressWarnings("unchecked")
+    public static <T> ImmutableOpenSet<T> of() {
+        return EMPTY;
+    }
+
+    /**
+     * @return a builder to create an immutable set
+     */
+    public static <T> Builder<T> builder() {
+        return new Builder<>();
+    }
+
+    /**
+     * @return a builder to create an immutable set and preset with the specified initial capacity.
+     */
+    public static <T> Builder<T> builder(int initialCapacity) {
+        return new Builder<>(initialCapacity);
+    }
+
+    /**
+     * @return a builder to create an immutable set based on existing immutable set.
+     */
+    public static <T> Builder<T> builder(ImmutableOpenSet<T> set) {
+        return new Builder<>(set);
+    }
+
+    public final static class Builder<KType> {
+
+        private ObjectOpenHashSet<KType> set;
+
+        Builder() {
+            //noinspection unchecked
+            this(EMPTY);
+        }
+
+        Builder(int initialCapacity) {
+            this.set = new ObjectOpenHashSet<>(initialCapacity);
+        }
+
+        Builder(ImmutableOpenSet<KType> set) {
+            this.set = set.set.clone();
+        }
+
+        /**
+         * @see ObjectOpenHashSet#add(Object)
+         */
+        public boolean add(KType k) {
+            return set.add(k);
+        }
+
+        /**
+         * @see ObjectOpenHashSet#remove(Object)
+         */
+        public boolean remove(KType key) {
+            return set.remove(key);
+        }
+
+        /**
+         * @return an immutable set with the elements added via this builder
+         */
+        public ImmutableOpenSet<KType> build() {
+            ObjectOpenHashSet<KType> set = this.set;
+            this.set = null;
+            return new ImmutableOpenSet<>(set);
+        }
+
+    }
+}

src/main/java/org/elasticsearch/index/codec/PerFieldMappingPostingFormatCodec.java

@@ -25,8 +25,10 @@
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.index.codec.docvaluesformat.DocValuesFormatProvider;
 import org.elasticsearch.index.codec.postingsformat.PostingsFormatProvider;
+import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.FieldMappers;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.internal.ParentFieldMapper;
 
 /**
  * {@link PerFieldMappingPostingFormatCodec This postings format} is the default
@@ -63,7 +65,22 @@ public PostingsFormat getPostingsFormatForField(String field) {
 
     @Override
     public DocValuesFormat getDocValuesFormatForField(String field) {
-        final FieldMappers indexName = mapperService.indexName(field);
+        FieldMappers indexName = mapperService.indexName(field);
+        if (indexName == null) {
+            // The _parent field produces doc values fields that start with the prefix: _parent
+            // Since those fields don't match with the index field name of the _parent field itself, this would print
+            // a warning. Catching this on this point seems like the cleanest solution to me.
+            if (field.startsWith(ParentFieldMapper.NAME)) {
+                int indexOf = field.indexOf('#');
+                if (indexOf != -1) {
+                    String type = field.substring(indexOf + 1);
+                    DocumentMapper documentMapper = mapperService.documentMapper(type);
+                    if (documentMapper != null) {
+                        indexName = documentMapper.mappers().indexName(ParentFieldMapper.NAME);
+                    }
+                }
+            }
+        }
         if (indexName == null) {
             logger.warn("no index mapper found for field: [{}] returning default doc values format", field);
             return defaultDocValuesFormat;

src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java

@@ -60,9 +60,14 @@
  */
 public class ParentChildIndexFieldData extends AbstractIndexFieldData<AtomicParentChildFieldData> implements IndexParentChildFieldData, DocumentTypeListener {
 
+    // From 2.0 and up this option should be removed and parent/child only work with doc values
+    public final static String PARENT_DOC_VALUES = "index._parent.doc_values";
+
     private final NavigableSet<BytesRef> parentTypes;
     private final CircuitBreakerService breakerService;
 
+    private final boolean docValues;
+
     // If child type (a type with _parent field) is added or removed, we want to make sure modifications don't happen
     // while loading.
     private final Object lock = new Object();
@@ -77,13 +82,46 @@ public ParentChildIndexFieldData(Index index, @IndexSettings Settings indexSetti
             beforeCreate(documentMapper);
         }
         mapperService.addTypeListener(this);
+        this.docValues = indexSettings.getAsBoolean(PARENT_DOC_VALUES, true);
     }
 
     @Override
     public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, MultiValueMode sortMode, Nested nested) {
         return new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested);
     }
 
+    @Override
+    public AtomicParentChildFieldData load(AtomicReaderContext context) {
+        if (docValues) {
+            final NavigableSet<BytesRef> parentTypes;
+            synchronized (lock) {
+                parentTypes = ImmutableSortedSet.copyOf(BytesRef.getUTF8SortedAsUnicodeComparator(), this.parentTypes);
+            }
+
+            AtomicReader reader = context.reader();
+            ObjectObjectOpenHashMap<String, SortedSetDVBytesAtomicFieldData> typeDvBuilders = ObjectObjectOpenHashMap.newInstance();
+            for (BytesRef parentType : parentTypes) {
+                String type = parentType.utf8ToString();
+                String field = ParentFieldMapper.NAME + "#" + type;
+                FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
+                if (fieldInfo != null && fieldInfo.hasDocValues()) {
+                    typeDvBuilders.put(type, new SortedSetDVBytesAtomicFieldData(reader, field));
+                }
+            }
+            if (!typeDvBuilders.isEmpty()) {
+                // An index has either doc values fields for p/c or none at all
+                assert typeDvBuilders.size() == parentTypes.size();
+                ImmutableOpenMap.Builder<String, AtomicOrdinalsFieldData> typeToAtomicFieldData = ImmutableOpenMap.builder(typeDvBuilders.size());
+                for (ObjectObjectCursor<String, SortedSetDVBytesAtomicFieldData> cursor : typeDvBuilders) {
+                    typeToAtomicFieldData.put(cursor.key, cursor.value);
+                }
+                return new ParentChildAtomicFieldData(typeToAtomicFieldData.build());
+            }
+            // If we got here then this is a < 1.4.0 index, which doesn't have doc values field for _parent mapping
+        }
+        return super.load(context);
+    }
+
     @Override
     public ParentChildAtomicFieldData loadDirect(AtomicReaderContext context) throws Exception {
         AtomicReader reader = context.reader();

src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -362,6 +362,7 @@ private DocumentMapper merge(DocumentMapper mapper) {
                 mapper.traverse(objectMappersAgg);
                 addObjectMappers(objectMappersAgg.mappers.toArray(new ObjectMapper[objectMappersAgg.mappers.size()]));
                 mapper.addObjectMapperListener(objectMapperListener, false);
+                mapper.parentFieldMapper().setMappingService(this);
 
                 for (DocumentTypeListener typeListener : typeListeners) {
                     typeListener.beforeCreate(mapper);
@@ -405,6 +406,7 @@ public void remove(String type) {
                 return;
             }
             docMapper.close();
+            removeTypeListener(docMapper.parentFieldMapper());
             mappers = newMapBuilder(mappers).remove(type).map();
             removeObjectAndFieldMappers(docMapper);
             for (DocumentTypeListener typeListener : typeListeners) {