Skip to content

Commit

Permalink
parent/child: Move over to Lucene's join util and doc values by default
Browse files Browse the repository at this point in the history 
This a breaking change:
1) A parent type needs be marked as parent in the _parent field mapping of the parent type.
2) top_children query will be removed. The top_children query was somewhat an alternative to has_child when it came to speed, but it isn't accurate and wasn't always faster.

Indices created before 2.0 will use field data and the old way of executing queries, but indices created on or after 2.0 will use the Lucene join.

Closes elastic#6107
Closes elastic#6511
Closes elastic#8134
  • Loading branch information
@martijnvg
martijnvg committed May 10, 2015
1 parent acdd9a5 commit 7c11ecd
Show file tree
Hide file tree
Showing 9 changed files with 370 additions and 104 deletions.
4 changes: 4 additions & 0 deletions src/main/java/org/elasticsearch/index/AbstractIndexComponent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ public Index index() {
return this.index;
}

public Settings indexSettings() {
return indexSettings;
}

public String nodeName() {
return indexSettings.get("name", "");
}
Expand Down
78 changes: 67 additions & 11 deletions src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,11 @@

import com.carrotsearch.hppc.ObjectObjectOpenHashMap;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedSet;

import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.*;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BytesRef;
Expand All @@ -39,6 +34,7 @@
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PackedLongValues;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.breaker.CircuitBreaker;
import org.elasticsearch.common.collect.ImmutableOpenMap;
Expand Down Expand Up @@ -114,7 +110,58 @@ public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, Mu
}

@Override
public ParentChildAtomicFieldData loadDirect(LeafReaderContext context) throws Exception {
public AtomicParentChildFieldData load(LeafReaderContext context) {
if (Version.indexCreated(indexSettings).onOrAfter(Version.V_2_0_0)) {
try {
LeafReader reader = context.reader();
final NavigableSet<BytesRef> parentTypes;
synchronized (lock) {
parentTypes = ImmutableSortedSet.copyOf(BytesRef.getUTF8SortedAsUnicodeComparator(), this.parentTypes);
}
final ImmutableMap.Builder<String, SortedDocValues> builder = ImmutableMap.builder();
for (BytesRef parentType : parentTypes) {
SortedDocValues docValues = DocValues.getSorted(reader, ParentFieldMapper.joinField(parentType.utf8ToString()));
builder.put(parentType.utf8ToString(), docValues);
}
return new AbstractAtomicParentChildFieldData() {

private final ImmutableMap<String, SortedDocValues> typeToJoinField = builder.build();

@Override
public Set<String> types() {
return typeToJoinField.keySet();
}

@Override
public SortedDocValues getOrdinalsValues(String type) {
return typeToJoinField.get(type);
}

@Override
public long ramBytesUsed() {
// unknown
return 0;
}

@Override
public Collection<Accountable> getChildResources() {
return Collections.emptyList();
}

@Override
public void close() throws ElasticsearchException {
}
};
} catch (IOException e) {
throw new ElasticsearchException("Couldn't access _parent sorted doc values", e);
}
} else {
return super.load(context);
}
}

@Override
public AbstractAtomicParentChildFieldData loadDirect(LeafReaderContext context) throws Exception {
LeafReader reader = context.reader();
final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat(
"acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO
Expand Down Expand Up @@ -329,12 +376,14 @@ public IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) thro

long ramBytesUsed = 0;
final Map<String, OrdinalMapAndAtomicFieldData> perType = new HashMap<>();
final Map<String, OrdinalMap> ordinalMapPerType = new HashMap<>();
for (String type : parentTypes) {
final AtomicParentChildFieldData[] fieldData = new AtomicParentChildFieldData[indexReader.leaves().size()];
for (LeafReaderContext context : indexReader.leaves()) {
fieldData[context.ord] = load(context);
}
final OrdinalMap ordMap = buildOrdinalMap(fieldData, type);
ordinalMapPerType.put(type, ordMap);
ramBytesUsed += ordMap.ramBytesUsed();
perType.put(type, new OrdinalMapAndAtomicFieldData(ordMap, fieldData));
}
Expand All @@ -352,7 +401,7 @@ public IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) thro
);
}

return new GlobalFieldData(indexReader, fielddata, ramBytesUsed);
return new GlobalFieldData(indexReader, fielddata, ramBytesUsed, ordinalMapPerType);
}

private static class GlobalAtomicFieldData extends AbstractAtomicParentChildFieldData {
Expand Down Expand Up @@ -436,16 +485,18 @@ public void close() {

}

private class GlobalFieldData implements IndexParentChildFieldData, Accountable {
public class GlobalFieldData implements IndexParentChildFieldData, Accountable {

private final AtomicParentChildFieldData[] fielddata;
private final IndexReader reader;
private final long ramBytesUsed;
private final Map<String, OrdinalMap> ordinalMapPerType;

GlobalFieldData(IndexReader reader, AtomicParentChildFieldData[] fielddata, long ramBytesUsed) {
GlobalFieldData(IndexReader reader, AtomicParentChildFieldData[] fielddata, long ramBytesUsed, Map<String, OrdinalMap> ordinalMapPerType) {
this.reader = reader;
this.ramBytesUsed = ramBytesUsed;
this.fielddata = fielddata;
this.ordinalMapPerType = ordinalMapPerType;
}

@Override
Expand Down Expand Up @@ -512,6 +563,11 @@ public IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) thro
return loadGlobal(indexReader);
}

// TODO: Need to find a better way to expose the OrdinalMap...
public OrdinalMap getOrdinalMap(String type) {
return ordinalMapPerType.get(type);
}

}

}
73 changes: 57 additions & 16 deletions src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@
package org.elasticsearch.index.mapper.internal;

import com.google.common.base.Objects;

import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.queries.TermsQuery;
import org.apache.lucene.search.Query;
Expand All @@ -34,16 +34,9 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.loader.SettingsLoader;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.InternalMapper;
import org.elasticsearch.index.mapper.Mapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.index.mapper.MergeResult;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.RootMapper;
import org.elasticsearch.index.mapper.Uid;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
import org.elasticsearch.index.query.QueryParseContext;

Expand All @@ -57,6 +50,7 @@
import static org.elasticsearch.common.settings.ImmutableSettings.builder;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeMapValue;
import static org.elasticsearch.index.mapper.MapperBuilders.booleanField;
import static org.elasticsearch.index.mapper.MapperBuilders.parent;

/**
Expand Down Expand Up @@ -87,6 +81,7 @@ public static class Builder extends Mapper.Builder<Builder, ParentFieldMapper> {

protected String indexName;

private boolean parent;
private String type;
protected Settings fieldDataSettings;

Expand All @@ -96,6 +91,11 @@ public Builder() {
builder = this;
}

public Builder parent(boolean parent) {
this.parent = parent;
return this;
}

public Builder type(String type) {
this.type = type;
return builder;
Expand All @@ -108,10 +108,19 @@ public Builder fieldDataSettings(Settings settings) {

@Override
public ParentFieldMapper build(BuilderContext context) {
if (type == null) {
if (!parent && type == null) {
throw new MapperParsingException("Parent mapping must contain the parent type");
}
return new ParentFieldMapper(name, indexName, type, fieldDataSettings, context.indexSettings());
boolean docValues;
if (context.indexCreatedVersion().onOrAfter(Version.V_2_0_0)) {
docValues = true;
} else {
docValues = false;
// Even if _parent is set to true, set this to false on 1.x indices, because we don't want mixed parent
// types with and without docvalues
parent = false;
}
return new ParentFieldMapper(name, indexName, type, fieldDataSettings, context.indexSettings(), parent, docValues);
}
}

Expand All @@ -126,6 +135,9 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
if (fieldName.equals("type")) {
builder.type(fieldNode.toString());
iterator.remove();
} else if (fieldName.equals("parent")) {
builder.parent(XContentMapValues.nodeBooleanValue(fieldNode));
iterator.remove();
} else if (fieldName.equals("postings_format") && parserContext.indexVersionCreated().before(Version.V_2_0_0)) {
// ignore before 2.0, reject on and after 2.0
iterator.remove();
Expand All @@ -143,18 +155,22 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
}
}

private final boolean parent;
private final String type;
private final BytesRef typeAsBytes;
private final boolean docValuesJoin;

protected ParentFieldMapper(String name, String indexName, String type, @Nullable Settings fieldDataSettings, Settings indexSettings) {
protected ParentFieldMapper(String name, String indexName, String type, @Nullable Settings fieldDataSettings, Settings indexSettings, boolean parent, boolean docValuesJoin) {
super(new Names(name, indexName, indexName, name), Defaults.BOOST, new FieldType(Defaults.FIELD_TYPE), false,
Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER, null, null, fieldDataSettings, indexSettings);
this.type = type;
this.parent = parent;
this.docValuesJoin = docValuesJoin;
this.typeAsBytes = type == null ? null : new BytesRef(type);
}

public ParentFieldMapper(Settings indexSettings) {
this(Defaults.NAME, Defaults.NAME, null, null, indexSettings);
this(Defaults.NAME, Defaults.NAME, null, null, indexSettings, false, Version.indexCreated(indexSettings).onOrAfter(Version.V_2_0_0));
this.fieldDataType = new FieldDataType("_parent", settingsBuilder().put(Loading.KEY, Loading.LAZY_VALUE));
}

Expand Down Expand Up @@ -188,6 +204,11 @@ public boolean includeInObject() {

@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
if (parent) {
assert docValuesJoin;
fields.add(createJoinField(context.type(), context.id()));
}

if (!active()) {
return;
}
Expand All @@ -197,6 +218,9 @@ protected void parseCreateField(ParseContext context, List<Field> fields) throws
String parentId = context.parser().text();
context.sourceToParse().parent(parentId);
fields.add(new Field(names.indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType));
if (docValuesJoin) {
fields.add(createJoinField(type, parentId));
}
} else {
// otherwise, we are running it post processing of the xcontent
String parsedParentId = context.doc().get(Defaults.NAME);
Expand All @@ -208,6 +232,9 @@ protected void parseCreateField(ParseContext context, List<Field> fields) throws
}
// we did not add it in the parsing phase, add it now
fields.add(new Field(names.indexName(), Uid.createUid(context.stringBuilder(), type, parentId), fieldType));
if (docValuesJoin) {
fields.add(createJoinField(type, parentId));
}
} else if (parentId != null && !parsedParentId.equals(Uid.createUid(context.stringBuilder(), type, parentId))) {
throw new MapperParsingException("Parent id mismatch, document value is [" + Uid.createUid(parsedParentId).id() + "], while external value is [" + parentId + "]");
}
Expand All @@ -216,6 +243,15 @@ protected void parseCreateField(ParseContext context, List<Field> fields) throws
// we have parent mapping, yet no value was set, ignore it...
}

private SortedDocValuesField createJoinField(String parentType, String id) {
String joinField = joinField(parentType);
return new SortedDocValuesField(joinField, new BytesRef(id));
}

public static String joinField(String parentType) {
return ParentFieldMapper.NAME + "#" + parentType;
}

@Override
public Uid value(Object value) {
if (value == null) {
Expand Down Expand Up @@ -304,13 +340,18 @@ protected String contentType() {

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
if (!active()) {
if (!parent && !active()) {
return builder;
}
boolean includeDefaults = params.paramAsBoolean("include_defaults", false);

builder.startObject(CONTENT_TYPE);
builder.field("type", type);
if (type != null) {
builder.field("type", type);
}
if (parent) {
builder.field("parent", parent);
}
if (customFieldDataSettings != null) {
builder.field("fielddata", (Map) customFieldDataSettings.getAsMap());
} else if (includeDefaults) {
Expand Down
Loading

0 comments on commit 7c11ecd

Please sign in to comment.