Skip to content

Commit

Permalink
parent/child: Move over to Lucene's join util and doc values by default
Browse files Browse the repository at this point in the history 
This a breaking change:
1) A parent type needs be marked as parent in the _parent field mapping of the parent type.
2) The has_child and has_parent queries can't be used in index aliases any more, because during query parse time it requires the search context to be set. During normal _search api usage this is the case, but not when adding an index alias.

Indices created before 2.0 will use field data and the old way of executing queries, but indices created on or after 2.0 will use the Lucene join and encode the parent/child relation at index time in a special join doc values field.

Closes elastic#6107
Closes elastic#6511
Closes elastic#8134
  • Loading branch information
@martijnvg
martijnvg committed May 10, 2015
1 parent acdd9a5 commit e7884d8
Show file tree
Hide file tree
Showing 15 changed files with 547 additions and 293 deletions.
3 changes: 1 addition & 2 deletions src/main/java/org/elasticsearch/action/index/IndexRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
package org.elasticsearch.action.index;

import com.google.common.base.Charsets;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchGenerationException;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.*;
Expand Down Expand Up @@ -606,7 +605,7 @@ public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, bool
throw new RoutingMissingException(concreteIndex, type, id);
}

if (parent != null && !mappingMd.hasParentField()) {
if (parent != null && !mappingMd.requiresParentId()) {
throw new IllegalArgumentException("Can't specify parent if no parent field has been configured");
}
} else {
Expand Down
20 changes: 9 additions & 11 deletions src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

package org.elasticsearch.cluster.metadata;

import com.google.common.collect.Maps;
import org.elasticsearch.Version;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.cluster.AbstractDiffable;
Expand All @@ -40,7 +39,6 @@

import java.io.IOException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

import static com.google.common.collect.Maps.newHashMap;
Expand Down Expand Up @@ -285,7 +283,7 @@ public int hashCode() {
private Id id;
private Routing routing;
private Timestamp timestamp;
private boolean hasParentField;
private boolean requiresParentId;

public MappingMetaData(DocumentMapper docMapper) {
this.type = docMapper.type();
Expand All @@ -295,7 +293,7 @@ public MappingMetaData(DocumentMapper docMapper) {
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(),
docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp(),
docMapper.timestampFieldMapper().ignoreMissing());
this.hasParentField = docMapper.parentFieldMapper().active();
this.requiresParentId = docMapper.parentFieldMapper().isChild();
}

public MappingMetaData(CompressedString mapping) throws IOException {
Expand Down Expand Up @@ -391,19 +389,19 @@ private void initMappers(Map<String, Object> withoutType) {
this.timestamp = Timestamp.EMPTY;
}
if (withoutType.containsKey("_parent")) {
this.hasParentField = true;
this.requiresParentId = true;
} else {
this.hasParentField = false;
this.requiresParentId = false;
}
}

public MappingMetaData(String type, CompressedString source, Id id, Routing routing, Timestamp timestamp, boolean hasParentField) {
public MappingMetaData(String type, CompressedString source, Id id, Routing routing, Timestamp timestamp, boolean requiresParentId) {
this.type = type;
this.source = source;
this.id = id;
this.routing = routing;
this.timestamp = timestamp;
this.hasParentField = hasParentField;
this.requiresParentId = requiresParentId;
}

void updateDefaultMapping(MappingMetaData defaultMapping) {
Expand All @@ -426,8 +424,8 @@ public CompressedString source() {
return this.source;
}

public boolean hasParentField() {
return hasParentField;
public boolean requiresParentId() {
return requiresParentId;
}

/**
Expand Down Expand Up @@ -575,7 +573,7 @@ public void writeTo(StreamOutput out) throws IOException {
if (out.getVersion().onOrAfter(Version.V_1_5_0)) {
out.writeOptionalBoolean(timestamp().ignoreMissing());
}
out.writeBoolean(hasParentField());
out.writeBoolean(requiresParentId());
}

@Override
Expand Down
4 changes: 4 additions & 0 deletions src/main/java/org/elasticsearch/index/AbstractIndexComponent.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ public Index index() {
return this.index;
}

public Settings indexSettings() {
return indexSettings;
}

public String nodeName() {
return indexSettings.get("name", "");
}
Expand Down
93 changes: 80 additions & 13 deletions src/main/java/org/elasticsearch/index/fielddata/plain/ParentChildIndexFieldData.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,11 @@

import com.carrotsearch.hppc.ObjectObjectOpenHashMap;
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedSet;

import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.*;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.BytesRef;
Expand All @@ -39,6 +34,7 @@
import org.apache.lucene.util.packed.PackedInts;
import org.apache.lucene.util.packed.PackedLongValues;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.breaker.CircuitBreaker;
import org.elasticsearch.common.collect.ImmutableOpenMap;
Expand Down Expand Up @@ -114,7 +110,58 @@ public XFieldComparatorSource comparatorSource(@Nullable Object missingValue, Mu
}

@Override
public ParentChildAtomicFieldData loadDirect(LeafReaderContext context) throws Exception {
public AtomicParentChildFieldData load(LeafReaderContext context) {
if (Version.indexCreated(indexSettings).onOrAfter(Version.V_2_0_0)) {
try {
LeafReader reader = context.reader();
final NavigableSet<BytesRef> parentTypes;
synchronized (lock) {
parentTypes = ImmutableSortedSet.copyOf(BytesRef.getUTF8SortedAsUnicodeComparator(), this.parentTypes);
}
final ImmutableMap.Builder<String, SortedDocValues> builder = ImmutableMap.builder();
for (BytesRef parentType : parentTypes) {
SortedDocValues docValues = DocValues.getSorted(reader, ParentFieldMapper.joinField(parentType.utf8ToString()));
builder.put(parentType.utf8ToString(), docValues);
}
return new AbstractAtomicParentChildFieldData() {

private final ImmutableMap<String, SortedDocValues> typeToJoinField = builder.build();

@Override
public Set<String> types() {
return typeToJoinField.keySet();
}

@Override
public SortedDocValues getOrdinalsValues(String type) {
return typeToJoinField.get(type);
}

@Override
public long ramBytesUsed() {
// unknown
return 0;
}

@Override
public Collection<Accountable> getChildResources() {
return Collections.emptyList();
}

@Override
public void close() throws ElasticsearchException {
}
};
} catch (IOException e) {
throw new ElasticsearchException("Couldn't access _parent sorted doc values", e);
}
} else {
return super.load(context);
}
}

@Override
public AbstractAtomicParentChildFieldData loadDirect(LeafReaderContext context) throws Exception {
LeafReader reader = context.reader();
final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat(
"acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO
Expand Down Expand Up @@ -189,7 +236,7 @@ public ParentChildAtomicFieldData loadDirect(LeafReaderContext context) throws E
public void beforeCreate(DocumentMapper mapper) {
synchronized (lock) {
ParentFieldMapper parentFieldMapper = mapper.parentFieldMapper();
if (parentFieldMapper.active()) {
if (parentFieldMapper.isChild()) {
// A _parent field can never be added to an existing mapping, so a _parent field either exists on
// a new created or doesn't exists. This is why we can update the known parent types via DocumentTypeListener
if (parentTypes.add(new BytesRef(parentFieldMapper.type()))) {
Expand All @@ -203,7 +250,7 @@ public void beforeCreate(DocumentMapper mapper) {
public void afterRemove(DocumentMapper mapper) {
synchronized (lock) {
ParentFieldMapper parentFieldMapper = mapper.parentFieldMapper();
if (parentFieldMapper.active()) {
if (parentFieldMapper.isChild()) {
parentTypes.remove(new BytesRef(parentFieldMapper.type()));
}
}
Expand Down Expand Up @@ -329,12 +376,14 @@ public IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) thro

long ramBytesUsed = 0;
final Map<String, OrdinalMapAndAtomicFieldData> perType = new HashMap<>();
final Map<String, OrdinalMap> ordinalMapPerType = new HashMap<>();
for (String type : parentTypes) {
final AtomicParentChildFieldData[] fieldData = new AtomicParentChildFieldData[indexReader.leaves().size()];
for (LeafReaderContext context : indexReader.leaves()) {
fieldData[context.ord] = load(context);
}
final OrdinalMap ordMap = buildOrdinalMap(fieldData, type);
ordinalMapPerType.put(type, ordMap);
ramBytesUsed += ordMap.ramBytesUsed();
perType.put(type, new OrdinalMapAndAtomicFieldData(ordMap, fieldData));
}
Expand All @@ -352,7 +401,7 @@ public IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) thro
);
}

return new GlobalFieldData(indexReader, fielddata, ramBytesUsed);
return new GlobalFieldData(indexReader, fielddata, ramBytesUsed, ordinalMapPerType);
}

private static class GlobalAtomicFieldData extends AbstractAtomicParentChildFieldData {
Expand Down Expand Up @@ -436,16 +485,18 @@ public void close() {

}

private class GlobalFieldData implements IndexParentChildFieldData, Accountable {
public class GlobalFieldData implements IndexParentChildFieldData, Accountable {

private final AtomicParentChildFieldData[] fielddata;
private final IndexReader reader;
private final long ramBytesUsed;
private final Map<String, OrdinalMap> ordinalMapPerType;

GlobalFieldData(IndexReader reader, AtomicParentChildFieldData[] fielddata, long ramBytesUsed) {
GlobalFieldData(IndexReader reader, AtomicParentChildFieldData[] fielddata, long ramBytesUsed, Map<String, OrdinalMap> ordinalMapPerType) {
this.reader = reader;
this.ramBytesUsed = ramBytesUsed;
this.fielddata = fielddata;
this.ordinalMapPerType = ordinalMapPerType;
}

@Override
Expand Down Expand Up @@ -514,4 +565,20 @@ public IndexParentChildFieldData localGlobalDirect(IndexReader indexReader) thro

}

/**
* Returns the global ordinal map for the specified type
*/
// TODO: OrdinalMap isn't expose in the field data framework, because it is an implementation detail.
// However the JoinUtil works directly with OrdinalMap, so this is a hack to get access to OrdinalMap
// I don't think we should expose OrdinalMap in IndexFieldData, because only parent/child relies on it and for the
// rest of the code OrdinalMap is an implementation detail, but maybe we can expose it in IndexParentChildFieldData interface?
public static MultiDocValues.OrdinalMap getOrdinalMap(IndexParentChildFieldData indexParentChildFieldData, String type) {
if (indexParentChildFieldData instanceof ParentChildIndexFieldData.GlobalFieldData) {
return ((ParentChildIndexFieldData.GlobalFieldData) indexParentChildFieldData).ordinalMapPerType.get(type);
} else {
// one segment, local ordinals are global
return null;
}
}

}
2 changes: 1 addition & 1 deletion src/main/java/org/elasticsearch/index/get/ShardGetService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,7 +225,7 @@ public GetResult innerGet(String type, String id, String[] gFields, boolean real
Object value = null;
if (field.equals(RoutingFieldMapper.NAME) && docMapper.routingFieldMapper().fieldType().stored()) {
value = source.routing;
} else if (field.equals(ParentFieldMapper.NAME) && docMapper.parentFieldMapper().active() && docMapper.parentFieldMapper().fieldType().stored()) {
} else if (field.equals(ParentFieldMapper.NAME) && docMapper.parentFieldMapper().isChild() && docMapper.parentFieldMapper().fieldType().stored()) {
value = source.parent;
} else if (field.equals(TimestampFieldMapper.NAME) && docMapper.timestampFieldMapper().fieldType().stored()) {
value = source.timestamp;
Expand Down
3 changes: 1 addition & 2 deletions src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,6 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;

/**
*
Expand Down Expand Up @@ -202,7 +201,7 @@ public DocumentMapper(MapperService mapperService, String index, @Nullable Setti

this.typeFilter = typeMapper().termQuery(type, null);

if (rootMapper(ParentFieldMapper.class).active()) {
if (rootMapper(ParentFieldMapper.class).isChild()) {
// mark the routing field mapper as required
rootMapper(RoutingFieldMapper.class).markAsRequired();
}
Expand Down
Loading

0 comments on commit e7884d8

Please sign in to comment.