Skip to content

Commit

Permalink
Use a mapping snapshot for fetching nested docs (backport of elastic#…
Browse files Browse the repository at this point in the history 
…66877)

This uses the mapping snapshot that we built for the search phase
in elastic#66295 for fetching nested documents. This is simpler to reason about
because the mapping snapshot is immutable.
  • Loading branch information
@nik9000
nik9000 committed Jan 13, 2021
1 parent 8660e8d commit 520b10f
Show file tree
Hide file tree
Showing 13 changed files with 151 additions and 158 deletions.
12 changes: 5 additions & 7 deletions server/src/main/java/org/elasticsearch/index/cache/bitset/BitsetFilterCache.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexWarmer;
import org.elasticsearch.index.IndexWarmer.TerminationHandle;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.MappingLookup;
import org.elasticsearch.index.mapper.ObjectMapper;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.index.shard.ShardId;
Expand Down Expand Up @@ -236,12 +236,10 @@ public IndexWarmer.TerminationHandle warmReader(final IndexShard indexShard, fin

final Set<Query> warmUp = new HashSet<>();
final MapperService mapperService = indexShard.mapperService();
DocumentMapper docMapper = mapperService.documentMapper();
if (docMapper != null) {
if (docMapper.hasNestedObjects()) {
warmUp.add(Queries.newNonNestedFilter(indexSettings.getIndexVersionCreated()));
docMapper.getNestedParentMappers().stream().map(ObjectMapper::nestedTypeFilter).forEach(warmUp::add);
}
MappingLookup lookup = mapperService.mappingLookup();
if (lookup.hasNested()) {
warmUp.add(Queries.newNonNestedFilter(indexSettings.getIndexVersionCreated()));
lookup.getNestedParentMappers().stream().map(ObjectMapper::nestedTypeFilter).forEach(warmUp::add);
}

final CountDownLatch latch = new CountDownLatch(reader.leaves().size() * warmUp.size());
Expand Down
95 changes: 2 additions & 93 deletions server/src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,8 @@
import org.elasticsearch.index.mapper.MapperService.MergeReason;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Function;
Expand Down Expand Up @@ -179,10 +177,6 @@ public IndexFieldMapper IndexFieldMapper() {
return metadataMapper(IndexFieldMapper.class);
}

public boolean hasNestedObjects() {
return mappers().hasNested();
}

public MappingLookup mappers() {
return this.mappingLookup;
}
Expand All @@ -207,91 +201,6 @@ public ParsedDocument createNoopTombstoneDoc(String index, String reason) throws
return parsedDoc;
}

/**
* Given an object path, checks to see if any of its parents are non-nested objects
*/
public boolean hasNonNestedParent(String path) {
ObjectMapper mapper = mappers().objectMappers().get(path);
if (mapper == null) {
return false;
}
while (mapper != null) {
if (mapper.nested().isNested() == false) {
return true;
}
if (path.contains(".") == false) {
return false;
}
path = path.substring(0, path.lastIndexOf("."));
mapper = mappers().objectMappers().get(path);
}
return false;
}

/**
* Returns all nested object mappers
*/
public List<ObjectMapper> getNestedMappers() {
List<ObjectMapper> childMappers = new ArrayList<>();
for (ObjectMapper mapper : mappers().objectMappers().values()) {
if (mapper.nested().isNested() == false) {
continue;
}
childMappers.add(mapper);
}
return childMappers;
}

/**
* Returns all nested object mappers which contain further nested object mappers
*
* Used by BitSetProducerWarmer
*/
public List<ObjectMapper> getNestedParentMappers() {
List<ObjectMapper> parents = new ArrayList<>();
for (ObjectMapper mapper : mappers().objectMappers().values()) {
String nestedParentPath = getNestedParent(mapper.fullPath());
if (nestedParentPath == null) {
continue;
}
ObjectMapper parent = mappers().objectMappers().get(nestedParentPath);
if (parent.nested().isNested()) {
parents.add(parent);
}
}
return parents;
}

/**
* Given a nested object path, returns the path to its nested parent
*
* In particular, if a nested field `foo` contains an object field
* `bar.baz`, then calling this method with `foo.bar.baz` will return
* the path `foo`, skipping over the object-but-not-nested `foo.bar`
*/
public String getNestedParent(String path) {
ObjectMapper mapper = mappers().objectMappers().get(path);
if (mapper == null) {
return null;
}
if (path.contains(".") == false) {
return null;
}
do {
path = path.substring(0, path.lastIndexOf("."));
mapper = mappers().objectMappers().get(path);
if (mapper == null) {
return null;
}
if (mapper.nested().isNested()) {
return path;
}
if (path.contains(".") == false) {
return null;
}
} while(true);
}

public DocumentMapper merge(Mapping mapping, MergeReason reason) {
Mapping merged = this.mapping().merge(mapping, reason);
return new DocumentMapper(mappingLookup.getIndexSettings(), mappingLookup.getIndexAnalyzers(), documentParser, merged);
Expand All @@ -305,7 +214,7 @@ public void validate(IndexSettings settings, boolean checkLimits) {
+ "required for partitioned index [" + settings.getIndex().getName() + "]");
}
}
if (settings.getIndexSortConfig().hasIndexSort() && hasNestedObjects()) {
if (settings.getIndexSortConfig().hasIndexSort() && mappers().hasNested()) {
throw new IllegalArgumentException("cannot have nested fields when index sort is activated");
}
if (checkLimits) {
Expand All @@ -327,7 +236,7 @@ public String toString() {
", documentParser=" + documentParser +
", mappingLookup=" + mappingLookup +
", objectMappers=" + mappers().objectMappers() +
", hasNestedObjects=" + hasNestedObjects() +
", hasNestedObjects=" + mappingLookup.hasNested() +
", deleteTombstoneMetadataFieldMappers=" + Arrays.toString(deleteTombstoneMetadataFieldMappers) +
", noopTombstoneMetadataFieldMappers=" + Arrays.toString(noopTombstoneMetadataFieldMappers) +
'}';
Expand Down
85 changes: 85 additions & 0 deletions server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,4 +341,89 @@ IndexSettings getIndexSettings() {
IndexAnalyzers getIndexAnalyzers() {
return indexAnalyzers;
}

/**
* Given an object path, checks to see if any of its parents are non-nested objects
*/
public boolean hasNonNestedParent(String path) {
ObjectMapper mapper = objectMappers().get(path);
if (mapper == null) {
return false;
}
while (mapper != null) {
if (mapper.nested().isNested() == false) {
return true;
}
if (path.contains(".") == false) {
return false;
}
path = path.substring(0, path.lastIndexOf("."));
mapper = objectMappers().get(path);
}
return false;
}

/**
* Returns all nested object mappers
*/
public List<ObjectMapper> getNestedMappers() {
List<ObjectMapper> childMappers = new ArrayList<>();
for (ObjectMapper mapper : objectMappers().values()) {
if (mapper.nested().isNested() == false) {
continue;
}
childMappers.add(mapper);
}
return childMappers;
}

/**
* Returns all nested object mappers which contain further nested object mappers
*
* Used by BitSetProducerWarmer
*/
public List<ObjectMapper> getNestedParentMappers() {
List<ObjectMapper> parents = new ArrayList<>();
for (ObjectMapper mapper : objectMappers().values()) {
String nestedParentPath = getNestedParent(mapper.fullPath());
if (nestedParentPath == null) {
continue;
}
ObjectMapper parent = objectMappers().get(nestedParentPath);
if (parent.nested().isNested()) {
parents.add(parent);
}
}
return parents;
}

/**
* Given a nested object path, returns the path to its nested parent
*
* In particular, if a nested field `foo` contains an object field
* `bar.baz`, then calling this method with `foo.bar.baz` will return
* the path `foo`, skipping over the object-but-not-nested `foo.bar`
*/
public String getNestedParent(String path) {
ObjectMapper mapper = objectMappers().get(path);
if (mapper == null) {
return null;
}
if (path.contains(".") == false) {
return null;
}
do {
path = path.substring(0, path.lastIndexOf("."));
mapper = objectMappers().get(path);
if (mapper == null) {
return null;
}
if (mapper.nested().isNested()) {
return path;
}
if (path.contains(".") == false) {
return null;
}
} while(true);
}
}
5 changes: 5 additions & 0 deletions server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptFactory;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.search.NestedDocuments;
import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry;
import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.transport.RemoteClusterAware;
Expand Down Expand Up @@ -677,4 +678,8 @@ private static Map<String, MappedFieldType> parseRuntimeMappings(Map<String, Obj
public MappingLookup.CacheKey mappingCacheKey() {
return mappingLookup.cacheKey();
}

public NestedDocuments getNestedDocuments() {
return new NestedDocuments(mappingLookup, indexVersionCreated(), bitsetFilterCache::getBitSetProducer);
}
}
5 changes: 0 additions & 5 deletions server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -711,11 +711,6 @@ public QuerySearchResult queryResult() {
return queryResult;
}

@Override
public NestedDocuments getNestedDocuments() {
return new NestedDocuments(indexService.mapperService(), bitsetFilterCache()::getBitSetProducer);
}

@Override
public FetchPhase fetchPhase() {
return fetchPhase;
Expand Down
28 changes: 14 additions & 14 deletions server/src/main/java/org/elasticsearch/search/NestedDocuments.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
import org.apache.lucene.util.BitSet;
import org.elasticsearch.Version;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.mapper.MappingLookup;
import org.elasticsearch.index.mapper.ObjectMapper;

import java.io.IOException;
Expand All @@ -48,26 +48,26 @@ public class NestedDocuments {
private final Map<String, Weight> childObjectFilters = new HashMap<>();
private final Map<String, ObjectMapper> childObjectMappers = new HashMap<>();
private final BitSetProducer parentDocumentFilter;
private final MapperService mapperService;
private final Version version;
private final MappingLookup mappingLookup;
private final Version indexVersionCreated;

/**
* Create a new NestedDocuments object for an index
* @param mapperService the index's MapperService
* @param mappingLookup the index's mapping
* @param filterProducer a function to build BitSetProducers from filter queries
*/
public NestedDocuments(MapperService mapperService, Function<Query, BitSetProducer> filterProducer) {
this.mapperService = mapperService;
this.version = mapperService.getIndexSettings().getIndexVersionCreated();
if (mapperService.hasNested() == false) {
public NestedDocuments(MappingLookup mappingLookup, Version indexVersionCreated, Function<Query, BitSetProducer> filterProducer) {
this.mappingLookup = mappingLookup;
this.indexVersionCreated = indexVersionCreated;
if (mappingLookup.hasNested() == false) {
this.parentDocumentFilter = null;
} else {
this.parentDocumentFilter = filterProducer.apply(Queries.newNonNestedFilter(version));
for (ObjectMapper mapper : mapperService.documentMapper().getNestedParentMappers()) {
this.parentDocumentFilter = filterProducer.apply(Queries.newNonNestedFilter(indexVersionCreated));
for (ObjectMapper mapper : mappingLookup.getNestedParentMappers()) {
parentObjectFilters.put(mapper.name(),
filterProducer.apply(mapper.nestedTypeFilter()));
}
for (ObjectMapper mapper : mapperService.documentMapper().getNestedMappers()) {
for (ObjectMapper mapper : mappingLookup.getNestedMappers()) {
childObjectFilters.put(mapper.name(), null);
childObjectMappers.put(mapper.name(), mapper);
}
Expand Down Expand Up @@ -101,7 +101,7 @@ private Weight getNestedChildWeight(LeafReaderContext ctx, String path) throws I
* Given an object path, returns whether or not any of its parents are plain objects
*/
public boolean hasNonNestedParent(String path) {
return mapperService.documentMapper().hasNonNestedParent(path);
return mappingLookup.hasNonNestedParent(path);
}

private class HasNestedDocuments implements LeafNestedDocuments {
Expand Down Expand Up @@ -188,7 +188,7 @@ private SearchHit.NestedIdentity loadNestedIdentity() throws IOException {
int parentNameLength;
String path = findObjectPath(doc);
while (path != null) {
String parent = mapperService.documentMapper().getNestedParent(path);
String parent = mappingLookup.getNestedParent(path);
// We have to pull a new scorer for each document here, because we advance from
// the last parent which will be behind the doc
Scorer childScorer = getNestedChildWeight(ctx, path).scorer(ctx);
Expand All @@ -208,7 +208,7 @@ private SearchHit.NestedIdentity loadNestedIdentity() throws IOException {
}
int offset = 0;
DocIdSetIterator childIt = childScorer.iterator();
if (version.onOrAfter(Version.V_6_5_0)) {
if (indexVersionCreated.onOrAfter(Version.V_6_5_0)) {
/*
* Starts from the previous parent and finds the offset of the
* <code>nestedSubDocID</code> within the nested children. Nested documents
Expand Down
2 changes: 1 addition & 1 deletion server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ public void execute(SearchContext context) {
SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()];

List<FetchSubPhaseProcessor> processors = getProcessors(context.shardTarget(), fetchContext);
NestedDocuments nestedDocuments = context.getNestedDocuments();
NestedDocuments nestedDocuments = context.getQueryShardContext().getNestedDocuments();

int currentReaderIndex = -1;
LeafReaderContext currentReaderContext = null;
Expand Down
6 changes: 0 additions & 6 deletions server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import org.elasticsearch.index.query.ParsedQuery;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.search.NestedDocuments;
import org.elasticsearch.search.SearchExtBuilder;
import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.search.aggregations.SearchContextAggregations;
Expand Down Expand Up @@ -390,11 +389,6 @@ public FetchSearchResult fetchResult() {
return in.fetchResult();
}

@Override
public NestedDocuments getNestedDocuments() {
return in.getNestedDocuments();
}

@Override
public FetchPhase fetchPhase() {
return in.fetchPhase();
Expand Down
3 changes: 0 additions & 3 deletions server/src/main/java/org/elasticsearch/search/internal/SearchContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import org.elasticsearch.index.query.ParsedQuery;
import org.elasticsearch.index.query.QueryShardContext;
import org.elasticsearch.index.shard.IndexShard;
import org.elasticsearch.search.NestedDocuments;
import org.elasticsearch.search.RescoreDocIds;
import org.elasticsearch.search.SearchExtBuilder;
import org.elasticsearch.search.SearchShardTarget;
Expand Down Expand Up @@ -311,8 +310,6 @@ public final void assignRescoreDocIds(RescoreDocIds rescoreDocIds) {

public abstract QuerySearchResult queryResult();

public abstract NestedDocuments getNestedDocuments();

public abstract FetchPhase fetchPhase();

public abstract FetchSearchResult fetchResult();
Expand Down
Loading

0 comments on commit 520b10f

Please sign in to comment.