Skip to content

Commit

Permalink
Close over lookup
Browse files Browse the repository at this point in the history
  • Loading branch information
nik9000 committed Sep 8, 2020
1 parent 6a6584b commit 4e5bd65
Show file tree
Hide file tree
Showing 36 changed files with 74 additions and 77 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue()) {
@Override
protected Float parseSourceValue(Object value) {
return objectToFloat(value);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue()) {
@Override
protected Object parseSourceValue(Object value) {
return value;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -401,7 +401,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue()) {
@Override
protected Double parseSourceValue(Object value) {
double doubleValue;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}

return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue(), nullValue) {
@Override
protected String parseSourceValue(Object value) {
return value.toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue()) {
@Override
protected Object parseSourceValue(Object value) {
return value;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -371,7 +371,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue()) {
@Override
protected Object parseSourceValue(Object value) {
return value;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -740,7 +740,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}

return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue(), nullValue) {
@Override
protected String parseSourceValue(Object value) {
String keywordValue = value.toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -594,7 +594,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue()) {
@Override
protected Object parseSourceValue(Object value) {
return value.toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue()) {
@Override
protected String parseSourceValue(Object value) {
return value.toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
Parser<Parsed> geometryParser = mappedFieldType.geometryParser();
Function<Object, Object> valueParser = value -> geometryParser.parseAndFormatObject(value, this, geoFormat);

return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue()) {
@Override
protected Object parseSourceValue(Object value) {
return valueParser.apply(value);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue()) {
@Override
protected Object parseSourceValue(Object value) {
return value;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}

return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue(), nullValue) {
@Override
protected Boolean parseSourceValue(Object value) {
if (value instanceof Boolean) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -546,7 +546,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}

return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue()) {
@Override
protected List<?> parseSourceValue(Object value) {
if (value instanceof List) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -612,7 +612,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale())
: defaultFormatter;

return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValueAsString) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue(), nullValueAsString) {
@Override
public String parseSourceValue(Object value) {
String date = value.toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.apache.lucene.index.LeafReaderContext;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.search.DocValueFormat;
import org.elasticsearch.search.lookup.SourceLookup;

import java.io.IOException;
import java.util.ArrayList;
Expand All @@ -46,8 +45,8 @@ public void setNextReader(LeafReaderContext context) {
}

@Override
public List<Object> fetchValues(SourceLookup lookup) throws IOException {
if (false == leaf.advanceExact(lookup.docId())) {
public List<Object> fetchValues(int docId) throws IOException {
if (false == leaf.advanceExact(docId)) {
return List.of();
}
List<Object> result = new ArrayList<Object>(leaf.docValueCount());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue(), nullValue) {
@Override
protected Object parseSourceValue(Object value) {
InetAddress address;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -398,7 +398,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}

return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue(), nullValue) {
@Override
protected String parseSourceValue(Object value) {
String keywordValue = value.toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1094,7 +1094,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}

return new SourceValueFetcher(name(), mapperService, parsesArrayValue(), nullValue) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue(), nullValue) {
@Override
protected Object parseSourceValue(Object value) {
if (value.equals("")) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -366,7 +366,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale())
: defaultFormatter;

return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue()) {

@Override
@SuppressWarnings("unchecked")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,26 +35,38 @@
*/
public abstract class SourceValueFetcher implements ValueFetcher {
private final Set<String> sourcePaths;
private final SourceLookup lookup;
private final @Nullable Object nullValue;
private final boolean parsesArrayValue;

public SourceValueFetcher(String fieldName, MapperService mapperService, boolean parsesArrayValue) {
this(fieldName, mapperService, parsesArrayValue, null);
public SourceValueFetcher(String fieldName, MapperService mapperService, SourceLookup lookup, boolean parsesArrayValue) {
this(fieldName, mapperService, lookup, parsesArrayValue, null);
}

/**
* @param fieldName The name of the field.
* @param parsesArrayValue Whether the fetcher handles array values during document parsing.
* @param nullValue A optional substitute value if the _source value is 'null'.
*/
public SourceValueFetcher(String fieldName, MapperService mapperService, boolean parsesArrayValue, Object nullValue) {
public SourceValueFetcher(
String fieldName,
MapperService mapperService,
SourceLookup lookup,
boolean parsesArrayValue,
Object nullValue
) {
this.sourcePaths = mapperService.sourcePath(fieldName);
this.lookup = lookup;
this.nullValue = nullValue;
this.parsesArrayValue = parsesArrayValue;
}

@Override
public List<Object> fetchValues(SourceLookup lookup) {
public List<Object> fetchValues(int docId) {
/*
* Pulls the values from the source, parses them and then them
* using parseSourceValue.
*/
List<Object> values = new ArrayList<>();
for (String path : sourcePaths) {
Object sourceValue = lookup.extractValue(path, nullValue);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -841,7 +841,7 @@ public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searc
if (format != null) {
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
}
return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue()) {
@Override
protected Object parseSourceValue(Object value) {
return value.toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

import org.apache.lucene.index.LeafReaderContext;
import org.elasticsearch.search.fetch.subphase.FetchFieldsPhase;
import org.elasticsearch.search.lookup.SourceLookup;

import java.io.IOException;
import java.util.List;
Expand All @@ -32,19 +31,16 @@
*/
public interface ValueFetcher {
/**
* Given access to a document's _source, return this field's values.
*
* In addition to pulling out the values, they will be parsed into a standard form.
* For example numeric field mappers make sure to parse the source value into a number
* of the right type.
*
* Fetch this field's values, converting them into a "standard form" and applying
* any formats configured when the {@linkplain ValueFetcher} was built.
* <p>
* Note that for array values, the order in which values are returned is undefined and
* should not be relied on.
*
* @param lookup a lookup structure over the document's source.
* @param docId the document's id.
* @return a list a standardized field values.
*/
List<Object> fetchValues(SourceLookup lookup) throws IOException;
List<Object> fetchValues(int docId) throws IOException;

/**
* Update the leaf reader used to fetch values.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ public void process(HitContext hit) throws IOException {
// docValues fields will still be document fields, and put under "fields" section of a hit.
hit.hit().setDocumentField(f.getKey(), hitField);
}
hitField.getValues().addAll(f.getValue().fetchValues(hit.sourceLookup()));
hitField.getValues().addAll(f.getValue().fetchValues(hit.docId()));
}
}
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import org.elasticsearch.search.fetch.FetchSubPhaseProcessor;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.search.lookup.SourceLookup;

import java.io.IOException;
import java.util.HashSet;
Expand Down Expand Up @@ -60,10 +59,8 @@ public void setNextReader(LeafReaderContext readerContext) {
@Override
public void process(HitContext hitContext) throws IOException {
SearchHit hit = hitContext.hit();
SourceLookup sourceLookup = hitContext.sourceLookup();

Set<String> ignoredFields = getIgnoredFields(hit);
Map<String, DocumentField> documentFields = retriever.retrieve(sourceLookup, ignoredFields);
Map<String, DocumentField> documentFields = retriever.retrieve(hit.docId(), ignoredFields);
for (Map.Entry<String, DocumentField> entry : documentFields.entrySet()) {
hit.setDocumentField(entry.getKey(), entry.getValue());
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
import org.elasticsearch.index.mapper.MappingLookup;
import org.elasticsearch.index.mapper.ValueFetcher;
import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.search.lookup.SourceLookup;

import java.io.IOException;
import java.util.ArrayList;
Expand Down Expand Up @@ -83,7 +82,7 @@ private FieldValueRetriever(List<FieldContext> fieldContexts) {
this.fieldContexts = fieldContexts;
}

public Map<String, DocumentField> retrieve(SourceLookup sourceLookup, Set<String> ignoredFields) throws IOException {
public Map<String, DocumentField> retrieve(int docId, Set<String> ignoredFields) throws IOException {
Map<String, DocumentField> documentFields = new HashMap<>();
for (FieldContext context : fieldContexts) {
String field = context.fieldName;
Expand All @@ -92,7 +91,7 @@ public Map<String, DocumentField> retrieve(SourceLookup sourceLookup, Set<String
}

ValueFetcher valueFetcher = context.valueFetcher;
List<Object> parsedValues = valueFetcher.fetchValues(sourceLookup);
List<Object> parsedValues = valueFetcher.fetchValues(docId);

if (parsedValues.isEmpty() == false) {
documentFields.put(field, new DocumentField(field, parsedValues));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ protected void parseCreateField(ParseContext context) throws IOException {

@Override
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue()) {
@Override
protected Object parseSourceValue(Object value) {
return value;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ protected void parseCreateField(ParseContext context) throws IOException {

@Override
public ValueFetcher valueFetcher(MapperService mapperService, SearchLookup searchLookup, String format) {
return new SourceValueFetcher(name(), mapperService, parsesArrayValue()) {
return new SourceValueFetcher(name(), mapperService, searchLookup.source(), parsesArrayValue()) {
@Override
protected String parseSourceValue(Object value) {
return value.toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.search.lookup.SourceLookup;
import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.test.ESSingleNodeTestCase;

import java.io.IOException;
Expand Down Expand Up @@ -370,11 +370,11 @@ private Map<String, DocumentField> retrieveFields(MapperService mapperService, X
private Map<String, DocumentField> retrieveFields(MapperService mapperService, XContentBuilder source, List<FieldAndFormat> fields)
throws IOException {

SourceLookup sourceLookup = new SourceLookup();
sourceLookup.setSource(BytesReference.bytes(source));
SearchLookup lookup = new SearchLookup(mapperService, null);
lookup.source().setSource(BytesReference.bytes(source));

FieldValueRetriever fetchFieldsLookup = FieldValueRetriever.create(mapperService, null, fields);
return fetchFieldsLookup.retrieve(sourceLookup, Set.of());
FieldValueRetriever fetchFieldsLookup = FieldValueRetriever.create(mapperService, lookup, fields);
return fetchFieldsLookup.retrieve(0, Set.of());
}

public MapperService createMapperService() throws IOException {
Expand Down
Loading

0 comments on commit 4e5bd65

Please sign in to comment.