diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java index aa4daf3179adb..e193e63452b56 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java @@ -64,7 +64,7 @@ public void setUp() throws Exception { when(fieldData.load(anyObject())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(mapperService, ignored -> fieldData); + lookup = new SearchLookup(mapperService, (ignored, lookupSource) -> fieldData); } private FieldScript.LeafFactory compile(String expression) { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java index 007f4d608c82d..3bb2aa7b15924 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java @@ -64,7 +64,7 @@ public void setUp() throws Exception { when(fieldData.load(anyObject())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(mapperService, ignored -> fieldData); + lookup = new SearchLookup(mapperService, (ignored, lookupSource) -> fieldData); } private NumberSortScript.LeafFactory compile(String expression) { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java index fd279a9fa14e9..6d3f35dc013d8 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java @@ -64,7 +64,7 @@ public void setUp() throws Exception { when(fieldData.load(anyObject())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(mapperService, ignored -> fieldData); + lookup = new SearchLookup(mapperService, (ignored, lookupSource) -> fieldData); } private TermsSetQueryScript.LeafFactory compile(String expression) { diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java index 92816426eaf47..fafa04581d63b 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java @@ -47,7 +47,7 @@ public void testNeedsScores() { PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts); QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null); - SearchLookup lookup = new SearchLookup(index.mapperService(), shardContext::getForField); + SearchLookup lookup = shardContext.lookup(); NumberSortScript.Factory factory = service.compile(null, "1.2", NumberSortScript.CONTEXT, Collections.emptyMap()); NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), lookup); diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java index 7437ac3ec02e1..f73dca1c11b39 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java @@ -183,11 +183,11 @@ private Float objectToFloat(Object value) { } @Override - protected Float parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return objectToFloat(value); + return ValueFetcher.fromSource(this, lookup, this::objectToFloat); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java index 478f8b73e249d..dd8e4751ea615 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java @@ -162,11 +162,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return value; + return ValueFetcher.fromSource(this, lookup, value -> value); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java index 4f2c81e239bb1..7841ed0947e4d 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java @@ -482,23 +482,24 @@ private static double objectToDouble(Object value) { } @Override - protected Double parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - - double doubleValue; - if (value.equals("")) { - if (nullValue == null) { - return null; + return ValueFetcher.fromSource(this, lookup, value -> { + double doubleValue; + if (value.equals("")) { + if (nullValue == null) { + return null; + } + doubleValue = nullValue; + } else { + doubleValue = objectToDouble(value); } - doubleValue = nullValue; - } else { - doubleValue = objectToDouble(value); - } - double scalingFactor = fieldType().getScalingFactor(); - return Math.round(doubleValue * scalingFactor) / scalingFactor; + double scalingFactor = fieldType().getScalingFactor(); + return Math.round(doubleValue * scalingFactor) / scalingFactor; + }); } private static class ScaledFloatIndexFieldData extends IndexNumericFieldData { @@ -624,5 +625,19 @@ public int docValueCount() { } } + @Override + public ValueFetcher.LeafValueFetcher buildFetcher(DocValueFormat format) { + SortedNumericDoubleValues doubles = getDoubleValues(); + return docId -> { + if (false == doubles.advanceExact(docId)) { + return List.of(); + } + List result = new ArrayList<>(doubles.docValueCount()); + for (int i = 0, count = doubles.docValueCount(); i < count; ++i) { + result.add(format.format(doubles.nextValue())); + } + return result; + }; + } } } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java index 48042c2d0ee2b..54c91fd6d5a01 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java @@ -54,6 +54,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.index.similarity.SimilarityService; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -419,7 +420,7 @@ protected void parseCreateField(ParseContext context) { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { throw new UnsupportedOperationException(); } @@ -465,7 +466,7 @@ protected void mergeOptions(FieldMapper other, List conflicts) { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { throw new UnsupportedOperationException(); } @@ -588,11 +589,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return value.toString(); + return ValueFetcher.fromSource(this, lookup, Object::toString); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java index 2b3041e0dc3a2..9925cde774a2f 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java @@ -25,6 +25,7 @@ import org.apache.lucene.document.FieldType; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Iterator; @@ -159,12 +160,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - - return value.toString(); + return ValueFetcher.fromSource(this, lookup, Object::toString); } /** diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java index 30df4394b7418..30961c979c49f 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java @@ -41,6 +41,9 @@ import java.util.Collection; import java.util.Set; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.hasSize; + public class RankFeatureFieldMapperTests extends FieldMapperTestCase { IndexService indexService; @@ -189,12 +192,14 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep e.getCause().getMessage()); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); RankFeatureFieldMapper mapper = new RankFeatureFieldMapper.Builder("field").build(context); - assertEquals(3.14f, mapper.parseSourceValue(3.14, null), 0.0001); - assertEquals(42.9f, mapper.parseSourceValue("42.9", null), 0.0001); + assertThat(fetchFromSource(mapper, null, 3.14), hasSize(1)); + assertThat(((Number) fetchFromSource(mapper, null, 3.14).get(0)).doubleValue(), closeTo(3.14, 0.0001)); + assertThat(fetchFromSource(mapper, null, "42.9"), hasSize(1)); + assertThat(((Number) fetchFromSource(mapper, null, "42.9").get(0)).doubleValue(), closeTo(42.9, 0.0001)); } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java index 6758750f66581..7bfe04c9d4f0a 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; @@ -43,7 +42,10 @@ import java.util.List; import java.util.Set; +import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; public class ScaledFloatFieldMapperTests extends FieldMapperTestCase { @@ -403,25 +405,27 @@ public void testMeta() throws Exception { assertEquals(mapping3, mapper.mappingSource().toString()); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); ScaledFloatFieldMapper mapper = new ScaledFloatFieldMapper.Builder("field") .scalingFactor(100) .build(context); - assertEquals(3.14, mapper.parseSourceValue(3.1415926, null), 0.00001); - assertEquals(3.14, mapper.parseSourceValue("3.1415", null), 0.00001); - assertNull(mapper.parseSourceValue("", null)); + assertThat(fetchFromSource(mapper, null, 3.1415926), hasSize(1)); + assertThat(((Double) fetchFromSource(mapper, null, 3.1415926).get(0)).doubleValue(), closeTo(3.14, 0.00001)); + assertThat(fetchFromSource(mapper, null, "3.1415"), hasSize(1)); + assertThat(((Double) fetchFromSource(mapper, null, "3.1415").get(0)).doubleValue(), closeTo(3.14, 0.00001)); + assertThat(fetchFromSource(mapper, null, ""), equalTo(List.of())); + assertThat(fetchFromSource(mapper, null, null), equalTo(List.of())); ScaledFloatFieldMapper nullValueMapper = new ScaledFloatFieldMapper.Builder("field") .scalingFactor(100) .nullValue(2.71) .build(context); - assertEquals(2.71, nullValueMapper.parseSourceValue("", null), 0.00001); - - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(Collections.singletonMap("field", null)); - assertEquals(List.of(2.71), nullValueMapper.lookupValues(sourceLookup, null)); + assertThat(fetchFromSource(nullValueMapper, null, ""), hasSize(1)); + assertThat(((Double) fetchFromSource(nullValueMapper, null, "").get(0)).doubleValue(), closeTo(2.71, 0.00001)); + assertThat(fetchFromSource(nullValueMapper, null, null), hasSize(1)); + assertThat(((Double) fetchFromSource(nullValueMapper, null, null).get(0)).doubleValue(), closeTo(2.71, 0.00001)); } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java index 94e72a37eba9e..349db8a3f8b12 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; @@ -138,7 +139,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 5f6af2e1085a5..866ad4dd79167 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -39,6 +39,7 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; @@ -188,7 +189,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 729e34dbcb9fe..8480d38a38370 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -44,6 +44,7 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; @@ -350,11 +351,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return value; + return ValueFetcher.fromSource(this, lookup, v -> v); } @Override diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java index 895c792b53ded..045d058ec088b 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java @@ -67,6 +67,7 @@ import org.elasticsearch.index.mapper.RangeFieldMapper; import org.elasticsearch.index.mapper.RangeType; import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.BoostingQueryBuilder; import org.elasticsearch.index.query.ConstantScoreQueryBuilder; @@ -76,6 +77,7 @@ import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -368,11 +370,11 @@ public void parse(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return value; + return ValueFetcher.fromSource(this, lookup, value -> value); } static void createQueryBuilderField(Version indexVersion, BinaryFieldMapper qbField, diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java index caedf4610e364..72eebe9e626ef 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java @@ -34,6 +34,7 @@ import org.elasticsearch.search.fetch.subphase.highlight.Highlighter; import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.ArrayList; @@ -76,7 +77,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept PercolateQuery.QueryStore queryStore = percolateQuery.getQueryStore(); LeafReaderContext percolatorLeafReaderContext = percolatorIndexSearcher.getIndexReader().leaves().get(0); - FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(); + FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(new SourceLookup()); for (SearchHit hit : hits) { LeafReaderContext ctx = ctxs.get(ReaderUtil.subIndex(hit.docId(), ctxs)); diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java index 89d59d0400eee..d0a980898a37d 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java @@ -23,6 +23,7 @@ import com.ibm.icu.text.RawCollationKey; import com.ibm.icu.text.RuleBasedCollator; import com.ibm.icu.util.ULocale; + import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; @@ -740,15 +741,17 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - String keywordValue = value.toString(); - if (keywordValue.length() > ignoreAbove) { - return null; - } - return keywordValue; + return ValueFetcher.fromSource(this, lookup, value -> { + String keywordValue = value.toString(); + if (keywordValue.length() > ignoreAbove) { + return null; + } + return keywordValue; + }); } } diff --git a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java index b809b0fb87139..423de2da852b5 100644 --- a/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java +++ b/plugins/analysis-icu/src/test/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapperTests.java @@ -489,26 +489,27 @@ public void testUpdateIgnoreAbove() throws IOException { indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); ICUCollationKeywordFieldMapper mapper = new ICUCollationKeywordFieldMapper.Builder("field").build(context); - assertEquals("42", mapper.parseSourceValue(42L, null)); - assertEquals("true", mapper.parseSourceValue(true, null)); + assertEquals(List.of("value"), fetchFromSource(mapper, null, "value")); + assertEquals(List.of("42"), fetchFromSource(mapper, null, 42L)); + assertEquals(List.of("true"), fetchFromSource(mapper, null, true)); ICUCollationKeywordFieldMapper ignoreAboveMapper = new ICUCollationKeywordFieldMapper.Builder("field") .ignoreAbove(4) .build(context); - assertNull(ignoreAboveMapper.parseSourceValue("value", null)); - assertEquals("42", ignoreAboveMapper.parseSourceValue(42L, null)); - assertEquals("true", ignoreAboveMapper.parseSourceValue(true, null)); + assertEquals(List.of(), fetchFromSource(ignoreAboveMapper, null, "value")); + assertEquals(List.of("42"), fetchFromSource(ignoreAboveMapper, null, 42L)); + assertEquals(List.of("true"), fetchFromSource(ignoreAboveMapper, null, true)); ICUCollationKeywordFieldMapper nullValueMapper = new ICUCollationKeywordFieldMapper.Builder("field") .nullValue("NULL") .build(context); SourceLookup sourceLookup = new SourceLookup(); sourceLookup.setSource(Collections.singletonMap("field", null)); - assertEquals(List.of("NULL"), nullValueMapper.lookupValues(sourceLookup, null)); + assertEquals(List.of("NULL"), fetchFromSource(nullValueMapper, null, null)); } } diff --git a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java index 48d1ee50e224e..66b66fc1a2710 100644 --- a/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java +++ b/plugins/mapper-annotated-text/src/internalClusterTest/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapperTests.java @@ -677,7 +677,7 @@ public void testEmptyName() throws IOException { assertThat(e.getMessage(), containsString("name cannot be empty string")); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); @@ -688,8 +688,8 @@ public void testParseSourceValue() { .build(context); AnnotatedTextFieldMapper mapper = (AnnotatedTextFieldMapper) fieldMapper; - assertEquals("value", mapper.parseSourceValue("value", null)); - assertEquals("42", mapper.parseSourceValue(42L, null)); - assertEquals("true", mapper.parseSourceValue(true, null)); + assertEquals(List.of("value"), fetchFromSource(mapper, null, "value")); + assertEquals(List.of("42"), fetchFromSource(mapper, null, 42L)); + assertEquals(List.of("true"), fetchFromSource(mapper, null, true)); } } diff --git a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java index dcab066ca8dd5..d6104f3db518e 100644 --- a/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java +++ b/plugins/mapper-annotated-text/src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java @@ -41,9 +41,11 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextFieldMapper; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotatedText.AnnotationToken; import org.elasticsearch.index.similarity.SimilarityProvider; import org.elasticsearch.search.fetch.FetchSubPhase.HitContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.io.Reader; @@ -584,11 +586,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return value.toString(); + return ValueFetcher.fromSource(this, lookup, Object::toString); } @Override diff --git a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/search/highlight/AnnotatedTextHighlighterTests.java b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/search/highlight/AnnotatedTextHighlighterTests.java index 8630dc870f8c7..6274d9fbd5983 100644 --- a/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/search/highlight/AnnotatedTextHighlighterTests.java +++ b/plugins/mapper-annotated-text/src/test/java/org/elasticsearch/search/highlight/AnnotatedTextHighlighterTests.java @@ -49,6 +49,7 @@ import org.elasticsearch.index.mapper.annotatedtext.AnnotatedTextFieldMapper.AnnotationAnalyzerWrapper; import org.elasticsearch.search.fetch.FetchSubPhase.HitContext; import org.elasticsearch.search.fetch.subphase.highlight.AnnotatedPassageFormatter; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import java.net.URLEncoder; @@ -68,7 +69,7 @@ private void assertHighlightOneDoc(String fieldName, String []markedUpInputs, // Annotated fields wrap the usual analyzer with one that injects extra tokens Analyzer wrapperAnalyzer = new AnnotationAnalyzerWrapper(new StandardAnalyzer()); - HitContext mockHitContext = new HitContext(); + HitContext mockHitContext = new HitContext(new SourceLookup()); AnnotatedHighlighterAnalyzer hiliteAnalyzer = new AnnotatedHighlighterAnalyzer(wrapperAnalyzer, mockHitContext); AnnotatedText[] annotations = new AnnotatedText[markedUpInputs.length]; diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index 4f0ce4ba7bca5..b73cc4da236da 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -37,6 +37,7 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.TypeParsers; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.search.lookup.SearchLookup; @@ -150,11 +151,11 @@ protected void parseCreateField(ParseContext context) } @Override - protected String parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return value.toString(); + return ValueFetcher.fromSource(this, lookup, Object::toString); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java index 34d3cc05aa22b..02f1583c3ebc4 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/LeafFieldData.java @@ -21,6 +21,11 @@ import org.apache.lucene.util.Accountable; import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.search.DocValueFormat; + +import java.util.ArrayList; +import java.util.List; /** * The thread safe {@link org.apache.lucene.index.LeafReader} level cache of the data. @@ -37,4 +42,17 @@ public interface LeafFieldData extends Accountable, Releasable { */ SortedBinaryDocValues getBytesValues(); + default ValueFetcher.LeafValueFetcher buildFetcher(DocValueFormat format) { + SortedBinaryDocValues bytes = getBytesValues(); + return docId -> { + if (false == bytes.advanceExact(docId)) { + return List.of(); + } + List result = new ArrayList<>(bytes.docValueCount()); + for (int i = 0, count = bytes.docValueCount(); i < count; ++i) { + result.add(format.format(bytes.nextValue())); + } + return result; + }; + } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/LeafNumericFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/LeafNumericFieldData.java index 1f4ab23fad758..098a2092aa78d 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/LeafNumericFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/LeafNumericFieldData.java @@ -39,5 +39,4 @@ public interface LeafNumericFieldData extends LeafFieldData { * same ones as you would get from casting to a double. */ SortedNumericDoubleValues getDoubleValues(); - } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java index 3ab7847f641d7..3aab495192724 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafDoubleFieldData.java @@ -21,14 +21,18 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.Accountable; -import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.FieldData; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.search.DocValueFormat; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.List; /** @@ -82,4 +86,18 @@ public Collection getChildResources() { public void close() { } + @Override + public ValueFetcher.LeafValueFetcher buildFetcher(DocValueFormat format) { + SortedNumericDoubleValues doubles = getDoubleValues(); + return docId -> { + if (false == doubles.advanceExact(docId)) { + return List.of(); + } + List result = new ArrayList<>(doubles.docValueCount()); + for (int i = 0, count = doubles.docValueCount(); i < count; ++i) { + result.add(format.format(doubles.nextValue())); + } + return result; + }; + } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java index 38dede8537b94..500957933cb72 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/LeafLongFieldData.java @@ -19,12 +19,18 @@ package org.elasticsearch.index.fielddata.plain; -import org.elasticsearch.index.fielddata.LeafNumericFieldData; +import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; +import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.search.DocValueFormat; + +import java.util.ArrayList; +import java.util.List; /** * Specialization of {@link LeafNumericFieldData} for integers. @@ -75,4 +81,22 @@ public final SortedNumericDoubleValues getDoubleValues() { @Override public void close() {} + + @Override + public ValueFetcher.LeafValueFetcher buildFetcher(DocValueFormat format) { + return buildLongFetcher(format, getLongValues()); + } + + protected ValueFetcher.LeafValueFetcher buildLongFetcher(DocValueFormat format, SortedNumericDocValues longs) { + return docId -> { + if (false == longs.advanceExact(docId)) { + return List.of(); + } + List result = new ArrayList<>(longs.docValueCount()); + for (int i = 0, count = longs.docValueCount(); i < count; ++i) { + result.add(format.format(longs.nextValue())); + } + return result; + }; + } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java index 41441ac36fced..8a42b567082c0 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java @@ -39,7 +39,9 @@ import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; import org.elasticsearch.search.aggregations.support.ValuesSourceType; @@ -173,6 +175,16 @@ public SortedNumericDocValues getLongValuesAsNanos() { throw new IllegalStateException("Cannot load doc values", e); } } + + @Override + public ValueFetcher.LeafValueFetcher buildFetcher(DocValueFormat format) { + /* + * When we fetch values from doc values we want nanos and + * the superclass implementation of this calls getLongValues + * which is milliseconds. + */ + return buildLongFetcher(format, getLongValuesAsNanos()); + } } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 24d0a11ba1729..dd546948fe3b8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -25,9 +25,9 @@ import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.geo.GeoJsonGeometryFormat; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.geo.SpatialStrategy; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; @@ -42,7 +42,6 @@ import org.elasticsearch.index.query.QueryShardException; import java.io.IOException; -import java.io.UncheckedIOException; import java.text.ParseException; import java.util.ArrayList; import java.util.Collections; @@ -50,6 +49,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.Function; /** * Base field mapper class for all spatial field types @@ -97,29 +97,30 @@ public abstract static class Parser { * Supported formats include 'geojson' and 'wkt'. The different formats are defined * as subclasses of {@link org.elasticsearch.common.geo.GeometryFormat}. */ - public abstract Object format(Parsed value, String format); + protected abstract Function formatter(String format); /** * Parses the given value, then formats it according to the 'format' string. * * By default, this method simply parses the value using {@link Parser#parse}, then formats - * it with {@link Parser#format}. However some {@link Parser} implementations override this - * as they can avoid parsing the value if it is already in the right format. + * it with {@link Parser#formatter(String)}. However some {@link Parser} implementations + * override this as they can avoid parsing the value if it is already in the right format. */ - public Object parseAndFormatObject(Object value, AbstractGeometryFieldMapper mapper, String format) { - Parsed geometry; - try (XContentParser parser = new MapXContentParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, - Collections.singletonMap("dummy_field", value), XContentType.JSON)) { - parser.nextToken(); // start object - parser.nextToken(); // field name - parser.nextToken(); // field value - geometry = parse(parser, mapper); - } catch (IOException e) { - throw new UncheckedIOException(e); - } catch (ParseException e) { - throw new RuntimeException(e); - } - return format(geometry, format); + public CheckedFunction formatter(AbstractGeometryFieldMapper mapper, String format) { + Function formatter = formatter(format); + return value -> { + Parsed geometry; + try (XContentParser parser = new MapXContentParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + Collections.singletonMap("dummy_field", value), XContentType.JSON)) { + parser.nextToken(); // start object + parser.nextToken(); // field name + parser.nextToken(); // field value + geometry = parse(parser, mapper); + } catch (ParseException e) { + throw new IOException(e); + } + return formatter.apply(geometry); + }; } } @@ -182,17 +183,6 @@ public Builder ignoreZValue(final boolean ignoreZValue) { } } - @Override - protected Object parseSourceValue(Object value, String format) { - if (format == null) { - format = GeoJsonGeometryFormat.NAME; - } - - AbstractGeometryFieldType mappedFieldType = fieldType(); - Parser geometryParser = mappedFieldType.geometryParser(); - return geometryParser.parseAndFormatObject(value, this, format); - } - public abstract static class TypeParser implements Mapper.TypeParser { protected abstract T newBuilder(String name, Map params); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java index 5160d97faf21a..ca530f82cf173 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractPointGeometryFieldMapper.java @@ -20,8 +20,10 @@ import org.apache.lucene.document.FieldType; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.geo.GeoJsonGeometryFormat; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeometryFormat; import org.elasticsearch.common.geo.GeometryParser; @@ -30,6 +32,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Point; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.text.ParseException; @@ -37,6 +40,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.Function; import static org.elasticsearch.index.mapper.TypeParsers.parseField; @@ -182,6 +186,18 @@ protected void parsePointIgnoringMalformed(XContentParser parser, ParsedPoint po } } + @Override + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { + if (format == null) { + format = GeoJsonGeometryFormat.NAME; + } + + AbstractGeometryFieldType mappedFieldType = fieldType(); + CheckedFunction formatter = mappedFieldType.geometryParser().formatter(this, format); + return ValueFetcher.fromSourceManualyHandlingLists(this, lookup, v -> (List) formatter.apply(v)); + } + + /** A parser implementation that can parse the various point formats */ public static class PointParser

extends Parser> { /** @@ -253,14 +269,16 @@ public List

parse(XContentParser parser, AbstractGeometryFieldMapper geometry } @Override - public Object format(List

points, String format) { - List result = new ArrayList<>(); + public Function, Object> formatter(String format) { GeometryFormat geometryFormat = geometryParser.geometryFormat(format); - for (ParsedPoint point : points) { - Geometry geometry = point.asGeometry(); - result.add(geometryFormat.toXContentAsObject(geometry)); - } - return result; + return points -> { + List result = new ArrayList<>(); + for (ParsedPoint point : points) { + Geometry geometry = point.asGeometry(); + result.add(geometryFormat.toXContentAsObject(geometry)); + } + return result; + }; } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java index 5ee6a19be93aa..b6dc8448e00f5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractShapeGeometryFieldMapper.java @@ -21,12 +21,14 @@ import org.apache.lucene.document.FieldType; import org.elasticsearch.common.Explicit; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.geo.GeoJsonGeometryFormat; import org.elasticsearch.common.geo.builders.ShapeBuilder; import org.elasticsearch.common.geo.builders.ShapeBuilder.Orientation; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.mapper.LegacyGeoShapeFieldMapper.DeprecatedParameters; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Iterator; @@ -179,11 +181,6 @@ protected AbstractShapeGeometryFieldMapper(String simpleName, FieldType fieldTyp this.orientation = orientation; } - @Override - public final boolean parsesArrayValue() { - return false; - } - @Override protected final void mergeOptions(FieldMapper other, List conflicts) { AbstractShapeGeometryFieldMapper gsfm = (AbstractShapeGeometryFieldMapper)other; @@ -217,4 +214,14 @@ public Explicit coerce() { public Orientation orientation() { return ((AbstractShapeGeometryFieldType)mappedFieldType).orientation(); } + + @Override + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { + if (format == null) { + format = GeoJsonGeometryFormat.NAME; + } + + AbstractGeometryFieldType mappedFieldType = fieldType(); + return ValueFetcher.fromSource(this, lookup, mappedFieldType.geometryParser().formatter(this, format)); + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index 7f1c529e35b19..2bf13c620c2c6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.hppc.ObjectArrayList; + import org.apache.lucene.document.StoredField; import org.apache.lucene.index.Term; import org.apache.lucene.search.DocValuesFieldExistsQuery; @@ -192,11 +193,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return value; + return ValueFetcher.fromSource(this, lookup, v -> v); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index c115d0623e6c5..27438bb7dc9f9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -252,17 +252,19 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public Boolean parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - if (value instanceof Boolean) { - return (Boolean) value; - } else { - String textValue = value.toString(); - return Booleans.parseBoolean(textValue.toCharArray(), 0, textValue.length(), false); - } + return ValueFetcher.fromSource(this, lookup, value -> { + if (value instanceof Boolean) { + return (Boolean) value; + } else { + String textValue = value.toString(); + return Booleans.parseBoolean(textValue.toCharArray(), 0, textValue.length(), false); + } + }); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java index 10ed72efaa8f9..2789deed3e868 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/CompletionFieldMapper.java @@ -45,6 +45,7 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.suggest.completion.CompletionSuggester; import org.elasticsearch.search.suggest.completion.context.ContextMapping; import org.elasticsearch.search.suggest.completion.context.ContextMappings; @@ -537,16 +538,18 @@ private void parse(ParseContext parseContext, Token token, } @Override - protected List parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - if (value instanceof List) { - return (List) value; - } else { - return List.of(value); - } + return ValueFetcher.fromSourceManualyHandlingLists(this, lookup, value -> { + if (value instanceof List) { + return (List) value; + } else { + return List.of(value); + } + }); } static class CompletionInputMetadata { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 2afe252c79a5d..0b4128635ae76 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -596,16 +596,17 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public String parseSourceValue(Object value, String format) { - String date = value.toString(); - long timestamp = fieldType().parse(date); - - ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC); - DateFormatter dateTimeFormatter = fieldType().dateTimeFormatter(); - if (format != null) { - dateTimeFormatter = DateFormatter.forPattern(format).withLocale(dateTimeFormatter.locale()); - } - return dateTimeFormatter.format(dateTime); + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { + DateFormatter dtFormatter = format == null + ? fieldType().dateTimeFormatter() + : DateFormatter.forPattern(format).withLocale(fieldType().dateTimeFormatter().locale()); + return ValueFetcher.fromSource(this, lookup, value -> { + String date = value.toString(); + long timestamp = fieldType().parse(date); + ZonedDateTime dateTime = fieldType().resolution().toInstant(timestamp).atZone(ZoneOffset.UTC); + + return dtFormatter.format(dateTime); + }); } public boolean getIgnoreMalformed() { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index b2513baeb42fe..f5408ad8433f4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -21,6 +21,7 @@ import com.carrotsearch.hppc.cursors.ObjectCursor; import com.carrotsearch.hppc.cursors.ObjectObjectCursor; + import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.IndexOptions; @@ -34,7 +35,7 @@ import org.elasticsearch.common.xcontent.support.AbstractXContentParser; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.FieldNamesFieldMapper.FieldNamesFieldType; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.ArrayList; @@ -235,12 +236,6 @@ protected Object nullValue() { return null; } - /** - * Whether this mapper can handle an array value during document parsing. If true, - * when an array is encountered during parsing, the document parser will pass the - * whole array to the mapper. If false, the array is split into individual values - * and each value is passed to the mapper for parsing. - */ public boolean parsesArrayValue() { return false; } @@ -283,13 +278,16 @@ public void parse(ParseContext context) throws IOException { protected abstract void parseCreateField(ParseContext context) throws IOException; /** - * Given access to a document's _source, return this field's values. + * Build a {@linkplain ValueFetcher} to fetch values for the fields fetch api. + * + * Subclasses should return a fetcher that resolves the values for the + * document, normalizes the values into a standard form, and then applies the + * specified format. They can use {@link ValueFetcher#fromSource} to implement + * this by loading from source. Or {@link ValueFetcher#fromDocValues} to load + * from doc values. * - * In addition to pulling out the values, mappers can parse them into a standard form. This - * method delegates parsing to {@link #parseSourceValue} for parsing. Most mappers will choose - * to override {@link #parseSourceValue} -- for example numeric field mappers make sure to - * parse the source value into a number of the right type. Some mappers may need more - * flexibility and can override this entire method instead. + * For example, {@link DateFieldMapper} parses the values as a date and then + * format the date with the format provided. * * Note that for array values, the order in which values are returned is undefined and should * not be relied on. @@ -298,35 +296,7 @@ public void parse(ParseContext context) throws IOException { * @param format an optional format string used when formatting values, for example a date format. * @return a list a standardized field values. */ - public List lookupValues(SourceLookup lookup, @Nullable String format) { - Object sourceValue = lookup.extractValue(name(), nullValue()); - if (sourceValue == null) { - return List.of(); - } - - List values = new ArrayList<>(); - if (parsesArrayValue()) { - return (List) parseSourceValue(sourceValue, format); - } else { - List sourceValues = sourceValue instanceof List ? (List) sourceValue : List.of(sourceValue); - for (Object value : sourceValues) { - Object parsedValue = parseSourceValue(value, format); - if (parsedValue != null) { - values.add(parsedValue); - } - } - } - return values; - } - - /** - * Given a value that has been extracted from a document's source, parse it into a standard - * format. This parsing logic should closely mirror the value parsing in - * {@link #parseCreateField} or {@link #parse}. - * - * Note that when overriding this method, {@link #lookupValues} should *not* be overridden. - */ - protected abstract Object parseSourceValue(Object value, @Nullable String format); + public abstract ValueFetcher valueFetcher(SearchLookup lookup, @Nullable String format); protected void createFieldNamesField(ParseContext context) { FieldNamesFieldType fieldNamesFieldType = context.docMapper().metadataMapper(FieldNamesFieldMapper.class).fieldType(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeParser.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeParser.java index 9a82646b95dd6..25817abbaee93 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoShapeParser.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.mapper; +import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.geo.GeometryFormat; import org.elasticsearch.common.geo.GeometryParser; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; @@ -29,9 +30,9 @@ import org.elasticsearch.geometry.Geometry; import java.io.IOException; -import java.io.UncheckedIOException; import java.text.ParseException; import java.util.Collections; +import java.util.function.Function; public class GeoShapeParser extends AbstractGeometryFieldMapper.Parser { private final GeometryParser geometryParser; @@ -46,29 +47,30 @@ public Geometry parse(XContentParser parser, AbstractGeometryFieldMapper mapper) } @Override - public Object format(Geometry value, String format) { - return geometryParser.geometryFormat(format).toXContentAsObject(value); + public Function formatter(String format) { + return geometryParser.geometryFormat(format)::toXContentAsObject; } @Override - public Object parseAndFormatObject(Object value, AbstractGeometryFieldMapper mapper, String format) { - try (XContentParser parser = new MapXContentParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, - Collections.singletonMap("dummy_field", value), XContentType.JSON)) { - parser.nextToken(); // start object - parser.nextToken(); // field name - parser.nextToken(); // field value - - GeometryFormat geometryFormat = geometryParser.geometryFormat(parser); - if (geometryFormat.name().equals(format)) { - return value; + public CheckedFunction formatter(AbstractGeometryFieldMapper mapper, String format) { + Function formatter = formatter(format); + return value -> { + try (XContentParser parser = new MapXContentParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, + Collections.singletonMap("dummy_field", value), XContentType.JSON)) { + parser.nextToken(); // start object + parser.nextToken(); // field name + parser.nextToken(); // field value + + GeometryFormat geometryFormat = geometryParser.geometryFormat(parser); + if (geometryFormat.name().equals(format)) { + return value; + } + + Geometry geometry = geometryFormat.fromXContent(parser); + return formatter.apply(geometry); + } catch (ParseException e) { + throw new IOException(e); } - - Geometry geometry = geometryFormat.fromXContent(parser); - return format(geometry, format); - } catch (IOException e) { - throw new UncheckedIOException(e); - } catch (ParseException e) { - throw new RuntimeException(e); - } + }; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 6a284d8a115f1..59693e5f4928f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -408,18 +408,20 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - InetAddress address; - if (value instanceof InetAddress) { - address = (InetAddress) value; - } else { - address = InetAddresses.forString(value.toString()); - } - return InetAddresses.toAddrString(address); + return ValueFetcher.fromSource(this, lookup, value -> { + InetAddress address; + if (value instanceof InetAddress) { + address = (InetAddress) value; + } else { + address = InetAddresses.forString(value.toString()); + } + return InetAddresses.toAddrString(address); + }); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 858d1888ca4cf..52799f266bcb6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -48,7 +48,6 @@ import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; -import java.io.UncheckedIOException; import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -406,26 +405,23 @@ private String normalizeValue(NamedAnalyzer normalizer, String value) throws IOE } @Override - protected String parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - String keywordValue = value.toString(); - if (keywordValue.length() > ignoreAbove) { - return null; - } - - NamedAnalyzer normalizer = fieldType().normalizer(); - if (normalizer == null) { - return keywordValue; - } + return ValueFetcher.fromSource(this, lookup, value -> { + String keywordValue = value.toString(); + if (keywordValue.length() > ignoreAbove) { + return null; + } - try { + NamedAnalyzer normalizer = fieldType().normalizer(); + if (normalizer == null) { + return keywordValue; + } return normalizeValue(normalizer, keywordValue); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + }); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java index 564302a973240..3885ad057587d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapper.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.geo.GeometryFormat; import org.elasticsearch.common.geo.GeometryParser; import org.elasticsearch.common.geo.ShapesAvailability; import org.elasticsearch.common.geo.SpatialStrategy; @@ -54,6 +55,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Function; /** * FieldMapper for indexing {@link org.locationtech.spatial4j.shape.Shape}s. @@ -297,9 +299,9 @@ private LegacyGeoShapeParser() { } @Override - public Object format(ShapeBuilder value, String format) { - Geometry geometry = value.buildGeometry(); - return geometryParser.geometryFormat(format).toXContentAsObject(geometry); + protected Function, Object> formatter(String format) { + GeometryFormat geoFormat = geometryParser.geometryFormat(format); + return value -> geoFormat.toXContentAsObject(value.buildGeometry()); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java index 03bdaaaa7b021..5df8f360dba63 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MetadataFieldMapper.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.Explicit; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.support.XContentMapValues; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Map; @@ -165,7 +166,7 @@ public void postParse(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source."); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 4c60e2f403a3e..3994abec0d2d8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1129,16 +1129,18 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Number parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - if (value.equals("")) { - return nullValue; - } + return ValueFetcher.fromSource(this, lookup, value -> { + if (value.equals("")) { + return nullValue; + } - return fieldType().type.parse(value, coerce.value()); + return fieldType().type.parse(value, coerce.value()); + }); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 9c2e76a26e5bc..f2a62f03917ce 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -377,28 +377,28 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - @SuppressWarnings("unchecked") - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { RangeType rangeType = fieldType().rangeType(); - if (!(value instanceof Map)) { - assert rangeType == RangeType.IP; - Tuple ipRange = InetAddresses.parseCidr(value.toString()); - return InetAddresses.toCidrString(ipRange.v1(), ipRange.v2()); - } - - DateFormatter dateTimeFormatter = fieldType().dateTimeFormatter(); - if (format != null) { - dateTimeFormatter = DateFormatter.forPattern(format).withLocale(dateTimeFormatter.locale()); - } - - Map range = (Map) value; - Map parsedRange = new HashMap<>(); - for (Map.Entry entry : range.entrySet()) { - Object parsedValue = rangeType.parseValue(entry.getValue(), coerce.value(), fieldType().dateMathParser); - Object formattedValue = rangeType.formatValue(parsedValue, dateTimeFormatter); - parsedRange.put(entry.getKey(), formattedValue); - } - return parsedRange; + DateFormatter dtFormatter = format == null + ? fieldType().dateTimeFormatter() + : DateFormatter.forPattern(format).withLocale(fieldType().dateTimeFormatter().locale()); + + return ValueFetcher.fromSource(this, lookup, value -> { + if (!(value instanceof Map)) { + assert rangeType == RangeType.IP; + Tuple ipRange = InetAddresses.parseCidr(value.toString()); + return InetAddresses.toCidrString(ipRange.v1(), ipRange.v2()); + } + + Map range = (Map) value; + Map parsedRange = new HashMap<>(); + for (Map.Entry entry : range.entrySet()) { + Object parsedValue = rangeType.parseValue(entry.getValue(), coerce.value(), fieldType().dateMathParser); + Object formattedValue = rangeType.formatValue(parsedValue, dtFormatter); + parsedRange.put(entry.getKey().toString(), formattedValue); + } + return parsedRange; + }); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index f6eb6ad704c59..e5abdaa7cd790 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -499,7 +499,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { throw new UnsupportedOperationException(); } @@ -530,7 +530,7 @@ protected void parseCreateField(ParseContext context) { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { throw new UnsupportedOperationException(); } @@ -837,11 +837,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return value.toString(); + return ValueFetcher.fromSource(this, lookup, Object::toString); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java new file mode 100644 index 0000000000000..b3b6aa2265710 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/ValueFetcher.java @@ -0,0 +1,131 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Reads values for documents in a leaf. + */ +@FunctionalInterface +public interface ValueFetcher { + /** + * Prepare to fetch values for a {@linkplain LeafReaderContext}. + */ + LeafValueFetcher leaf(LeafReaderContext context) throws IOException; + + /** + * Fetch values for a particular document. + */ + @FunctionalInterface + interface LeafValueFetcher { + /** + * Fetch values for a particular document. + */ + List fetch(int docId) throws IOException; + } + + /** + * Build a {@linkplain ValueFetcher} that reads values from the source + * then calls {@code convertSourceValues} to parse, normalize, and format each value. + *

+ * If the source doesn't contain any values then this will return an + * empty list and won't call {@code convertSourceValues} at all. + */ + static ValueFetcher fromSource( + FieldMapper fieldMapper, + SearchLookup lookup, + CheckedFunction convertSourceValue + ) { + assert fieldMapper.parsesArrayValue() == false; + return uncheckedFromSource(fieldMapper, lookup, sourceValue -> { + if (sourceValue instanceof List) { + List sourceValues = (List) sourceValue; + List values = new ArrayList<>(sourceValues.size()); + for (Object value : sourceValues) { + Object converted = convertSourceValue.apply(value); + if (converted != null) { + values.add(converted); + } + } + return values; + } + Object converted = convertSourceValue.apply(sourceValue); + if (converted != null) { + return List.of(converted); + } + return List.of(); + }); + } + + /** + * Build a {@linkplain ValueFetcher} that reads values from the source + * then calls {@code convertSourceValues} to parse, normalize, and format the value. + *

+ * If the source doesn't contain the value then this will return an + * empty list and won't call {@code convertSourceValues} at all. + */ + static ValueFetcher fromSourceManualyHandlingLists( + FieldMapper fieldMapper, + SearchLookup lookup, + CheckedFunction, IOException> convertSourceValues + ) { + assert fieldMapper.parsesArrayValue(); + return uncheckedFromSource(fieldMapper, lookup, convertSourceValues); + } + + private static ValueFetcher uncheckedFromSource( + FieldMapper fieldMapper, + SearchLookup lookup, + CheckedFunction, IOException> convertSourceValues + ) { + return ctx -> docId -> { + /* + * setSegmentAndDocument out of pure paranoia. The fetch phase should already + * have positioned us at the right document. So we assert that too. + */ + boolean alreadyPositioned = lookup.source().setSegmentAndDocument(ctx, docId); + assert alreadyPositioned; + Object sourceValue = lookup.source().extractValue(fieldMapper.name(), fieldMapper.nullValue()); + if (sourceValue == null) { + return List.of(); + } + return convertSourceValues.apply(sourceValue); + }; + } + + /** + * Build a {@linkplain ValueFetcher} that reads values from doc values using the + * same logic as the doc values fetch phase. + */ + static ValueFetcher fromDocValues(MappedFieldType ft, SearchLookup lookup, String format) { + DocValueFormat dvFormat = ft.docValueFormat(format, null); + IndexFieldData fd = lookup.doc().getForField(ft); + return ctx -> fd.load(ctx).buildFetcher(dvFormat); + } +} diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 68ac85ee7f576..507224a183720 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -291,11 +291,21 @@ MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMap public SearchLookup lookup() { if (lookup == null) { - lookup = new SearchLookup(getMapperService(), this::getForField); + lookup = new SearchLookup( + getMapperService(), + (ft, lookupSource) -> indexFieldDataService.apply(ft, fullyQualifiedIndex.getName(), lookupSource) + ); } return lookup; } + public SearchLookup lookupForFetch() { + return new SearchLookup( + mapperService, + (ft, lookupSource) -> indexFieldDataService.apply(ft, fullyQualifiedIndex.getName(), lookupSource) + ); + } + public NestedScope nestedScope() { return nestedScope; } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java index f185c54b953f2..52ae08f5bae5b 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java @@ -53,6 +53,7 @@ import org.elasticsearch.search.fetch.subphase.InnerHitsContext; import org.elasticsearch.search.fetch.subphase.InnerHitsPhase; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.tasks.TaskCancelledException; @@ -96,6 +97,11 @@ public void execute(SearchContext context) { FieldsVisitor fieldsVisitor = createStoredFieldsVisitor(context, storedToRequestedFields); try { + SearchLookup lookup = context.getQueryShardContext().lookupForFetch(); + if (context.fetchFieldsContext() != null) { + context.fetchFieldsContext().prepare(lookup); + } + DocIdToIndex[] docs = new DocIdToIndex[context.docIdsToLoadSize()]; for (int index = 0; index < context.docIdsToLoadSize(); index++) { docs[index] = new DocIdToIndex(context.docIdsToLoad()[context.docIdsToLoadFrom() + index], index); @@ -104,7 +110,7 @@ public void execute(SearchContext context) { SearchHit[] hits = new SearchHit[context.docIdsToLoadSize()]; SearchHit[] sortedHits = new SearchHit[context.docIdsToLoadSize()]; - FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(); + FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(lookup.source()); for (int index = 0; index < context.docIdsToLoadSize(); index++) { if (context.isCancelled()) { throw new TaskCancelledException("cancelled"); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java index 85bc2ebc58377..1a0fb86ff81ab 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/FetchSubPhase.java @@ -40,9 +40,13 @@ class HitContext { private IndexSearcher searcher; private LeafReaderContext readerContext; private int docId; - private final SourceLookup sourceLookup = new SourceLookup(); + private final SourceLookup sourceLookup; private Map cache; + public HitContext(SourceLookup sourceLookup) { + this.sourceLookup = sourceLookup; + } + public void reset(SearchHit hit, LeafReaderContext context, int docId, IndexSearcher searcher) { this.hit = hit; this.readerContext = context; diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index 576ae1d112582..bcb414c709236 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -20,16 +20,12 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.ReaderUtil; -import org.apache.lucene.index.SortedNumericDocValues; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData; -import org.elasticsearch.index.fielddata.LeafFieldData; -import org.elasticsearch.index.fielddata.LeafNumericFieldData; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; -import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; -import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; +import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.FetchSubPhase; @@ -38,9 +34,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; -import java.util.List; -import static org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; import static org.elasticsearch.search.DocValueFormat.withNanosecondResolution; /** @@ -87,33 +81,13 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept format = fieldType.docValueFormat(formatDesc, null); } LeafReaderContext subReaderContext = null; - LeafFieldData data = null; - SortedBinaryDocValues binaryValues = null; // binary / string / ip fields - SortedNumericDocValues longValues = null; // int / date fields - SortedNumericDoubleValues doubleValues = null; // floating-point fields + ValueFetcher.LeafValueFetcher fetcher = null; for (SearchHit hit : hits) { // if the reader index has changed we need to get a new doc values reader instance if (subReaderContext == null || hit.docId() >= subReaderContext.docBase + subReaderContext.reader().maxDoc()) { int readerIndex = ReaderUtil.subIndex(hit.docId(), context.searcher().getIndexReader().leaves()); subReaderContext = context.searcher().getIndexReader().leaves().get(readerIndex); - data = indexFieldData.load(subReaderContext); - if (indexFieldData instanceof IndexNumericFieldData) { - NumericType numericType = ((IndexNumericFieldData) indexFieldData).getNumericType(); - if (numericType.isFloatingPoint()) { - doubleValues = ((LeafNumericFieldData) data).getDoubleValues(); - } else { - // by default nanoseconds are cut to milliseconds within aggregations - // however for doc value fields we need the original nanosecond longs - if (isNanosecond) { - longValues = ((SortedNumericIndexFieldData.NanoSecondFieldData) data).getLongValuesAsNanos(); - } else { - longValues = ((LeafNumericFieldData) data).getLongValues(); - } - } - } else { - data = indexFieldData.load(subReaderContext); - binaryValues = data.getBytesValues(); - } + fetcher = indexFieldData.load(subReaderContext).buildFetcher(format); } DocumentField hitField = hit.field(field); if (hitField == null) { @@ -122,30 +96,8 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept // docValues fields will still be document fields, and put under "fields" section of a hit. hit.setDocumentField(field, hitField); } - final List values = hitField.getValues(); - int subDocId = hit.docId() - subReaderContext.docBase; - if (binaryValues != null) { - if (binaryValues.advanceExact(subDocId)) { - for (int i = 0, count = binaryValues.docValueCount(); i < count; ++i) { - values.add(format.format(binaryValues.nextValue())); - } - } - } else if (longValues != null) { - if (longValues.advanceExact(subDocId)) { - for (int i = 0, count = longValues.docValueCount(); i < count; ++i) { - values.add(format.format(longValues.nextValue())); - } - } - } else if (doubleValues != null) { - if (doubleValues.advanceExact(subDocId)) { - for (int i = 0, count = doubleValues.docValueCount(); i < count; ++i) { - values.add(format.format(doubleValues.nextValue())); - } - } - } else { - throw new AssertionError("Unreachable code"); - } + hitField.getValues().addAll(fetcher.fetch(subDocId)); } } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java index 79b33003da788..4acddddfa1d33 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsContext.java @@ -20,7 +20,9 @@ import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.search.lookup.SearchLookup; +import java.io.IOException; import java.util.List; /** @@ -28,7 +30,7 @@ */ public class FetchFieldsContext { - private FieldValueRetriever fieldValueRetriever; + private final FieldValueRetriever fieldValueRetriever; public static FetchFieldsContext create(String indexName, MapperService mapperService, @@ -50,4 +52,8 @@ private FetchFieldsContext(FieldValueRetriever fieldValueRetriever) { public FieldValueRetriever fieldValueRetriever() { return fieldValueRetriever; } + + public void prepare(SearchLookup lookup) throws IOException { + fieldValueRetriever.prepare(lookup); + } } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java index e2cf622001565..2c5bfec15f80a 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchFieldsPhase.java @@ -24,8 +24,8 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.internal.SearchContext; -import org.elasticsearch.search.lookup.SourceLookup; +import java.io.IOException; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -37,18 +37,17 @@ public final class FetchFieldsPhase implements FetchSubPhase { @Override - public void hitExecute(SearchContext context, HitContext hitContext) { + public void hitExecute(SearchContext context, HitContext hitContext) throws IOException { FetchFieldsContext fetchFieldsContext = context.fetchFieldsContext(); if (fetchFieldsContext == null) { return; } SearchHit hit = hitContext.hit(); - SourceLookup sourceLookup = hitContext.sourceLookup(); FieldValueRetriever fieldValueRetriever = fetchFieldsContext.fieldValueRetriever(); Set ignoredFields = getIgnoredFields(hit); - Map documentFields = fieldValueRetriever.retrieve(sourceLookup, ignoredFields); + Map documentFields = fieldValueRetriever.retrieve(hitContext, ignoredFields); for (Map.Entry entry : documentFields.entrySet()) { hit.setDocumentField(entry.getKey(), entry.getValue()); } diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java index affc5bc05b49c..4d058cb1b1626 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldValueRetriever.java @@ -19,13 +19,18 @@ package org.elasticsearch.search.fetch.subphase; +import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.index.mapper.MappingLookup; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.mapper.ValueFetcher.LeafValueFetcher; +import org.elasticsearch.search.fetch.FetchSubPhase.HitContext; +import org.elasticsearch.search.lookup.SearchLookup; +import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -33,14 +38,13 @@ import java.util.Map; import java.util.Set; +import static java.util.stream.Collectors.toList; + /** * A helper class to {@link FetchFieldsPhase} that's initialized with a list of field patterns to fetch. * Then given a specific document, it can retrieve the corresponding fields from the document's source. */ public class FieldValueRetriever { - private final MappingLookup fieldMappers; - private final List fieldContexts; - public static FieldValueRetriever create(MapperService mapperService, Collection fieldAndFormats) { MappingLookup fieldMappers = mapperService.documentMapper().mappers(); @@ -53,23 +57,42 @@ public static FieldValueRetriever create(MapperService mapperService, Collection concreteFields = mapperService.simpleMatchToFullName(fieldPattern); for (String field : concreteFields) { if (fieldMappers.getMapper(field) != null && mapperService.isMetadataField(field) == false) { - Set sourcePath = mapperService.sourcePath(field); - fields.add(new FieldContext(field, sourcePath, format)); + Set sourcePaths = mapperService.sourcePath(field); + List mappers = sourcePaths.stream() + .map(path -> (FieldMapper) fieldMappers.getMapper(path)) + .collect(toList()); + fields.add(new FieldContext(field, mappers, format)); } } } - return new FieldValueRetriever(fieldMappers, fields); + return new FieldValueRetriever(fields); } + private final List fieldContexts; + private LeafReaderContext lastLeaf; - private FieldValueRetriever(MappingLookup fieldMappers, - List fieldContexts) { - this.fieldMappers = fieldMappers; + FieldValueRetriever(List fieldContexts) { this.fieldContexts = fieldContexts; } - public Map retrieve(SourceLookup sourceLookup, Set ignoredFields) { + public void prepare(SearchLookup lookup) throws IOException { + fieldContexts.stream().forEach(field -> field.buildFetchers(lookup)); + } + + public Map retrieve(HitContext hitContext, Set ignoredFields) throws IOException { + /* + * The fields fetch APIs are leaf by leaf but Fetch sub phases don't + * work that way so we have to track the last leaf. This should be + * fairly safe because this is called in sorted order which sorts + * by leafs. + */ + if (lastLeaf != hitContext.readerContext()) { + lastLeaf = hitContext.readerContext(); + for (FieldContext fieldContext : fieldContexts) { + fieldContext.prepareForLeaf(hitContext.readerContext()); + } + } Map documentFields = new HashMap<>(); for (FieldContext context : fieldContexts) { String field = context.fieldName; @@ -78,12 +101,9 @@ public Map retrieve(SourceLookup sourceLookup, Set parsedValues = new ArrayList<>(); - for (String path : context.sourcePath) { - FieldMapper fieldMapper = (FieldMapper) fieldMappers.getMapper(path); - List values = fieldMapper.lookupValues(sourceLookup, context.format); - parsedValues.addAll(values); + for (LeafValueFetcher fetcher : context.leafFetchers) { + parsedValues.addAll(fetcher.fetch(hitContext.docId())); } - if (parsedValues.isEmpty() == false) { documentFields.put(field, new DocumentField(field, parsedValues)); } @@ -91,17 +111,34 @@ public Map retrieve(SourceLookup sourceLookup, Set sourcePath; + final List mappers; final @Nullable String format; + final List fetchers; + final List leafFetchers; FieldContext(String fieldName, - Set sourcePath, + List mappers, @Nullable String format) { this.fieldName = fieldName; - this.sourcePath = sourcePath; + this.mappers = mappers; this.format = format; + fetchers = new ArrayList<>(mappers.size()); + leafFetchers = new ArrayList<>(mappers.size()); + } + + private void buildFetchers(SearchLookup lookup) { + for (FieldMapper mapper : mappers) { + fetchers.add(mapper.valueFetcher(lookup, format)); + }; + } + + private void prepareForLeaf(LeafReaderContext context) throws IOException { + leafFetchers.clear(); + for (ValueFetcher fetcher : fetchers) { + leafFetchers.add(fetcher.leaf(context)); + } } } } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index 04aef7d2e8f63..de825759b9f50 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -24,7 +24,8 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; -import java.util.function.Function; +import java.util.function.BiFunction; +import java.util.function.Supplier; public class SearchLookup { @@ -34,8 +35,11 @@ public class SearchLookup { final FieldsLookup fieldsLookup; - public SearchLookup(MapperService mapperService, Function> fieldDataLookup) { - docMap = new DocLookup(mapperService, fieldDataLookup); + public SearchLookup( + MapperService mapperService, + BiFunction, IndexFieldData> fieldDataLookup + ) { + docMap = new DocLookup(mapperService, ft -> fieldDataLookup.apply(ft, () -> this)); sourceLookup = new SourceLookup(); fieldsLookup = new FieldsLookup(mapperService); } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java index d63caed14adb0..42c4bfa16eecd 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SourceLookup.java @@ -95,15 +95,20 @@ public static Map sourceAsMap(BytesReference source) throws Elas return sourceAsMapAndType(source).v2(); } - public void setSegmentAndDocument(LeafReaderContext context, int docId) { + /** + * Point this lookup at a document, returning true if it was already + * pointed at that document, false otherwise. + */ + public boolean setSegmentAndDocument(LeafReaderContext context, int docId) { if (this.reader == context.reader() && this.docId == docId) { // if we are called with the same document, don't invalidate source - return; + return true; } this.reader = context.reader(); this.source = null; this.sourceAsBytes = null; this.docId = docId; + return false; } public void setSource(BytesReference source) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java index 253084ec4ddfa..ac2f2bea3ff4f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldMapperTests.java @@ -47,7 +47,6 @@ import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; @@ -60,6 +59,7 @@ import java.util.Map; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; public class BooleanFieldMapperTests extends ESSingleNodeTestCase { @@ -300,22 +300,21 @@ public void testBoosts() throws Exception { assertEquals(new BoostQuery(new TermQuery(new Term("field", "T")), 2.0f), ft.termQuery("true", null)); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); BooleanFieldMapper mapper = new BooleanFieldMapper.Builder("field").build(context); - assertTrue(mapper.parseSourceValue(true, null)); - assertFalse(mapper.parseSourceValue("false", null)); - assertFalse(mapper.parseSourceValue("", null)); + assertThat(fetchFromSource(mapper, null, true), equalTo(List.of(true))); + assertThat(fetchFromSource(mapper, null, false), equalTo(List.of(false))); + assertThat(fetchFromSource(mapper, null, "false"), equalTo(List.of(false))); + assertThat(fetchFromSource(mapper, null, ""), equalTo(List.of(false))); Map mapping = Map.of("type", "boolean", "null_value", true); BooleanFieldMapper.Builder builder = new BooleanFieldMapper.Builder("field"); builder.parse("field", null, new HashMap<>(mapping)); BooleanFieldMapper nullValueMapper = builder.build(context); - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(Collections.singletonMap("field", null)); - assertEquals(List.of(true), nullValueMapper.lookupValues(sourceLookup, null)); + assertEquals(List.of(true), fetchFromSource(nullValueMapper, null, null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java index b18808116b68c..1e41ba9e5277c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompletionFieldMapperTests.java @@ -938,19 +938,19 @@ public void testLimitOfContextMappings() throws Throwable { CompletionFieldMapper.COMPLETION_CONTEXTS_LIMIT + "] has been exceeded")); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); NamedAnalyzer defaultAnalyzer = new NamedAnalyzer("standard", AnalyzerScope.INDEX, new StandardAnalyzer()); CompletionFieldMapper mapper = new CompletionFieldMapper.Builder("completion", defaultAnalyzer, Version.CURRENT).build(context); - assertEquals(List.of("value"), mapper.parseSourceValue("value", null)); + assertEquals(List.of("value"), fetchFromSource(mapper, null, "value")); List list = List.of("first", "second"); - assertEquals(list, mapper.parseSourceValue(list, null)); + assertEquals(list, fetchFromSource(mapper, null, list)); Map object = Map.of("input", List.of("first", "second"), "weight", "2.718"); - assertEquals(List.of(object), mapper.parseSourceValue(object, null)); + assertEquals(List.of(object), fetchFromSource(mapper, null, object)); } private Matcher suggestField(String value) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java index d0b95f1b1bf23..979245b267a19 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java @@ -36,7 +36,7 @@ import org.elasticsearch.index.mapper.MapperService.MergeReason; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; @@ -454,51 +454,60 @@ public void testMeta() throws Exception { assertEquals(mapping3, mapper.mappingSource().toString()); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { DateFieldMapper mapper = createMapper(Resolution.MILLISECONDS, null); String date = "2020-05-15T21:33:02.000Z"; - assertEquals(date, mapper.parseSourceValue(date, null)); - assertEquals(date, mapper.parseSourceValue(1589578382000L, null)); + assertEquals(List.of(date), fetchFromSource(mapper, null, date)); + assertEquals(List.of(date), fetchFromSource(mapper, null, 1589578382000L)); DateFieldMapper mapperWithFormat = createMapper(Resolution.MILLISECONDS, "yyyy/MM/dd||epoch_millis"); String dateInFormat = "1990/12/29"; - assertEquals(dateInFormat, mapperWithFormat.parseSourceValue(dateInFormat, null)); - assertEquals(dateInFormat, mapperWithFormat.parseSourceValue(662428800000L, null)); + assertEquals(List.of(dateInFormat), fetchFromSource(mapperWithFormat, null, dateInFormat)); + assertEquals(List.of(dateInFormat), fetchFromSource(mapperWithFormat, null, 662428800000L)); DateFieldMapper mapperWithMillis = createMapper(Resolution.MILLISECONDS, "epoch_millis"); String dateInMillis = "662428800000"; - assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(dateInMillis, null)); - assertEquals(dateInMillis, mapperWithMillis.parseSourceValue(662428800000L, null)); + assertEquals(List.of(dateInMillis), fetchFromSource(mapperWithMillis, null, dateInMillis)); + assertEquals(List.of(dateInMillis), fetchFromSource(mapperWithMillis, null, 662428800000L)); String nullValueDate = "2020-05-15T21:33:02.000Z"; DateFieldMapper nullValueMapper = createMapper(Resolution.MILLISECONDS, null, nullValueDate); - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(Collections.singletonMap("field", null)); - assertEquals(List.of(nullValueDate), nullValueMapper.lookupValues(sourceLookup, null)); + assertEquals(List.of(nullValueDate), fetchFromSource(nullValueMapper, null, null)); } - public void testParseSourceValueWithFormat() { + public void testFetchValuesWithFormat() throws IOException { DateFieldMapper mapper = createMapper(Resolution.NANOSECONDS, "strict_date_time", "1970-12-29T00:00:00.000Z"); String date = "1990-12-29T00:00:00.000Z"; - assertEquals("1990/12/29", mapper.parseSourceValue(date, "yyyy/MM/dd")); - assertEquals("662428800000", mapper.parseSourceValue(date, "epoch_millis")); - - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(Collections.singletonMap("field", null)); - assertEquals(List.of("1970/12/29"), mapper.lookupValues(sourceLookup, "yyyy/MM/dd")); + assertEquals(List.of("1990/12/29"), fetchFromSource(mapper, "yyyy/MM/dd", date)); + assertEquals(List.of("662428800000"), fetchFromSource(mapper, "epoch_millis", date)); + assertEquals(List.of("1970/12/29"), fetchFromSource(mapper, "yyyy/MM/dd", null)); } - public void testParseSourceValueNanos() { + public void testFetchValuesNanos() throws IOException { DateFieldMapper mapper = createMapper(Resolution.NANOSECONDS, "strict_date_time||epoch_millis"); String date = "2020-05-15T21:33:02.123456789Z"; - assertEquals("2020-05-15T21:33:02.123456789Z", mapper.parseSourceValue(date, null)); - assertEquals("2020-05-15T21:33:02.123Z", mapper.parseSourceValue(1589578382123L, null)); + assertEquals(List.of(date), fetchFromSource(mapper, null, date)); + assertEquals(List.of("2020-05-15T21:33:02.123Z"), fetchFromSource(mapper, null, 1589578382123L)); String nullValueDate = "2020-05-15T21:33:02.123456789Z"; DateFieldMapper nullValueMapper = createMapper(Resolution.NANOSECONDS, "strict_date_time||epoch_millis", nullValueDate); - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(Collections.singletonMap("field", null)); - assertEquals(List.of(nullValueDate), nullValueMapper.lookupValues(sourceLookup, null)); + assertEquals(List.of(nullValueDate), fetchFromSource(nullValueMapper, null, null)); + } + + public void testFetchDocValuesMillis() throws IOException { + DateFieldMapper mapper = createMapper(Resolution.MILLISECONDS, "strict_date_time||epoch_millis"); + DocValueFormat format = mapper.mappedFieldType.docValueFormat(null, null); + String date = "2020-05-15T21:33:02.123Z"; + assertEquals(List.of(date), fetchFromDocValues(mapper, format, date)); + assertEquals(List.of(date), fetchFromDocValues(mapper, format, 1589578382123L)); + } + + public void testFetchDocValuesNanos() throws IOException { + DateFieldMapper mapper = createMapper(Resolution.NANOSECONDS, "strict_date_time||epoch_millis"); + DocValueFormat format = DocValueFormat.withNanosecondResolution(mapper.mappedFieldType.docValueFormat(null, null)); + String date = "2020-05-15T21:33:02.123456789Z"; + assertEquals(List.of(date), fetchFromDocValues(mapper, format, date)); + assertEquals(List.of("2020-05-15T21:33:02.123Z"), fetchFromDocValues(mapper, format, 1589578382123L)); } private DateFieldMapper createMapper(Resolution resolution, String format) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java index 6d1c539157cd9..a42bbbb1d1b60 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentFieldMapperTests.java @@ -32,6 +32,7 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.io.StringReader; @@ -103,7 +104,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java index 44ee214e75594..b2c023b0bacb1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ExternalMapper.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.geometry.Point; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.nio.charset.Charset; @@ -201,8 +202,8 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value, String format) { - return value; + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { + return ValueFetcher.fromSource(this, lookup, v -> v); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java index 8c68014522eaf..b1c9895df357b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FakeStringFieldMapper.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collections; @@ -134,8 +135,8 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value, String format) { - return value.toString(); + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { + return ValueFetcher.fromSource(this, lookup, Object::toString); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java index 2d3a0a3a9851c..cac1ebbaf15a3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldMapperTests.java @@ -34,14 +34,12 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.geo.RandomGeoGenerator; import org.hamcrest.CoreMatchers; import java.io.IOException; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -593,37 +591,39 @@ public void testInvalidGeopointValuesIgnored() throws Exception { ), XContentType.JSON)).rootDoc().getField("location"), nullValue()); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); AbstractGeometryFieldMapper mapper = new GeoPointFieldMapper.Builder("field").build(context); - SourceLookup sourceLookup = new SourceLookup(); - Map jsonPoint = Map.of("type", "Point", "coordinates", List.of(42.0, 27.1)); Map otherJsonPoint = Map.of("type", "Point", "coordinates", List.of(30.0, 50.0)); String wktPoint = "POINT (42.0 27.1)"; String otherWktPoint = "POINT (30.0 50.0)"; // Test a single point in [lon, lat] array format. - sourceLookup.setSource(Collections.singletonMap("field", List.of(42.0, 27.1))); - assertEquals(List.of(jsonPoint), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktPoint), mapper.lookupValues(sourceLookup, "wkt")); + Object value = List.of(42.0, 27.1); + assertEquals(List.of(jsonPoint), fetchFromSource(mapper, null, value)); + assertEquals(List.of(jsonPoint), fetchFromSource(mapper, "geojson", value)); + assertEquals(List.of(wktPoint), fetchFromSource(mapper, "wkt", value)); // Test a single point in "lat, lon" string format. - sourceLookup.setSource(Collections.singletonMap("field", "27.1,42.0")); - assertEquals(List.of(jsonPoint), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktPoint), mapper.lookupValues(sourceLookup, "wkt")); + value = "27.1,42.0"; + assertEquals(List.of(jsonPoint), fetchFromSource(mapper, null, value)); + assertEquals(List.of(jsonPoint), fetchFromSource(mapper, "geojson", value)); + assertEquals(List.of(wktPoint), fetchFromSource(mapper, "wkt", value)); // Test a list of points in [lon, lat] array format. - sourceLookup.setSource(Collections.singletonMap("field", List.of(List.of(42.0, 27.1), List.of(30.0, 50.0)))); - assertEquals(List.of(jsonPoint, otherJsonPoint), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktPoint, otherWktPoint), mapper.lookupValues(sourceLookup, "wkt")); + value = List.of(List.of(42.0, 27.1), List.of(30.0, 50.0)); + assertEquals(List.of(jsonPoint, otherJsonPoint), fetchFromSource(mapper, null, value)); + assertEquals(List.of(jsonPoint, otherJsonPoint), fetchFromSource(mapper, "geojson", value)); + assertEquals(List.of(wktPoint, otherWktPoint), fetchFromSource(mapper, "wkt", value)); // Test a single point in well-known text format. - sourceLookup.setSource(Collections.singletonMap("field", "POINT (42.0 27.1)")); - assertEquals(List.of(jsonPoint), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktPoint), mapper.lookupValues(sourceLookup, "wkt")); + value = "POINT (42.0 27.1)"; + assertEquals(List.of(jsonPoint), fetchFromSource(mapper, null, value)); + assertEquals(List.of(jsonPoint), fetchFromSource(mapper, "geojson", value)); + assertEquals(List.of(wktPoint), fetchFromSource(mapper, "wkt", value)); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java index 8e1a88fe6b9c9..7c3ab25cd946d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoShapeFieldMapperTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.junit.Before; @@ -363,12 +362,11 @@ public String toXContentString(GeoShapeFieldMapper mapper) throws IOException { return toXContentString(mapper, true); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); GeoShapeFieldMapper mapper = new GeoShapeFieldMapper.Builder("field").build(context); - SourceLookup sourceLookup = new SourceLookup(); Map jsonLineString = Map.of("type", "LineString", "coordinates", List.of(List.of(42.0, 27.1), List.of(30.0, 50.0))); @@ -377,23 +375,25 @@ public void testParseSourceValue() { String wktPoint = "POINT (14.0 15.0)"; // Test a single shape in geojson format. - sourceLookup.setSource(Collections.singletonMap("field", jsonLineString)); - assertEquals(List.of(jsonLineString), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktLineString), mapper.lookupValues(sourceLookup, "wkt")); + assertEquals(List.of(jsonLineString), fetchFromSource(mapper, null, jsonLineString)); + assertEquals(List.of(jsonLineString), fetchFromSource(mapper, "geojson", jsonLineString)); + assertEquals(List.of(wktLineString), fetchFromSource(mapper, "wkt", jsonLineString)); // Test a list of shapes in geojson format. - sourceLookup.setSource(Collections.singletonMap("field", List.of(jsonLineString, jsonPoint))); - assertEquals(List.of(jsonLineString, jsonPoint), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktLineString, wktPoint), mapper.lookupValues(sourceLookup, "wkt")); + Object value = List.of(jsonLineString, jsonPoint); + assertEquals(List.of(jsonLineString, jsonPoint), fetchFromSource(mapper, null, value)); + assertEquals(List.of(jsonLineString, jsonPoint), fetchFromSource(mapper, "geojson", value)); + assertEquals(List.of(wktLineString, wktPoint), fetchFromSource(mapper, "wkt", value)); // Test a single shape in wkt format. - sourceLookup.setSource(Collections.singletonMap("field", wktLineString)); - assertEquals(List.of(jsonLineString), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktLineString), mapper.lookupValues(sourceLookup, "wkt")); + assertEquals(List.of(jsonLineString), fetchFromSource(mapper, null, wktLineString)); + assertEquals(List.of(jsonLineString), fetchFromSource(mapper, "geojson", wktLineString)); + assertEquals(List.of(wktLineString), fetchFromSource(mapper, "wkt", wktLineString)); // Test a list of shapes in wkt format. - sourceLookup.setSource(Collections.singletonMap("field", List.of(wktLineString, wktPoint))); - assertEquals(List.of(jsonLineString, jsonPoint), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktLineString, wktPoint), mapper.lookupValues(sourceLookup, "wkt")); + value = List.of(wktLineString, wktPoint); + assertEquals(List.of(jsonLineString, jsonPoint), fetchFromSource(mapper, null, value)); + assertEquals(List.of(jsonLineString, jsonPoint), fetchFromSource(mapper, "geojson", value)); + assertEquals(List.of(wktLineString, wktPoint), fetchFromSource(mapper, "wkt", value)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java index e2dc4f2bf59e6..dac5c0352f19f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldMapperTests.java @@ -40,7 +40,6 @@ import org.elasticsearch.index.IndexService; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; @@ -48,7 +47,6 @@ import java.io.IOException; import java.net.InetAddress; import java.util.Collection; -import java.util.Collections; import java.util.List; import static org.hamcrest.Matchers.containsString; @@ -295,21 +293,19 @@ public void testEmptyName() throws IOException { assertThat(e.getMessage(), containsString("name cannot be empty string")); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); IpFieldMapper mapper = new IpFieldMapper.Builder("field", true).build(context); - assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8::2:1", null)); - assertEquals("2001:db8::2:1", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1", null)); - assertEquals("::1", mapper.parseSourceValue("0:0:0:0:0:0:0:1", null)); + assertEquals(List.of("2001:db8::2:1"), fetchFromSource(mapper, null, "2001:db8::2:1")); + assertEquals(List.of("2001:db8::2:1"), fetchFromSource(mapper, null, "2001:db8:0:0:0:0:2:1")); + assertEquals(List.of("::1"), fetchFromSource(mapper, null, "0:0:0:0:0:0:0:1")); IpFieldMapper nullValueMapper = new IpFieldMapper.Builder("field", true) .nullValue(InetAddresses.forString("2001:db8:0:0:0:0:2:7")) .build(context); - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(Collections.singletonMap("field", null)); - assertEquals(List.of("2001:db8::2:7"), nullValueMapper.lookupValues(sourceLookup, null)); + assertEquals(List.of("2001:db8::2:7"), fetchFromSource(nullValueMapper, null, null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java index 663fec56b5e6a..256b5ca9c6118 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpRangeFieldMapperTests.java @@ -34,7 +34,9 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import org.junit.Before; +import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; import static org.hamcrest.Matchers.containsString; @@ -84,13 +86,13 @@ public void testStoreCidr() throws Exception { } } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); RangeFieldMapper mapper = new RangeFieldMapper.Builder("field", RangeType.IP).build(context); Map range = Map.of("gte", "2001:db8:0:0:0:0:2:1"); - assertEquals(Map.of("gte", "2001:db8::2:1"), mapper.parseSourceValue(range, null)); - assertEquals("2001:db8::2:1/32", mapper.parseSourceValue("2001:db8:0:0:0:0:2:1/32", null)); + assertEquals(List.of(Map.of("gte", "2001:db8::2:1")), fetchFromSource(mapper, null, range)); + assertEquals(List.of("2001:db8::2:1/32"), fetchFromSource(mapper, null, "2001:db8:0:0:0:0:2:1/32")); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index ecc3e507af7d3..a5c2770b015f0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -46,7 +46,6 @@ import org.elasticsearch.indices.analysis.AnalysisModule; import org.elasticsearch.plugins.AnalysisPlugin; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.junit.Before; @@ -171,7 +170,7 @@ public void testDefaults() throws Exception { assertArrayEquals(new String[] { "1234" }, TermVectorsService.getValues(doc.rootDoc().getFields("field"))); FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); - assertEquals("1234", fieldMapper.parseSourceValue("1234", null)); + assertEquals(List.of("1234"), fetchFromSource(fieldMapper, null, "1234")); } public void testIgnoreAbove() throws IOException { @@ -631,37 +630,35 @@ public void testMeta() throws Exception { assertEquals(mapping3, mapper.mappingSource().toString()); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); KeywordFieldMapper mapper = new KeywordFieldMapper.Builder("field").build(context); - assertEquals("value", mapper.parseSourceValue("value", null)); - assertEquals("42", mapper.parseSourceValue(42L, null)); - assertEquals("true", mapper.parseSourceValue(true, null)); + assertEquals(List.of("value"), fetchFromSource(mapper, null, "value")); + assertEquals(List.of("42"), fetchFromSource(mapper, null, 42L)); + assertEquals(List.of("true"), fetchFromSource(mapper, null, true)); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> mapper.parseSourceValue(true, "format")); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> mapper.valueFetcher(null, "format")); assertEquals("Field [field] of type [keyword] doesn't support formats.", e.getMessage()); KeywordFieldMapper ignoreAboveMapper = new KeywordFieldMapper.Builder("field") .ignoreAbove(4) .build(context); - assertNull(ignoreAboveMapper.parseSourceValue("value", null)); - assertEquals("42", ignoreAboveMapper.parseSourceValue(42L, null)); - assertEquals("true", ignoreAboveMapper.parseSourceValue(true, null)); + assertEquals(List.of(), fetchFromSource(ignoreAboveMapper, null, "value")); + assertEquals(List.of("42"), fetchFromSource(ignoreAboveMapper, null, 42L)); + assertEquals(List.of("true"), fetchFromSource(ignoreAboveMapper, null, true)); KeywordFieldMapper normalizerMapper = new KeywordFieldMapper.Builder("field") .normalizer(indexService.getIndexAnalyzers(), "lowercase") .build(context); - assertEquals("value", normalizerMapper.parseSourceValue("VALUE", null)); - assertEquals("42", normalizerMapper.parseSourceValue(42L, null)); - assertEquals("value", normalizerMapper.parseSourceValue("value", null)); + assertEquals(List.of("value"), fetchFromSource(normalizerMapper, null, "VALUE")); + assertEquals(List.of("42"), fetchFromSource(normalizerMapper, null, 42L)); + assertEquals(List.of("value"), fetchFromSource(normalizerMapper, null, "value")); KeywordFieldMapper nullValueMapper = new KeywordFieldMapper.Builder("field") .nullValue("NULL") .build(context); - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(Collections.singletonMap("field", null)); - assertEquals(List.of("NULL"), nullValueMapper.lookupValues(sourceLookup, null)); + assertEquals(List.of("NULL"), fetchFromSource(nullValueMapper, null, null)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java index bd54d663541fe..ffb00d59fba5b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LegacyGeoShapeFieldMapperTests.java @@ -43,7 +43,6 @@ import org.elasticsearch.geometry.Point; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.test.TestGeoShapeFieldMapperPlugin; import org.junit.Before; @@ -846,12 +845,11 @@ public String toXContentString(LegacyGeoShapeFieldMapper mapper) throws IOExcept return toXContentString(mapper, true); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); LegacyGeoShapeFieldMapper mapper = new LegacyGeoShapeFieldMapper.Builder("field").build(context); - SourceLookup sourceLookup = new SourceLookup(); Map jsonLineString = Map.of("type", "LineString", "coordinates", List.of(List.of(42.0, 27.1), List.of(30.0, 50.0))); @@ -860,23 +858,25 @@ public void testParseSourceValue() { String wktPoint = "POINT (14.0 15.0)"; // Test a single shape in geojson format. - sourceLookup.setSource(Collections.singletonMap("field", jsonLineString)); - assertEquals(List.of(jsonLineString), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktLineString), mapper.lookupValues(sourceLookup, "wkt")); + assertEquals(List.of(jsonLineString), fetchFromSource(mapper, null, jsonLineString)); + assertEquals(List.of(jsonLineString), fetchFromSource(mapper, "geojson", jsonLineString)); + assertEquals(List.of(wktLineString), fetchFromSource(mapper, "wkt", jsonLineString)); // Test a list of shapes in geojson format. - sourceLookup.setSource(Collections.singletonMap("field", List.of(jsonLineString, jsonPoint))); - assertEquals(List.of(jsonLineString, jsonPoint), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktLineString, wktPoint), mapper.lookupValues(sourceLookup, "wkt")); + Object value = List.of(jsonLineString, jsonPoint); + assertEquals(List.of(jsonLineString, jsonPoint), fetchFromSource(mapper, null, value)); + assertEquals(List.of(jsonLineString, jsonPoint), fetchFromSource(mapper, "geojson", value)); + assertEquals(List.of(wktLineString, wktPoint), fetchFromSource(mapper, "wkt", value)); // Test a single shape in wkt format. - sourceLookup.setSource(Collections.singletonMap("field", wktLineString)); - assertEquals(List.of(jsonLineString), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktLineString), mapper.lookupValues(sourceLookup, "wkt")); + assertEquals(List.of(jsonLineString), fetchFromSource(mapper, null, wktLineString)); + assertEquals(List.of(jsonLineString), fetchFromSource(mapper, "geojson", wktLineString)); + assertEquals(List.of(wktLineString), fetchFromSource(mapper, "wkt", wktLineString)); // Test a list of shapes in wkt format. - sourceLookup.setSource(Collections.singletonMap("field", List.of(wktLineString, wktPoint))); - assertEquals(List.of(jsonLineString, jsonPoint), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktLineString, wktPoint), mapper.lookupValues(sourceLookup, "wkt")); + value = List.of(wktLineString, wktPoint); + assertEquals(List.of(jsonLineString, jsonPoint), fetchFromSource(mapper, null, value)); + assertEquals(List.of(jsonLineString, jsonPoint), fetchFromSource(mapper, "geojson", value)); + assertEquals(List.of(wktLineString, wktPoint), fetchFromSource(mapper, "wkt", value)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java index c2a60272225bb..f87ef2f77f9e7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldMapperTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.randomizedtesting.annotations.Timeout; + import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; import org.elasticsearch.Version; @@ -38,19 +39,19 @@ import org.elasticsearch.index.mapper.NumberFieldTypeTests.OutOfRangeSpec; import org.elasticsearch.index.termvectors.TermVectorsService; import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.search.lookup.SourceLookup; import java.io.ByteArrayInputStream; import java.io.IOException; import java.math.BigInteger; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; +import static org.hamcrest.Matchers.closeTo; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasSize; public class NumberFieldMapperTests extends AbstractNumericFieldMapperTestCase { @@ -406,22 +407,23 @@ public void testEmptyName() throws IOException { } } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); NumberFieldMapper mapper = new NumberFieldMapper.Builder("field", NumberType.INTEGER).build(context); - assertEquals(3, mapper.parseSourceValue(3.14, null)); - assertEquals(42, mapper.parseSourceValue("42.9", null)); - - NumberFieldMapper nullValueMapper = new NumberFieldMapper.Builder("field", NumberType.FLOAT) - .nullValue(2.71f) - .build(context); - assertEquals(2.71f, (float) nullValueMapper.parseSourceValue("", null), 0.00001); - - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(Collections.singletonMap("field", null)); - assertEquals(List.of(2.71f), nullValueMapper.lookupValues(sourceLookup, null)); + assertEquals(List.of(3), fetchFromSource(mapper, null, 3.14)); + assertEquals(List.of(42), fetchFromSource(mapper, null, 42.9)); + assertEquals(List.of(), fetchFromSource(mapper, null, "")); + assertEquals(List.of(), fetchFromSource(mapper, null, null)); + + NumberFieldMapper nullValueMapper = new NumberFieldMapper.Builder("field", NumberType.FLOAT).nullValue(2.71f).build(context); + assertThat(fetchFromSource(nullValueMapper, null, "42.9"), hasSize(1)); + assertThat(((Number) fetchFromSource(nullValueMapper, null, "42.9").get(0)).doubleValue(), closeTo(42.9f, 0.00001)); + assertThat(fetchFromSource(nullValueMapper, null, ""), hasSize(1)); + assertThat(((Number) fetchFromSource(nullValueMapper, null, "").get(0)).doubleValue(), closeTo(2.71f, 0.00001)); + assertThat(fetchFromSource(nullValueMapper, null, null), hasSize(1)); + assertThat(((Number) fetchFromSource(nullValueMapper, null, null).get(0)).doubleValue(), closeTo(2.71f, 0.00001)); } @Timeout(millis = 30000) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 7eb75339f67bb..72b53e23d52e9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.mapper.ParametrizedFieldMapper.Parameter; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; @@ -185,8 +186,8 @@ protected void parseCreateField(ParseContext context) { } @Override - protected Object parseSourceValue(Object value, String format) { - return null; + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { + throw new UnsupportedOperationException(); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java index b7f4f8cc7d962..539a5ef9c7207 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java @@ -42,6 +42,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashSet; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; @@ -491,35 +492,30 @@ public void testIllegalFormatField() throws Exception { assertEquals("Invalid format: [[test_format]]: Unknown pattern letter: t", e.getMessage()); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); RangeFieldMapper longMapper = new RangeFieldMapper.Builder("field", RangeType.LONG).build(context); Map longRange = Map.of("gte", 3.14, "lt", "42.9"); - assertEquals(Map.of("gte", 3L, "lt", 42L), longMapper.parseSourceValue(longRange, null)); + assertEquals(List.of(Map.of("gte", 3L, "lt", 42L)), fetchFromSource(longMapper, null, longRange)); RangeFieldMapper dateMapper = new RangeFieldMapper.Builder("field", RangeType.DATE) .format("yyyy/MM/dd||epoch_millis") .build(context); Map dateRange = Map.of("lt", "1990/12/29", "gte", 597429487111L); - assertEquals(Map.of("lt", "1990/12/29", "gte", "1988/12/06"), - dateMapper.parseSourceValue(dateRange, null)); + assertEquals(List.of(Map.of("lt", "1990/12/29", "gte", "1988/12/06")), fetchFromSource(dateMapper, null, dateRange)); } - public void testParseSourceValueWithFormat() { + public void testFetchValuesWithFormat() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); - RangeFieldMapper longMapper = new RangeFieldMapper.Builder("field", RangeType.LONG).build(context); - Map longRange = Map.of("gte", 3.14, "lt", "42.9"); - assertEquals(Map.of("gte", 3L, "lt", 42L), longMapper.parseSourceValue(longRange, null)); - RangeFieldMapper dateMapper = new RangeFieldMapper.Builder("field", RangeType.DATE) .format("strict_date_time") .build(context); Map dateRange = Map.of("lt", "1990-12-29T00:00:00.000Z"); - assertEquals(Map.of("lt", "1990/12/29"), dateMapper.parseSourceValue(dateRange, "yyy/MM/dd")); - assertEquals(Map.of("lt", "662428800000"), dateMapper.parseSourceValue(dateRange, "epoch_millis")); + assertEquals(List.of(Map.of("lt", "1990/12/29")), fetchFromSource(dateMapper, "yyy/MM/dd", dateRange)); + assertEquals(List.of(Map.of("lt", "662428800000")), fetchFromSource(dateMapper, "epoch_millis", dateRange)); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 2043b830d8664..d9870ef37451f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -85,6 +85,7 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; @@ -1338,15 +1339,15 @@ public void testMeta() throws Exception { assertEquals(mapping3, mapper.mappingSource().toString()); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); FieldMapper fieldMapper = newBuilder().build(context); TextFieldMapper mapper = (TextFieldMapper) fieldMapper; - assertEquals("value", mapper.parseSourceValue("value", null)); - assertEquals("42", mapper.parseSourceValue(42L, null)); - assertEquals("true", mapper.parseSourceValue(true, null)); + assertEquals(List.of("value"), fetchFromSource(mapper, null, "value")); + assertEquals(List.of("42"), fetchFromSource(mapper, null, 42L)); + assertEquals(List.of("true"), fetchFromSource(mapper, null, true)); } } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java index e0fddbd94485f..c58f5770a9fef 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FetchSourcePhaseTests.java @@ -30,6 +30,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.TestSearchContext; @@ -151,7 +152,7 @@ private FetchSubPhase.HitContext hitExecuteMultiple(XContentBuilder source, bool FetchSourceContext fetchSourceContext = new FetchSourceContext(fetchSource, includes, excludes); SearchContext searchContext = new FetchSourcePhaseTestSearchContext(fetchSourceContext); - FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(); + FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(new SourceLookup()); final SearchHit searchHit = new SearchHit(1, null, nestedIdentity, null, null); // We don't need a real index, just a LeafReaderContext which cannot be mocked. diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java index 3826e0ef09c07..88d485acdff1c 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldValueRetrieverTests.java @@ -19,14 +19,21 @@ package org.elasticsearch.search.fetch.subphase; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.fetch.FetchSubPhase; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.test.ESSingleNodeTestCase; import java.io.IOException; @@ -36,6 +43,9 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.hasSize; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class FieldValueRetrieverTests extends ESSingleNodeTestCase { @@ -331,17 +341,58 @@ public void testObjectFields() throws IOException { assertFalse(fields.containsKey("object")); } - private Map retrieveFields(MapperService mapperService, XContentBuilder source, String fieldPattern) { + public void testFetchsSubDocId() throws IOException { + SearchLookup lookup = new SearchLookup(null, null); + FieldMapper mapper = mock(FieldMapper.class); + when(mapper.valueFetcher(lookup, null)).thenReturn(ctx -> id -> List.of(id)); + FieldValueRetriever fetchFieldsLookup = new FieldValueRetriever( + List.of(new FieldValueRetriever.FieldContext("test", List.of(mapper), null)) + ); + fetchFieldsLookup.prepare(lookup); + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of()); + iw.flush(); + iw.addDocument(List.of()); + try (DirectoryReader reader = iw.getReader()) { + assertThat(reader.leaves(), hasSize(2)); + IndexSearcher indexSearcher = newSearcher(reader); + FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(lookup.source()); + for (LeafReaderContext ctx : reader.leaves()) { + for (int docId = 0; docId < ctx.reader().maxDoc(); docId++) { + hitContext.reset(null, ctx, docId, indexSearcher); + assertThat( + fetchFieldsLookup.retrieve(hitContext, Set.of()), + equalTo(Map.of("test", new DocumentField("test", List.of(docId)))) + ); + } + } + } + } + } + + private Map retrieveFields(MapperService mapperService, XContentBuilder source, String fieldPattern) + throws IOException { List fields = List.of(new FieldAndFormat(fieldPattern, null)); return retrieveFields(mapperService, source, fields); } - private Map retrieveFields(MapperService mapperService, XContentBuilder source, List fields) { - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(BytesReference.bytes(source)); - + private Map retrieveFields(MapperService mapperService, XContentBuilder source, List fields) + throws IOException { FieldValueRetriever fetchFieldsLookup = FieldValueRetriever.create(mapperService, fields); - return fetchFieldsLookup.retrieve(sourceLookup, Set.of()); + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of()); // An empty document is fine because we're setting the source. + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher indexSearcher = newSearcher(reader); + SearchLookup lookup = new SearchLookup(null, null); + FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(lookup.source()); + fetchFieldsLookup.prepare(lookup); + LeafReaderContext context = reader.leaves().get(0); + int docId = 0; + hitContext.reset(null, context, docId, indexSearcher); + hitContext.sourceLookup().setSource(BytesReference.bytes(source)); + return fetchFieldsLookup.retrieve(hitContext, Set.of()); + } + } } public MapperService createMapperService() throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java index 05a9466f109ce..9e94ea4cf6b28 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MockFieldMapper.java @@ -28,6 +28,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; import java.util.Collections; @@ -90,7 +91,7 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { throw new UnsupportedOperationException(); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java index 4d00f33000990..b2f0a6683f349 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESSingleNodeTestCase.java @@ -19,6 +19,18 @@ package org.elasticsearch.test; import com.carrotsearch.randomizedtesting.RandomizedContext; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; @@ -32,26 +44,45 @@ import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings; import org.elasticsearch.common.Priority; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; +import org.elasticsearch.common.xcontent.XContentParserUtils; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.SourceToParse; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.mapper.ValueFetcher.LeafValueFetcher; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.breaker.HierarchyCircuitBreakerService; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.node.MockNode; import org.elasticsearch.node.Node; import org.elasticsearch.node.NodeValidationException; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.script.MockScriptService; +import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.transport.TransportSettings; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -62,14 +93,21 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import java.util.function.Supplier; +import static java.util.Collections.singletonMap; import static org.elasticsearch.cluster.coordination.ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING; import static org.elasticsearch.discovery.SettingsBasedSeedHostsProvider.DISCOVERY_SEED_HOSTS_SETTING; import static org.elasticsearch.test.NodeRoles.dataNode; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * A test that keep a singleton node started for all tests that can be used to get @@ -367,4 +405,105 @@ protected boolean forbidPrivateIndexSettings() { return true; } + /** + * Use a {@linkplain FieldMapper} to extract values from {@code _source}. + */ + protected static List fetchFromSource(FieldMapper mapper, String format, Object sourceValue) throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of()); // An empty document is fine because we're setting the source. + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher indexSearcher = newSearcher(reader); + SearchLookup lookup = new SearchLookup(null, null); + SetOnce> result = new SetOnce<>(); + indexSearcher.search(new MatchAllDocsQuery(), new Collector() { + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + ValueFetcher.LeafValueFetcher lvf = mapper.valueFetcher(lookup, format).leaf(context); + return new LeafCollector() { + @Override + public void setScorer(Scorable scorer) throws IOException {} + + @Override + public void collect(int doc) throws IOException { + // Position the lookup and set the source to mimick the fetch phase + lookup.source().setSegmentAndDocument(context, doc); + lookup.source().setSource(singletonMap(mapper.name(), sourceValue)); + result.set(lvf.fetch(doc)); + } + }; + } + }); + return result.get(); + } + } + } + + /** + * Use a {@linkplain FieldMapper} to extract values from doc values. + */ + protected static List fetchFromDocValues(FieldMapper mapper, DocValueFormat format, Object sourceValue) throws IOException { + MapperService mapperService = mock(MapperService.class); + when(mapperService.fieldType(any())).thenReturn(mapper.fieldType()); + BiFunction, IndexFieldData> fieldDataLookup = (mft, lookupSource) -> mft + .fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService(), mapperService); + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + BytesReference source = BytesReference.bytes( + JsonXContent.contentBuilder().startObject().field(mapper.name(), sourceValue).endObject() + ); + try ( + XContentParser parser = JsonXContent.jsonXContent.createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + source.streamInput() + ) + ) { + ParseContext.Document doc = new ParseContext.Document(); + ParseContext context = mock(ParseContext.class); + when(context.doc()).thenReturn(doc); + when(context.sourceToParse()).thenReturn(new SourceToParse("test", "id", source, XContentType.JSON)); + when(context.parser()).thenReturn(parser); + parser.nextToken(); + XContentParserUtils.ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation); + XContentParserUtils.ensureFieldName(parser, parser.nextToken(), mapper.name()); + parser.nextToken(); + mapper.parse(context); + parser.nextToken(); + XContentParserUtils.ensureExpectedToken(Token.END_OBJECT, parser.currentToken(), parser::getTokenLocation); + iw.addDocument(doc); + } + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher indexSearcher = newSearcher(reader); + SearchLookup lookup = new SearchLookup(mapperService, fieldDataLookup); + IndexFieldData ifd = lookup.doc().getForField(mapper.fieldType()); + SetOnce> result = new SetOnce<>(); + indexSearcher.search(new MatchAllDocsQuery(), new Collector() { + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + LeafValueFetcher lvf = ifd.load(context).buildFetcher(format); + return new LeafCollector() { + @Override + public void setScorer(Scorable scorer) throws IOException {} + + @Override + public void collect(int doc) throws IOException { + result.set(lvf.fetch(doc)); + } + }; + } + }); + return result.get(); + } + } + } } diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index b154d6b9a6205..5d5e37aa231c9 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -46,6 +46,7 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.TypeParsers; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.indices.breaker.CircuitBreakerService; @@ -167,11 +168,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return value; + return ValueFetcher.fromSource(this, lookup, v -> v); } public static class HistogramFieldType extends MappedFieldType { diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 40c23e0ac0c6d..044acd5725af9 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -36,10 +36,10 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TypeParsers; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.time.ZoneId; @@ -266,19 +266,15 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - public List lookupValues(SourceLookup lookup, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return fieldType().value == null + List result = fieldType().value == null ? List.of() : List.of(fieldType().value); - } - - @Override - protected Object parseSourceValue(Object value, String format) { - throw new UnsupportedOperationException("This should never be called, since lookupValues is implemented directly."); + return ctx -> docId -> result; } @Override diff --git a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java index 0552c63f26a68..9968fc4021705 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java +++ b/x-pack/plugin/mapper-constant-keyword/src/test/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapperTests.java @@ -20,7 +20,6 @@ import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.constantkeyword.ConstantKeywordMapperPlugin; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.junit.Before; @@ -30,6 +29,8 @@ import java.util.List; import java.util.Set; +import static org.hamcrest.Matchers.empty; + public class ConstantKeywordFieldMapperTests extends FieldMapperTestCase { @Override @@ -146,8 +147,7 @@ public void testLookupValues() throws Exception { assertEquals(mapping, mapper.mappingSource().toString()); FieldMapper fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); - List values = fieldMapper.lookupValues(new SourceLookup(), null); - assertTrue(values.isEmpty()); + assertThat(fetchFromSource(fieldMapper, null, null), empty()); String mapping2 = Strings.toString(XContentFactory.jsonBuilder().startObject().startObject("_doc") .startObject("properties").startObject("field").field("type", "constant_keyword") @@ -155,7 +155,7 @@ public void testLookupValues() throws Exception { mapper = indexService.mapperService().merge("_doc", new CompressedXContent(mapping2), MergeReason.MAPPING_UPDATE); fieldMapper = (FieldMapper) mapper.mappers().getMapper("field"); - values = fieldMapper.lookupValues(new SourceLookup(), null); + List values = fetchFromSource(fieldMapper, null, null); assertEquals(1, values.size()); assertEquals("foo", values.get(0)); } diff --git a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java index 28dae36486a28..b971c4af694e5 100644 --- a/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java +++ b/x-pack/plugin/mapper-flattened/src/main/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapper.java @@ -43,6 +43,7 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; @@ -567,11 +568,11 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return value; + return ValueFetcher.fromSource(this, lookup, value -> value); } @Override diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java index 64b55b246f2de..33339f7b6922e 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/index/mapper/FlatObjectFieldLookupTests.java @@ -8,8 +8,8 @@ import org.elasticsearch.Version; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.search.lookup.LeafDocLookup; import org.elasticsearch.search.lookup.SearchLookup; @@ -23,7 +23,8 @@ import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.function.Function; +import java.util.function.BiFunction; +import java.util.function.Supplier; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; @@ -161,7 +162,7 @@ public void testScriptDocValuesLookup() { = new KeyedFlatObjectFieldType( "field", true, true, "key2", false, Collections.emptyMap()); when(mapperService.fieldType("json.key2")).thenReturn(fieldType2); - Function> fieldDataSupplier = fieldType -> { + BiFunction, IndexFieldData> fieldDataSupplier = (fieldType, lookupSupplier) -> { KeyedFlatObjectFieldType keyedFieldType = (KeyedFlatObjectFieldType) fieldType; return keyedFieldType.key().equals("key1") ? fieldData1 : fieldData2; }; diff --git a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java index 21f12b3a16d0e..0461f93f0eaeb 100644 --- a/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java +++ b/x-pack/plugin/mapper-flattened/src/test/java/org/elasticsearch/xpack/flattened/mapper/FlatObjectFieldMapperTests.java @@ -32,7 +32,6 @@ import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.flattened.FlattenedMapperPlugin; import org.elasticsearch.xpack.flattened.mapper.FlatObjectFieldMapper.KeyedFlatObjectFieldType; @@ -42,7 +41,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -515,19 +513,17 @@ public void testSplitQueriesOnWhitespace() throws IOException { new String[] {"Hello", "World"}); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); Map sourceValue = Map.of("key", "value"); FlatObjectFieldMapper mapper = new FlatObjectFieldMapper.Builder("field").build(context); - assertEquals(sourceValue, mapper.parseSourceValue(sourceValue, null)); + assertEquals(List.of(sourceValue), fetchFromSource(mapper, null, sourceValue)); FlatObjectFieldMapper nullValueMapper = new FlatObjectFieldMapper.Builder("field") .nullValue("NULL") .build(context); - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(Collections.singletonMap("field", null)); - assertEquals(List.of("NULL"), nullValueMapper.lookupValues(sourceLookup, null)); + assertEquals(List.of("NULL"), fetchFromSource(nullValueMapper, null, null)); } } diff --git a/x-pack/plugin/runtime-fields/qa/rest/build.gradle b/x-pack/plugin/runtime-fields/qa/rest/build.gradle index 1cef6c812f800..b31b18d72ece1 100644 --- a/x-pack/plugin/runtime-fields/qa/rest/build.gradle +++ b/x-pack/plugin/runtime-fields/qa/rest/build.gradle @@ -25,7 +25,6 @@ yamlRestTest { systemProperty 'tests.rest.blacklist', [ /////// TO FIX /////// - 'search/330_fetch_fields/*', // The whole API is not yet supported 'search.aggregation/20_terms/Global ordinals are not loaded with the map execution hint', // Broken. Gotta fix. 'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // Broken. Gotta fix. 'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix. diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java index bf1366b09daa3..1a1ff9bb5488b 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java @@ -16,9 +16,11 @@ import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.ParametrizedFieldMapper; import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xpack.runtimefields.DateScriptFieldScript; import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; @@ -71,8 +73,8 @@ protected void parseCreateField(ParseContext context) { } @Override - protected Object parseSourceValue(Object value, String format) { - throw new UnsupportedOperationException(); + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { + return ValueFetcher.fromDocValues(fieldType(), lookup, format); } @Override diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java index 684057c9c90af..4ad7a0f50ddaa 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/AbstractScriptMappedFieldTypeTestCase.java @@ -78,7 +78,7 @@ protected static QueryShardContext mockContext(boolean allowExpensiveQueries, Ab when(context.allowExpensiveQueries()).thenReturn(allowExpensiveQueries); SearchLookup lookup = new SearchLookup( mapperService, - mft -> mft.fielddataBuilder("test", context::lookup).build(null, null, mapperService) + (mft, lookupSource) -> mft.fielddataBuilder("test", lookupSource).build(null, null, mapperService) ); when(context.lookup()).thenReturn(lookup); return context; diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java index 56f2d3fd8d33c..cbea68647ef80 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/PointFieldMapperTests.java @@ -21,16 +21,14 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.spatial.common.CartesianPoint; import org.hamcrest.CoreMatchers; import java.io.IOException; -import java.util.Collections; import java.util.List; import java.util.Map; -import static org.elasticsearch.index.mapper.AbstractPointGeometryFieldMapper.Names.IGNORE_Z_VALUE; +import static org.elasticsearch.index.mapper.AbstractGeometryFieldMapper.Names.IGNORE_Z_VALUE; import static org.elasticsearch.index.mapper.AbstractPointGeometryFieldMapper.Names.NULL_VALUE; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -305,12 +303,11 @@ public void testIgnoreZValue() throws IOException { assertThat(ignoreZValue, equalTo(false)); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); AbstractGeometryFieldMapper mapper = new PointFieldMapper.Builder("field").build(context); - SourceLookup sourceLookup = new SourceLookup(); Map jsonPoint = Map.of("type", "Point", "coordinates", List.of(42.0, 27.1)); String wktPoint = "POINT (42.0 27.1)"; @@ -318,23 +315,27 @@ public void testParseSourceValue() { String otherWktPoint = "POINT (30.0 50.0)"; // Test a single point in [x, y] array format. - sourceLookup.setSource(Collections.singletonMap("field", List.of(42.0, 27.1))); - assertEquals(List.of(jsonPoint), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktPoint), mapper.lookupValues(sourceLookup, "wkt")); + Object value = List.of(42.0, 27.1); + assertEquals(List.of(jsonPoint), fetchFromSource(mapper, null, value)); + assertEquals(List.of(jsonPoint), fetchFromSource(mapper, "geojson", value)); + assertEquals(List.of(wktPoint), fetchFromSource(mapper, "wkt", value)); // Test a single point in "x, y" string format. - sourceLookup.setSource(Collections.singletonMap("field", "42.0,27.1")); - assertEquals(List.of(jsonPoint), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktPoint), mapper.lookupValues(sourceLookup, "wkt")); + value = "42.0,27.1"; + assertEquals(List.of(jsonPoint), fetchFromSource(mapper, null, value)); + assertEquals(List.of(jsonPoint), fetchFromSource(mapper, "geojson", value)); + assertEquals(List.of(wktPoint), fetchFromSource(mapper, "wkt", value)); // Test a list of points in [x, y] array format. - sourceLookup.setSource(Collections.singletonMap("field", List.of(List.of(42.0, 27.1), List.of(30.0, 50.0)))); - assertEquals(List.of(jsonPoint, otherJsonPoint), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktPoint, otherWktPoint), mapper.lookupValues(sourceLookup, "wkt")); + value = List.of(List.of(42.0, 27.1), List.of(30.0, 50.0)); + assertEquals(List.of(jsonPoint, otherJsonPoint), fetchFromSource(mapper, null, value)); + assertEquals(List.of(jsonPoint, otherJsonPoint), fetchFromSource(mapper, "geojson", value)); + assertEquals(List.of(wktPoint, otherWktPoint), fetchFromSource(mapper, "wkt", value)); // Test a single point in well-known text format. - sourceLookup.setSource(Collections.singletonMap("field", "POINT (42.0 27.1)")); - assertEquals(List.of(jsonPoint), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktPoint), mapper.lookupValues(sourceLookup, "wkt")); + value = "POINT (42.0 27.1)"; + assertEquals(List.of(jsonPoint), fetchFromSource(mapper, null, value)); + assertEquals(List.of(jsonPoint), fetchFromSource(mapper, "geojson", value)); + assertEquals(List.of(wktPoint), fetchFromSource(mapper, "wkt", value)); } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java index a6b6a3fe3840e..bfa1944e6534d 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/index/mapper/ShapeFieldMapperTests.java @@ -26,14 +26,13 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; import org.elasticsearch.index.mapper.SourceToParse; -import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.Collections; import java.util.List; import java.util.Map; -import static org.elasticsearch.index.mapper.AbstractPointGeometryFieldMapper.Names.IGNORE_Z_VALUE; +import static org.elasticsearch.index.mapper.AbstractGeometryFieldMapper.Names.IGNORE_Z_VALUE; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; @@ -332,12 +331,11 @@ public String toXContentString(ShapeFieldMapper mapper) throws IOException { return toXContentString(mapper, true); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); ShapeFieldMapper mapper = new ShapeFieldMapper.Builder("field").build(context); - SourceLookup sourceLookup = new SourceLookup(); Map jsonLineString = Map.of("type", "LineString", "coordinates", List.of(List.of(42.0, 27.1), List.of(30.0, 50.0))); @@ -346,23 +344,27 @@ public void testParseSourceValue() { String wktPoint = "POINT (14.3 15.0)"; // Test a single shape in geojson format. - sourceLookup.setSource(Collections.singletonMap("field", jsonLineString)); - assertEquals(List.of(jsonLineString), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktLineString), mapper.lookupValues(sourceLookup, "wkt")); + assertEquals(List.of(jsonLineString), fetchFromSource(mapper, null, jsonLineString)); + assertEquals(List.of(jsonLineString), fetchFromSource(mapper, "geojson", jsonLineString)); + assertEquals(List.of(wktLineString), fetchFromSource(mapper, "wkt", jsonLineString)); // Test a list of shapes in geojson format. - sourceLookup.setSource(Collections.singletonMap("field", List.of(jsonLineString, jsonPoint))); - assertEquals(List.of(jsonLineString, jsonPoint), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktLineString, wktPoint), mapper.lookupValues(sourceLookup, "wkt")); + Object value = List.of(jsonLineString, jsonPoint); + assertEquals(List.of(jsonLineString, jsonPoint), fetchFromSource(mapper, null, value)); + assertEquals(List.of(jsonLineString, jsonPoint), fetchFromSource(mapper, "geojson", value)); + assertEquals(List.of(wktLineString, wktPoint), fetchFromSource(mapper, "wkt", value)); // Test a single shape in wkt format. - sourceLookup.setSource(Collections.singletonMap("field", wktLineString)); - assertEquals(List.of(jsonLineString), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktLineString), mapper.lookupValues(sourceLookup, "wkt")); + assertEquals(List.of(jsonLineString), fetchFromSource(mapper, null, wktLineString)); + assertEquals(List.of(jsonLineString), fetchFromSource(mapper, "geojson", wktLineString)); + assertEquals(List.of(wktLineString), fetchFromSource(mapper, "wkt", wktLineString)); + // Test a list of shapes in wkt format. - sourceLookup.setSource(Collections.singletonMap("field", List.of(wktLineString, wktPoint))); - assertEquals(List.of(jsonLineString, jsonPoint), mapper.lookupValues(sourceLookup, null)); - assertEquals(List.of(wktLineString, wktPoint), mapper.lookupValues(sourceLookup, "wkt")); + value = List.of(wktLineString, wktPoint); + assertEquals(List.of(jsonLineString, jsonPoint), fetchFromSource(mapper, null, value)); + assertEquals(List.of(jsonLineString, jsonPoint), fetchFromSource(mapper, "geojson", value)); + assertEquals(List.of(wktLineString, wktPoint), fetchFromSource(mapper, "wkt", value)); + } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml index c90d219166380..b1d0ba933d6a6 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword.yml @@ -112,6 +112,20 @@ setup: - match: {hits.hits.0.fields.day_of_week_letters: [T, a, d, h, r, s, u, y] } - match: {hits.hits.0.fields.prefixed_node: [node_c] } +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [day_of_week, day_of_week_from_source, day_of_week_letters, prefixed_node] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.day_of_week: [Thursday] } + - match: {hits.hits.0.fields.day_of_week_from_source: [Thursday] } + - match: {hits.hits.0.fields.day_of_week_letters: [T, a, d, h, r, s, u, y] } + - match: {hits.hits.0.fields.prefixed_node: [node_c] } + --- "terms agg": - do: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/20_long.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/20_long.yml index 041f22284bef6..aaf4cdc8f5bec 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/20_long.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/20_long.yml @@ -102,6 +102,25 @@ setup: - match: {hits.hits.4.fields.voltage_times_ten: [58] } - match: {hits.hits.5.fields.voltage_times_ten: [52] } +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [voltage_times_ten, voltage_times_ten_from_source, temperature_digits] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.voltage_times_ten: [40] } + - match: {hits.hits.0.fields.voltage_times_ten_from_source: [40] } + - match: {hits.hits.0.fields.temperature_digits: [0, 2, 2] } + - match: {hits.hits.0.fields.voltage_times_ten: [40] } + - match: {hits.hits.1.fields.voltage_times_ten: [42] } + - match: {hits.hits.2.fields.voltage_times_ten: [56] } + - match: {hits.hits.3.fields.voltage_times_ten: [51] } + - match: {hits.hits.4.fields.voltage_times_ten: [58] } + - match: {hits.hits.5.fields.voltage_times_ten: [52] } + --- "terms agg": - do: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/30_double.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/30_double.yml index 6b745f8fe396e..800a829c67b48 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/30_double.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/30_double.yml @@ -102,6 +102,25 @@ setup: - match: {hits.hits.4.fields.voltage_percent: [1.0] } - match: {hits.hits.5.fields.voltage_percent: [0.896551724137931] } +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [voltage_percent, voltage_percent_from_source, voltage_sqrts] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.voltage_percent: [0.6896551724137931] } + - match: {hits.hits.0.fields.voltage_percent_from_source: [0.6896551724137931] } + # Scripts that scripts that emit multiple values are supported and their results are sorted + - match: {hits.hits.0.fields.voltage_sqrts: [1.4142135623730951, 2.0, 4.0] } + - match: {hits.hits.1.fields.voltage_percent: [0.7241379310344828] } + - match: {hits.hits.2.fields.voltage_percent: [0.9655172413793103] } + - match: {hits.hits.3.fields.voltage_percent: [0.8793103448275862] } + - match: {hits.hits.4.fields.voltage_percent: [1.0] } + - match: {hits.hits.5.fields.voltage_percent: [0.896551724137931] } + --- "terms agg": - do: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml index c0a6944df00d1..97fc995b4bea5 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml @@ -132,6 +132,29 @@ setup: - 2018-01-24T17:41:34.000Z - match: {hits.hits.0.fields.formatted_tomorrow: [2018-01-19] } +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [tomorrow, tomorrow_from_source, the_past, all_week, formatted_tomorrow] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.tomorrow: [2018-01-19T17:41:34.000Z] } + - match: {hits.hits.0.fields.tomorrow_from_source: [2018-01-19T17:41:34.000Z] } + - match: {hits.hits.0.fields.the_past: [2018-01-18T17:41:33.000Z] } + - match: + hits.hits.0.fields.all_week: + - 2018-01-18T17:41:34.000Z + - 2018-01-19T17:41:34.000Z + - 2018-01-20T17:41:34.000Z + - 2018-01-21T17:41:34.000Z + - 2018-01-22T17:41:34.000Z + - 2018-01-23T17:41:34.000Z + - 2018-01-24T17:41:34.000Z + - match: {hits.hits.0.fields.formatted_tomorrow: [2018-01-19] } + --- "terms agg": - do: diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml index e572ce7462f30..71472437eb351 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml @@ -95,6 +95,25 @@ setup: - 40.135.0.3 - 40.135.0.4 +--- +"fetch fields": + - do: + search: + index: http_logs + body: + sort: timestamp + fields: [ip, ip_from_source, ip_many] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.ip: ["40.135.0.0"] } + - match: {hits.hits.0.fields.ip_from_source: ["40.135.0.0"] } + - match: + hits.hits.0.fields.ip_many: + - 40.135.0.0 + - 40.135.0.1 + - 40.135.0.2 + - 40.135.0.3 + - 40.135.0.4 + --- "terms agg": - do: diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java index c65319a9d8879..9aeefecb96aea 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/DenseVectorFieldMapper.java @@ -25,6 +25,7 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; @@ -208,11 +209,11 @@ public void parse(ParseContext context) throws IOException { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - return value; + return ValueFetcher.fromSourceManualyHandlingLists(this, lookup, v -> (List) v); } @Override diff --git a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java index 91cdeb63d8577..c47a1acf7fb85 100644 --- a/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java +++ b/x-pack/plugin/vectors/src/main/java/org/elasticsearch/xpack/vectors/mapper/SparseVectorFieldMapper.java @@ -19,8 +19,10 @@ import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; import java.time.ZoneId; import java.util.List; @@ -143,7 +145,7 @@ protected void parseCreateField(ParseContext context) { } @Override - protected Object parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { throw new UnsupportedOperationException(ERROR_MESSAGE_7X); } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index da3b746f0ceb7..c0575e6910d83 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -61,6 +61,7 @@ import org.elasticsearch.index.mapper.ParseContext; import org.elasticsearch.index.mapper.ParseContext.Document; import org.elasticsearch.index.mapper.TextSearchInfo; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; @@ -940,16 +941,17 @@ protected void parseCreateField(ParseContext context) throws IOException { } @Override - protected String parseSourceValue(Object value, String format) { + public ValueFetcher valueFetcher(SearchLookup lookup, String format) { if (format != null) { throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); } - - String keywordValue = value.toString(); - if (keywordValue.length() > ignoreAbove) { - return null; - } - return keywordValue; + return ValueFetcher.fromSource(this, lookup, value -> { + String keywordValue = value.toString(); + if (keywordValue.length() > ignoreAbove) { + return null; + } + return keywordValue; + }); } void createFields(String value, Document parseDoc, Listfields) throws IOException { diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 4918f40087ae0..f681a7f8bd457 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -14,6 +14,7 @@ import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.queryparser.classic.ParseException; @@ -21,12 +22,16 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Collector; import org.apache.lucene.search.DocValuesFieldExistsQuery; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.ScoreMode; import org.apache.lucene.search.Sort; import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; @@ -35,6 +40,7 @@ import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.SetOnce; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.ByteRunAutomaton; import org.apache.lucene.util.automaton.RegExp; @@ -50,14 +56,15 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.mapper.ContentPath; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.IndexSettingsModule; @@ -67,11 +74,11 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.function.Supplier; +import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -779,28 +786,26 @@ protected String convertToRandomRegex(String randomValue) { return result.toString(); } - public void testParseSourceValue() { + public void testFetchValues() throws IOException { Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build(); Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath()); WildcardFieldMapper mapper = new WildcardFieldMapper.Builder("field").build(context); - assertEquals("value", mapper.parseSourceValue("value", null)); - assertEquals("42", mapper.parseSourceValue(42L, null)); - assertEquals("true", mapper.parseSourceValue(true, null)); + assertEquals(List.of("value"), fetchFromSource(mapper, null, "value")); + assertEquals(List.of("42"), fetchFromSource(mapper, null, 42L)); + assertEquals(List.of("true"), fetchFromSource(mapper, null, true)); WildcardFieldMapper ignoreAboveMapper = new WildcardFieldMapper.Builder("field") .ignoreAbove(4) .build(context); - assertNull(ignoreAboveMapper.parseSourceValue("value", null)); - assertEquals("42", ignoreAboveMapper.parseSourceValue(42L, null)); - assertEquals("true", ignoreAboveMapper.parseSourceValue(true, null)); + assertEquals(List.of(), fetchFromSource(ignoreAboveMapper, null, "value")); + assertEquals(List.of("42"), fetchFromSource(ignoreAboveMapper, null, 42L)); + assertEquals(List.of("true"), fetchFromSource(ignoreAboveMapper, null, true)); WildcardFieldMapper nullValueMapper = new WildcardFieldMapper.Builder("field") .nullValue("NULL") .build(context); - SourceLookup sourceLookup = new SourceLookup(); - sourceLookup.setSource(Collections.singletonMap("field", null)); - assertEquals(List.of("NULL"), nullValueMapper.lookupValues(sourceLookup, null)); + assertEquals(List.of("NULL"), fetchFromSource(nullValueMapper, null, null)); } protected MappedFieldType provideMappedFieldType(String name) { @@ -912,4 +917,42 @@ private String getRandomWildcardPattern() { } return sb.toString(); } + + /** + * Use a {@linkplain FieldMapper} to extract values from {@code _source}. + */ + protected static List fetchFromSource(FieldMapper mapper, String format, Object sourceValue) throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of()); // An empty document is fine because we're setting the source. + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher indexSearcher = newSearcher(reader); + SearchLookup lookup = new SearchLookup(null, null); + SetOnce> result = new SetOnce<>(); + indexSearcher.search(new MatchAllDocsQuery(), new Collector() { + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + ValueFetcher.LeafValueFetcher lvf = mapper.valueFetcher(lookup, format).leaf(context); + return new LeafCollector() { + @Override + public void setScorer(Scorable scorer) throws IOException {} + + @Override + public void collect(int doc) throws IOException { + // Position the lookup and set the source to mimick the fetch phase + lookup.source().setSegmentAndDocument(context, doc); + lookup.source().setSource(singletonMap(mapper.name(), sourceValue)); + result.set(lvf.fetch(doc)); + } + }; + } + }); + return result.get(); + } + } + } }