Add fetch fields support for runtime fields

modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java

@@ -64,7 +64,7 @@ public void setUp() throws Exception {
         when(fieldData.load(anyObject())).thenReturn(atomicFieldData);
 
         service = new ExpressionScriptEngine();
-        lookup = new SearchLookup(mapperService, ignored -> fieldData);
+        lookup = new SearchLookup(mapperService, (ignored, lookupSource) -> fieldData);
     }
 
     private FieldScript.LeafFactory compile(String expression) {

modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java

@@ -64,7 +64,7 @@ public void setUp() throws Exception {
         when(fieldData.load(anyObject())).thenReturn(atomicFieldData);
 
         service = new ExpressionScriptEngine();
-        lookup = new SearchLookup(mapperService, ignored -> fieldData);
+        lookup = new SearchLookup(mapperService, (ignored, lookupSource) -> fieldData);
     }
 
     private NumberSortScript.LeafFactory compile(String expression) {

modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java

@@ -64,7 +64,7 @@ public void setUp() throws Exception {
         when(fieldData.load(anyObject())).thenReturn(atomicFieldData);
 
         service = new ExpressionScriptEngine();
-        lookup = new SearchLookup(mapperService, ignored -> fieldData);
+        lookup = new SearchLookup(mapperService, (ignored, lookupSource) -> fieldData);
     }
 
     private TermsSetQueryScript.LeafFactory compile(String expression) {

modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java

@@ -47,7 +47,7 @@ public void testNeedsScores() {
         PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts);
 
         QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null);
-        SearchLookup lookup = new SearchLookup(index.mapperService(), shardContext::getForField);
+        SearchLookup lookup = shardContext.lookup();
 
         NumberSortScript.Factory factory = service.compile(null, "1.2", NumberSortScript.CONTEXT, Collections.emptyMap());
         NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), lookup);

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeatureFieldMapper.java

@@ -183,11 +183,11 @@ private Float objectToFloat(Object value) {
     }
 
     @Override
-    protected Float parseSourceValue(Object value, String format) {
+    public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
         if (format != null) {
             throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
         }
-        return objectToFloat(value);
+        return sourceValueFetcher(lookup, this::objectToFloat);
     }
 
     @Override

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/RankFeaturesFieldMapper.java

@@ -162,11 +162,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected Object parseSourceValue(Object value, String format) {
+    public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
         if (format != null) {
             throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
         }
-        return value;
+        return sourceValueFetcher(lookup, value -> value);
     }
 
     @Override

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapper.java

@@ -482,23 +482,24 @@ private static double objectToDouble(Object value) {
     }
 
     @Override
-    protected Double parseSourceValue(Object value, String format) {
+    public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
         if (format != null) {
             throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
         }
-
-        double doubleValue;
-        if (value.equals("")) {
-            if (nullValue == null) {
-                return null;
+        return sourceValueFetcher(lookup, value -> {
+            double doubleValue;
+            if (value.equals("")) {
+                if (nullValue == null) {
+                    return null;
+                }
+                doubleValue = nullValue;
+            } else {
+                doubleValue = objectToDouble(value);
             }
-            doubleValue = nullValue;
-        } else {
-            doubleValue = objectToDouble(value);
-        }
 
-        double scalingFactor = fieldType().getScalingFactor();
-        return Math.round(doubleValue * scalingFactor) / scalingFactor;
+            double scalingFactor = fieldType().getScalingFactor();
+            return Math.round(doubleValue * scalingFactor) / scalingFactor;
+        });
     }
 
     private static class ScaledFloatIndexFieldData extends IndexNumericFieldData {
@@ -624,5 +625,19 @@ public int docValueCount() {
             }
         }
 
+        @Override
+        public LeafValueFetcher buildFetcher(DocValueFormat format) {
+            SortedNumericDoubleValues doubles = getDoubleValues();
+            return docId -> {
+                if (false == doubles.advanceExact(docId)) {
+                    return List.of();
+                }
+                List<Object> result = new ArrayList<>(doubles.docValueCount());
+                for (int i = 0, count = doubles.docValueCount(); i < count; ++i) {
+                    result.add(format.format(doubles.nextValue()));
+                }
+                return result;
+            };
+        }
     }
 }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java

@@ -54,6 +54,7 @@
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.similarity.SimilarityProvider;
 import org.elasticsearch.index.similarity.SimilarityService;
+import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -419,7 +420,7 @@ protected void parseCreateField(ParseContext context) {
         }
 
         @Override
-        protected Object parseSourceValue(Object value, String format) {
+        public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
             throw new UnsupportedOperationException();
         }
 
@@ -465,7 +466,7 @@ protected void mergeOptions(FieldMapper other, List<String> conflicts) {
         }
 
         @Override
-        protected Object parseSourceValue(Object value, String format) {
+        public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
             throw new UnsupportedOperationException();
         }
 
@@ -588,11 +589,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected String parseSourceValue(Object value, String format) {
+    public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
         if (format != null) {
             throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
         }
-        return value.toString();
+        return sourceValueFetcher(lookup, Object::toString);
     }
 
     @Override

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/TokenCountFieldMapper.java

@@ -25,6 +25,7 @@
 import org.apache.lucene.document.FieldType;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
+import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
 import java.util.Iterator;
@@ -159,12 +160,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected String parseSourceValue(Object value, String format) {
+    public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
         if (format != null) {
             throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
         }
-
-        return value.toString();
+        return sourceValueFetcher(lookup, Object::toString); 
     }
 
     /**

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/RankFeatureFieldMapperTests.java

@@ -41,6 +41,9 @@
 import java.util.Collection;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.closeTo;
+import static org.hamcrest.Matchers.hasSize;
+
 public class RankFeatureFieldMapperTests extends FieldMapperTestCase<RankFeatureFieldMapper.Builder> {
 
     IndexService indexService;
@@ -189,12 +192,14 @@ public void testRejectMultiValuedFields() throws MapperParsingException, IOExcep
                 e.getCause().getMessage());
     }
 
-    public void testParseSourceValue() {
+    public void testFetchValues() throws IOException {
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
         Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
         RankFeatureFieldMapper mapper = new RankFeatureFieldMapper.Builder("field").build(context);
 
-        assertEquals(3.14f, mapper.parseSourceValue(3.14, null), 0.0001);
-        assertEquals(42.9f, mapper.parseSourceValue("42.9", null), 0.0001);
+        assertThat(fetchFromSource(mapper, null, 3.14), hasSize(1));
+        assertThat(((Number) fetchFromSource(mapper, null, 3.14).get(0)).doubleValue(), closeTo(3.14, 0.0001));
+        assertThat(fetchFromSource(mapper, null, "42.9"), hasSize(1));
+        assertThat(((Number) fetchFromSource(mapper, null, "42.9").get(0)).doubleValue(), closeTo(42.9, 0.0001));
     }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/ScaledFloatFieldMapperTests.java

@@ -32,7 +32,6 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
 import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.search.lookup.SourceLookup;
 import org.elasticsearch.test.InternalSettingsPlugin;
 import org.junit.Before;
 
@@ -43,7 +42,10 @@
 import java.util.List;
 import java.util.Set;
 
+import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
 
 public class ScaledFloatFieldMapperTests extends FieldMapperTestCase<ScaledFloatFieldMapper.Builder> {
 
@@ -403,25 +405,27 @@ public void testMeta() throws Exception {
         assertEquals(mapping3, mapper.mappingSource().toString());
     }
 
-    public void testParseSourceValue() {
+    public void testFetchValues() throws IOException {
         Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
         Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
 
         ScaledFloatFieldMapper mapper = new ScaledFloatFieldMapper.Builder("field")
             .scalingFactor(100)
             .build(context);
-        assertEquals(3.14, mapper.parseSourceValue(3.1415926, null), 0.00001);
-        assertEquals(3.14, mapper.parseSourceValue("3.1415", null), 0.00001);
-        assertNull(mapper.parseSourceValue("", null));
+        assertThat(fetchFromSource(mapper, null, 3.1415926), hasSize(1));
+        assertThat(((Double) fetchFromSource(mapper, null, 3.1415926).get(0)).doubleValue(), closeTo(3.14, 0.00001));
+        assertThat(fetchFromSource(mapper, null, "3.1415"), hasSize(1));
+        assertThat(((Double) fetchFromSource(mapper, null, "3.1415").get(0)).doubleValue(), closeTo(3.14, 0.00001));
+        assertThat(fetchFromSource(mapper, null, ""), equalTo(List.of()));
+        assertThat(fetchFromSource(mapper, null, null), equalTo(List.of()));
 
         ScaledFloatFieldMapper nullValueMapper = new ScaledFloatFieldMapper.Builder("field")
             .scalingFactor(100)
             .nullValue(2.71)
             .build(context);
-        assertEquals(2.71, nullValueMapper.parseSourceValue("", null), 0.00001);
-
-        SourceLookup sourceLookup = new SourceLookup();
-        sourceLookup.setSource(Collections.singletonMap("field", null));
-        assertEquals(List.of(2.71), nullValueMapper.lookupValues(sourceLookup, null));
+        assertThat(fetchFromSource(nullValueMapper, null, ""), hasSize(1));
+        assertThat(((Double) fetchFromSource(nullValueMapper, null, "").get(0)).doubleValue(), closeTo(2.71, 0.00001));
+        assertThat(fetchFromSource(nullValueMapper, null, null), hasSize(1));
+        assertThat(((Double) fetchFromSource(nullValueMapper, null, null).get(0)).doubleValue(), closeTo(2.71, 0.00001));
     }
 }

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/MetaJoinFieldMapper.java

@@ -138,7 +138,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected Object parseSourceValue(Object value, String format) {
+    public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
         throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source.");
     }
 

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java

@@ -188,7 +188,7 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected Object parseSourceValue(Object value, String format) {
+    public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
         throw new UnsupportedOperationException("The " + typeName() + " field is not stored in _source.");
     }
 

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java

@@ -350,11 +350,11 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected Object parseSourceValue(Object value, String format) {
+    public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
         if (format != null) {
             throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
         }
-        return value;
+        return sourceValueFetcher(lookup, v -> v);
     }
 
     @Override

modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java

@@ -76,6 +76,7 @@
 import org.elasticsearch.index.query.QueryShardException;
 import org.elasticsearch.index.query.Rewriteable;
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
+import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
@@ -368,11 +369,11 @@ public void parse(ParseContext context) throws IOException {
     }
 
     @Override
-    protected Object parseSourceValue(Object value, String format) {
+    public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
         if (format != null) {
             throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
         }
-        return value;
+        return sourceValueFetcher(lookup, value -> value);
     }
 
     static void createQueryBuilderField(Version indexVersion, BinaryFieldMapper qbField,

modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorHighlightSubFetchPhase.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.search.fetch.subphase.highlight.Highlighter;
 import org.elasticsearch.search.fetch.subphase.highlight.SearchHighlightContext;
 import org.elasticsearch.search.internal.SearchContext;
+import org.elasticsearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -76,7 +77,7 @@ public void hitsExecute(SearchContext context, SearchHit[] hits) throws IOExcept
             PercolateQuery.QueryStore queryStore = percolateQuery.getQueryStore();
 
             LeafReaderContext percolatorLeafReaderContext = percolatorIndexSearcher.getIndexReader().leaves().get(0);
-            FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext();
+            FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(new SourceLookup());
 
             for (SearchHit hit : hits) {
                 LeafReaderContext ctx = ctxs.get(ReaderUtil.subIndex(hit.docId(), ctxs));

plugins/analysis-icu/src/main/java/org/elasticsearch/index/mapper/ICUCollationKeywordFieldMapper.java

@@ -23,6 +23,7 @@
 import com.ibm.icu.text.RawCollationKey;
 import com.ibm.icu.text.RuleBasedCollator;
 import com.ibm.icu.util.ULocale;
+
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.SortedSetDocValuesField;
@@ -740,15 +741,17 @@ protected void parseCreateField(ParseContext context) throws IOException {
     }
 
     @Override
-    protected String parseSourceValue(Object value, String format) {
+    public ValueFetcher valueFetcher(SearchLookup lookup, String format) {
         if (format != null) {
             throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
         }
 
-        String keywordValue = value.toString();
-        if (keywordValue.length() > ignoreAbove) {
-            return null;
-        }
-        return keywordValue;
+        return sourceValueFetcher(lookup, value -> {
+            String keywordValue = value.toString();
+            if (keywordValue.length() > ignoreAbove) {
+                return null;
+            }
+            return keywordValue;
+        });
     }
 }