Explicitly pass document _source to the fragments builder.

rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/20_fvh.yml

@@ -65,6 +65,9 @@ setup:
 
 ---
 "Highlight multiple documents":
+  - skip:
+      version: " - 7.99.99"
+      reason: Bug fix not yet backported
   - do:
       search:
         rest_total_hits_as_int: true
@@ -86,6 +89,9 @@ setup:
 
 ---
 "Highlight multiple nested documents":
+  - skip:
+      version: " - 7.99.99"
+      reason: Bug fix not yet backported
   - do:
       search:
         rest_total_hits_as_int: true

server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/FastVectorHighlighter.java

@@ -99,7 +99,7 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc
                     fragmentsBuilder = new SimpleFragmentsBuilder(fieldType, fixBrokenAnalysis, field.fieldOptions().preTags(),
                         field.fieldOptions().postTags(), boundaryScanner);
                 } else {
-                    fragmentsBuilder = new SourceSimpleFragmentsBuilder(fieldType, fixBrokenAnalysis, hitContext.sourceLookup(),
+                    fragmentsBuilder = new SourceSimpleFragmentsBuilder(fieldType, fixBrokenAnalysis,
                         field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner);
                 }
             } else {
@@ -110,7 +110,7 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc
                         fragmentsBuilder = new ScoreOrderFragmentsBuilder(field.fieldOptions().preTags(),
                             field.fieldOptions().postTags(), boundaryScanner);
                     } else {
-                        fragmentsBuilder = new SourceScoreOrderFragmentsBuilder(fieldType, fixBrokenAnalysis, hitContext.sourceLookup(),
+                        fragmentsBuilder = new SourceScoreOrderFragmentsBuilder(fieldType, fixBrokenAnalysis,
                             field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner);
                     }
                 } else {
@@ -119,7 +119,7 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc
                             field.fieldOptions().postTags(), boundaryScanner);
                     } else {
                         fragmentsBuilder =
-                            new SourceSimpleFragmentsBuilder(fieldType, fixBrokenAnalysis, hitContext.sourceLookup(),
+                            new SourceSimpleFragmentsBuilder(fieldType, fixBrokenAnalysis,
                                 field.fieldOptions().preTags(), field.fieldOptions().postTags(), boundaryScanner);
                     }
                 }
@@ -160,8 +160,13 @@ public HighlightField highlight(FieldHighlightContext fieldContext) throws IOExc
         }
         cache.fvh.setPhraseLimit(field.fieldOptions().phraseLimit());
 
-        String[] fragments;
+        // If the fragments builder requires document _source, pass it the source lookup from the hit context.
+        if (entry.fragmentsBuilder instanceof SourceBasedFragmentsBuilder) {
+            SourceBasedFragmentsBuilder fragmentsBuilder = ((SourceBasedFragmentsBuilder) entry.fragmentsBuilder);
+            fragmentsBuilder.setSource(hitContext.sourceLookup());
+        }
 
+        String[] fragments;
         // a HACK to make highlighter do highlighting, even though its using the single frag list builder
         int numberOfFragments = field.fieldOptions().numberOfFragments() == 0 ?
             Integer.MAX_VALUE : field.fieldOptions().numberOfFragments();

server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceBasedFragmentsBuilder.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.search.fetch.subphase.highlight;
+
+import org.apache.lucene.search.vectorhighlight.FragmentsBuilder;
+import org.elasticsearch.search.lookup.SourceLookup;
+
+public interface SourceBasedFragmentsBuilder extends FragmentsBuilder {
+    /**
+     * Set the source lookup of the current document. Note that this
+     * must be called before creating fragments.
+     */
+    void setSource(SourceLookup sourceLookup);
+}

server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceScoreOrderFragmentsBuilder.java

@@ -31,28 +31,32 @@
 import java.io.IOException;
 import java.util.List;
 
-public class SourceScoreOrderFragmentsBuilder extends ScoreOrderFragmentsBuilder {
+public class SourceScoreOrderFragmentsBuilder extends ScoreOrderFragmentsBuilder implements SourceBasedFragmentsBuilder {
 
     private final MappedFieldType fieldType;
-    private final SourceLookup sourceLookup;
+    private SourceLookup sourceLookup;
     private final boolean fixBrokenAnalysis;
 
     public SourceScoreOrderFragmentsBuilder(MappedFieldType fieldType,
                                             boolean fixBrokenAnalysis,
-                                            SourceLookup sourceLookup,
                                             String[] preTags,
                                             String[] postTags,
                                             BoundaryScanner boundaryScanner) {
         super(preTags, postTags, boundaryScanner);
         this.fieldType = fieldType;
-        this.sourceLookup = sourceLookup;
         this.fixBrokenAnalysis = fixBrokenAnalysis;
     }
 
+    @Override
+    public void setSource(SourceLookup sourceLookup) {
+        this.sourceLookup = sourceLookup;
+    }
+
     @Override
     protected Field[] getFields(IndexReader reader, int docId, String fieldName) throws IOException {
-        // we know its low level reader, and matching docId, since that's how we call the highlighter with
+        assert sourceLookup != null;
         List<Object> values = sourceLookup.extractRawValues(fieldType.name());
+
         Field[] fields = new Field[values.size()];
         for (int i = 0; i < values.size(); i++) {
             fields[i] = new Field(fieldType.name(), values.get(i).toString(), TextField.TYPE_NOT_STORED);

server/src/main/java/org/elasticsearch/search/fetch/subphase/highlight/SourceSimpleFragmentsBuilder.java

@@ -28,25 +28,27 @@
 import java.io.IOException;
 import java.util.List;
 
-public class SourceSimpleFragmentsBuilder extends SimpleFragmentsBuilder {
-
-    private final SourceLookup sourceLookup;
+public class SourceSimpleFragmentsBuilder extends SimpleFragmentsBuilder implements SourceBasedFragmentsBuilder {
+    private SourceLookup sourceLookup;
 
     public SourceSimpleFragmentsBuilder(MappedFieldType fieldType,
                                         boolean fixBrokenAnalysis,
-                                        SourceLookup sourceLookup,
                                         String[] preTags,
                                         String[] postTags,
                                         BoundaryScanner boundaryScanner) {
         super(fieldType, fixBrokenAnalysis, preTags, postTags, boundaryScanner);
-        this.sourceLookup = sourceLookup;
     }
 
     public static final Field[] EMPTY_FIELDS = new Field[0];
 
+    @Override
+    public void setSource(SourceLookup sourceLookup) {
+        this.sourceLookup = sourceLookup;
+    }
+
     @Override
     protected Field[] getFields(IndexReader reader, int docId, String fieldName) throws IOException {
-        // we know its low level reader, and matching docId, since that's how we call the highlighter with
+        assert sourceLookup != null;
         List<Object> values = sourceLookup.extractRawValues(fieldType.name());
         if (values.isEmpty()) {
             return EMPTY_FIELDS;
@@ -57,5 +59,4 @@ protected Field[] getFields(IndexReader reader, int docId, String fieldName) thr
         }
         return fields;
     }
-
 }