Add sourceMode to SourceBlockLoader so that the right StoredFieldsSpe…

…c can be selected.
martijnvg · Oct 17, 2024 · 6702a2e · 6702a2e
1 parent 980675e
commit 6702a2e
Show file tree

Hide file tree

Showing 13 changed files with 96 additions and 45 deletions.
diff --git a/...-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/...-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java
@@ -364,7 +364,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
             SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
             // MatchOnlyText never has norms, so we have to use the field names field
             BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name());
-            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup);
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup, sourceMode);
         }
 
         @Override

diff --git a/...er-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/...er-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java
@@ -319,7 +319,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
             BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
                 ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
                 : BlockSourceReader.lookupMatchingAll();
-            return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup);
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup, sourceMode);
         }
 
         @Override

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java
@@ -189,7 +189,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
         protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) {
             ValueFetcher fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB);
             // TODO consider optimization using BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
-            return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll());
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll(), sourceMode);
         }
 
         protected abstract Object nullValueAsSource(T nullValue);

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java
@@ -101,10 +101,12 @@ public interface LeafIteratorLookup {
     private abstract static class SourceBlockLoader implements BlockLoader {
         protected final ValueFetcher fetcher;
         private final LeafIteratorLookup lookup;
+        private final SourceFieldMapper.Mode sourceMode;
 
-        private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
+        private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
             this.fetcher = fetcher;
             this.lookup = lookup;
+            this.sourceMode = sourceMode;
         }
 
         @Override
@@ -114,7 +116,7 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context)
 
         @Override
         public final StoredFieldsSpec rowStrideStoredFieldSpec() {
-            return NEEDS_SOURCE_AND_IGNORED_SOURCE;
+            return sourceMode == SourceFieldMapper.Mode.SYNTHETIC ? NEEDS_SOURCE_AND_IGNORED_SOURCE : StoredFieldsSpec.NEEDS_SOURCE;
         }
 
         @Override
@@ -150,8 +152,8 @@ public final String toString() {
      * Load {@code boolean}s from {@code _source}.
      */
     public static class BooleansBlockLoader extends SourceBlockLoader {
-        public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
-            super(fetcher, lookup);
+        public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
+            super(fetcher, lookup, sourceMode);
         }
 
         @Override
@@ -190,8 +192,8 @@ public String toString() {
      * Load {@link BytesRef}s from {@code _source}.
      */
     public static class BytesRefsBlockLoader extends SourceBlockLoader {
-        public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
-            super(fetcher, lookup);
+        public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
+            super(fetcher, lookup, sourceMode);
         }
 
         @Override
@@ -201,7 +203,7 @@ public final Builder builder(BlockFactory factory, int expectedCount) {
 
         @Override
         protected RowStrideReader rowStrideReader(LeafReaderContext context, DocIdSetIterator iter) throws IOException {
-            return new BytesRefs(fetcher, iter);
+            return new BytesRefs(fetcher, iter, null);
         }
 
         @Override
@@ -211,8 +213,8 @@ protected String name() {
     }
 
     public static class GeometriesBlockLoader extends SourceBlockLoader {
-        public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
-            super(fetcher, lookup);
+        public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
+            super(fetcher, lookup, sourceMode);
         }
 
         @Override
@@ -222,7 +224,7 @@ public final Builder builder(BlockFactory factory, int expectedCount) {
 
         @Override
         protected RowStrideReader rowStrideReader(LeafReaderContext context, DocIdSetIterator iter) {
-            return new Geometries(fetcher, iter);
+            return new Geometries(fetcher, iter, null);
         }
 
         @Override
@@ -234,7 +236,7 @@ protected String name() {
     private static class BytesRefs extends BlockSourceReader {
         private final BytesRef scratch = new BytesRef();
 
-        BytesRefs(ValueFetcher fetcher, DocIdSetIterator iter) {
+        BytesRefs(ValueFetcher fetcher, DocIdSetIterator iter, SourceFieldMapper.Mode sourceMode) {
             super(fetcher, iter);
         }
 
@@ -251,7 +253,7 @@ public String toString() {
 
     private static class Geometries extends BlockSourceReader {
 
-        Geometries(ValueFetcher fetcher, DocIdSetIterator iter) {
+        Geometries(ValueFetcher fetcher, DocIdSetIterator iter, SourceFieldMapper.Mode sourceMode) {
             super(fetcher, iter);
         }
 
@@ -274,8 +276,8 @@ public String toString() {
      * Load {@code double}s from {@code _source}.
      */
     public static class DoublesBlockLoader extends SourceBlockLoader {
-        public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
-            super(fetcher, lookup);
+        public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
+            super(fetcher, lookup, sourceMode);
         }
 
         @Override
@@ -314,8 +316,8 @@ public String toString() {
      * Load {@code int}s from {@code _source}.
      */
     public static class IntsBlockLoader extends SourceBlockLoader {
-        public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
-            super(fetcher, lookup);
+        public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
+            super(fetcher, lookup, sourceMode);
         }
 
         @Override
@@ -354,8 +356,8 @@ public String toString() {
      * Load {@code long}s from {@code _source}.
      */
     public static class LongsBlockLoader extends SourceBlockLoader {
-        public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) {
-            super(fetcher, lookup);
+        public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) {
+            super(fetcher, lookup, sourceMode);
         }
 
         @Override

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java
@@ -314,7 +314,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
             BlockSourceReader.LeafIteratorLookup lookup = isIndexed() || isStored()
                 ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
                 : BlockSourceReader.lookupMatchingAll();
-            return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup);
+            return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup, blContext.indexSettings().getIndexMappingSourceMode());
         }
 
         @Override

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
@@ -792,7 +792,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
             BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
                 ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
                 : BlockSourceReader.lookupMatchingAll();
-            return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup);
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, sourceMode);
         }
 
         @Override

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
@@ -636,7 +636,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
                 return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name());
             }
             SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name()));
-            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext));
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext), sourceMode);
         }
 
         private BlockSourceReader.LeafIteratorLookup sourceBlockLoaderLookup(BlockLoaderContext blContext) {

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java
@@ -461,8 +461,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
             }
 
             @Override
-            BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
-                return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
+            BlockLoader blockLoaderFromSource(
+                SourceValueFetcher sourceValueFetcher,
+                BlockSourceReader.LeafIteratorLookup lookup,
+                SourceFieldMapper.Mode sourceMode
+            ) {
+                return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, sourceMode);
             }
         },
         FLOAT("float", NumericType.FLOAT) {
@@ -645,8 +649,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
             }
 
             @Override
-            BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
-                return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
+            BlockLoader blockLoaderFromSource(
+                SourceValueFetcher sourceValueFetcher,
+                BlockSourceReader.LeafIteratorLookup lookup,
+                SourceFieldMapper.Mode sourceMode
+            ) {
+                return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, sourceMode);
             }
         },
         DOUBLE("double", NumericType.DOUBLE) {
@@ -795,8 +803,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
             }
 
             @Override
-            BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
-                return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup);
+            BlockLoader blockLoaderFromSource(
+                SourceValueFetcher sourceValueFetcher,
+                BlockSourceReader.LeafIteratorLookup lookup,
+                SourceFieldMapper.Mode sourceMode
+            ) {
+                return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, sourceMode);
             }
         },
         BYTE("byte", NumericType.BYTE) {
@@ -908,8 +920,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
             }
 
             @Override
-            BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
-                return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
+            BlockLoader blockLoaderFromSource(
+                SourceValueFetcher sourceValueFetcher,
+                BlockSourceReader.LeafIteratorLookup lookup,
+                SourceFieldMapper.Mode sourceMode
+            ) {
+                return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, sourceMode);
             }
 
             private boolean isOutOfRange(Object value) {
@@ -1021,8 +1037,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
             }
 
             @Override
-            BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
-                return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
+            BlockLoader blockLoaderFromSource(
+                SourceValueFetcher sourceValueFetcher,
+                BlockSourceReader.LeafIteratorLookup lookup,
+                SourceFieldMapper.Mode sourceMode
+            ) {
+                return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, sourceMode);
             }
 
             private boolean isOutOfRange(Object value) {
@@ -1208,8 +1228,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
             }
 
             @Override
-            BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
-                return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup);
+            BlockLoader blockLoaderFromSource(
+                SourceValueFetcher sourceValueFetcher,
+                BlockSourceReader.LeafIteratorLookup lookup,
+                SourceFieldMapper.Mode sourceMode
+            ) {
+                return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, sourceMode);
             }
         },
         LONG("long", NumericType.LONG) {
@@ -1355,8 +1379,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) {
             }
 
             @Override
-            BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) {
-                return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup);
+            BlockLoader blockLoaderFromSource(
+                SourceValueFetcher sourceValueFetcher,
+                BlockSourceReader.LeafIteratorLookup lookup,
+                SourceFieldMapper.Mode sourceMode
+            ) {
+                return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup, sourceMode);
             }
 
             private boolean isOutOfRange(Object value) {
@@ -1634,7 +1662,11 @@ protected void writeValue(XContentBuilder b, long value) throws IOException {
 
         abstract BlockLoader blockLoaderFromDocValues(String fieldName);
 
-        abstract BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup);
+        abstract BlockLoader blockLoaderFromSource(
+            SourceValueFetcher sourceValueFetcher,
+            BlockSourceReader.LeafIteratorLookup lookup,
+            SourceFieldMapper.Mode sourceMode
+        );
     }
 
     public static class NumberFieldType extends SimpleMappedFieldType {
@@ -1773,7 +1805,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) {
             BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
                 ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
                 : BlockSourceReader.lookupMatchingAll();
-            return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup);
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup, sourceMode);
         }
 
         @Override

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
@@ -1022,7 +1022,8 @@ protected String delegatingTo() {
                 return null;
             }
             SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
-            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext));
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext), sourceMode);
         }
 
         /**

diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java
@@ -50,7 +50,7 @@ public void testEmptyArray() throws IOException {
     private void loadBlock(LeafReaderContext ctx, Consumer<TestBlock> test) throws IOException {
         ValueFetcher valueFetcher = SourceValueFetcher.toString(Set.of("field"));
         BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromNorms("field");
-        BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup);
+        BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup, null);
         assertThat(loader.columnAtATimeReader(ctx), nullValue());
         BlockLoader.RowStrideReader reader = loader.rowStrideReader(ctx);
         assertThat(loader.rowStrideStoredFieldSpec(), equalTo(BlockSourceReader.NEEDS_SOURCE_AND_IGNORED_SOURCE));

diff --git a/.../src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java b/.../src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java
@@ -26,6 +26,7 @@
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
@@ -546,7 +547,10 @@ public String indexName() {
 
             @Override
             public IndexSettings indexSettings() {
-                throw new UnsupportedOperationException();
+                var imd = IndexMetadata.builder("test_index")
+                    .settings(ValueSourceReaderTypeConversionTests.indexSettings(IndexVersion.current(), 1, 1).put(Settings.EMPTY))
+                    .build();
+                return new IndexSettings(imd, Settings.EMPTY);
             }
 
             @Override

diff --git a/...mpute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java b/...mpute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java
@@ -24,9 +24,11 @@
 import org.apache.lucene.tests.mockfile.HandleLimitFS;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
@@ -500,7 +502,10 @@ public String indexName() {
 
             @Override
             public IndexSettings indexSettings() {
-                throw new UnsupportedOperationException();
+                var imd = IndexMetadata.builder("test_index")
+                    .settings(ValueSourceReaderTypeConversionTests.indexSettings(IndexVersion.current(), 1, 1).put(Settings.EMPTY))
+                    .build();
+                return new IndexSettings(imd, Settings.EMPTY);
             }
 
             @Override

diff --git a/...gned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/...gned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java
@@ -339,7 +339,8 @@ protected Object parseSourceValue(Object value) {
             BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed()
                 ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name())
                 : BlockSourceReader.lookupMatchingAll();
-            return new BlockSourceReader.LongsBlockLoader(valueFetcher, lookup);
+            var sourceMode = blContext.indexSettings().getIndexMappingSourceMode();
+            return new BlockSourceReader.LongsBlockLoader(valueFetcher, lookup, sourceMode);
         }
 
         @Override