diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 5904169308fab..cd252fcff2376 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -364,7 +364,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); // MatchOnlyText never has norms, so we have to use the field names field BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup); + var sourceMode = blContext.indexSettings().getIndexMappingSourceMode(); + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, lookup, sourceMode); } @Override diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index b845545133e19..1f647cb977cf5 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -319,7 +319,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup); + var sourceMode = blContext.indexSettings().getIndexMappingSourceMode(); + return new BlockSourceReader.DoublesBlockLoader(valueFetcher, lookup, sourceMode); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index c38b5beeb55a0..3512989c115ee 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -189,7 +189,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { protected BlockLoader blockLoaderFromSource(BlockLoaderContext blContext) { ValueFetcher fetcher = valueFetcher(blContext.sourcePaths(name()), nullValue, GeometryFormatterFactory.WKB); // TODO consider optimization using BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) - return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll()); + var sourceMode = blContext.indexSettings().getIndexMappingSourceMode(); + return new BlockSourceReader.GeometriesBlockLoader(fetcher, BlockSourceReader.lookupMatchingAll(), sourceMode); } protected abstract Object nullValueAsSource(T nullValue); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java index 19a1cce746172..105943c732a5e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BlockSourceReader.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Set; /** * Loads values from {@code _source}. This whole process is very slow and cast-tastic, @@ -29,6 +30,14 @@ * slow. */ public abstract class BlockSourceReader implements BlockLoader.RowStrideReader { + + // _ignored_source is needed when source mode is synthetic. + static final StoredFieldsSpec NEEDS_SOURCE_AND_IGNORED_SOURCE = new StoredFieldsSpec( + true, + false, + Set.of(IgnoredSourceFieldMapper.NAME) + ); + private final ValueFetcher fetcher; private final List ignoredValues = new ArrayList<>(); private final DocIdSetIterator iter; @@ -91,10 +100,12 @@ public interface LeafIteratorLookup { private abstract static class SourceBlockLoader implements BlockLoader { protected final ValueFetcher fetcher; private final LeafIteratorLookup lookup; + private final SourceFieldMapper.Mode sourceMode; - private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { + private SourceBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) { this.fetcher = fetcher; this.lookup = lookup; + this.sourceMode = sourceMode; } @Override @@ -104,7 +115,7 @@ public final ColumnAtATimeReader columnAtATimeReader(LeafReaderContext context) @Override public final StoredFieldsSpec rowStrideStoredFieldSpec() { - return StoredFieldsSpec.NEEDS_SOURCE; + return sourceMode == SourceFieldMapper.Mode.SYNTHETIC ? NEEDS_SOURCE_AND_IGNORED_SOURCE : StoredFieldsSpec.NEEDS_SOURCE; } @Override @@ -140,8 +151,8 @@ public final String toString() { * Load {@code boolean}s from {@code _source}. */ public static class BooleansBlockLoader extends SourceBlockLoader { - public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { - super(fetcher, lookup); + public BooleansBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) { + super(fetcher, lookup, sourceMode); } @Override @@ -180,8 +191,8 @@ public String toString() { * Load {@link BytesRef}s from {@code _source}. */ public static class BytesRefsBlockLoader extends SourceBlockLoader { - public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { - super(fetcher, lookup); + public BytesRefsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) { + super(fetcher, lookup, sourceMode); } @Override @@ -191,7 +202,7 @@ public final Builder builder(BlockFactory factory, int expectedCount) { @Override protected RowStrideReader rowStrideReader(LeafReaderContext context, DocIdSetIterator iter) throws IOException { - return new BytesRefs(fetcher, iter); + return new BytesRefs(fetcher, iter, null); } @Override @@ -201,8 +212,8 @@ protected String name() { } public static class GeometriesBlockLoader extends SourceBlockLoader { - public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { - super(fetcher, lookup); + public GeometriesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) { + super(fetcher, lookup, sourceMode); } @Override @@ -212,7 +223,7 @@ public final Builder builder(BlockFactory factory, int expectedCount) { @Override protected RowStrideReader rowStrideReader(LeafReaderContext context, DocIdSetIterator iter) { - return new Geometries(fetcher, iter); + return new Geometries(fetcher, iter, null); } @Override @@ -224,7 +235,7 @@ protected String name() { private static class BytesRefs extends BlockSourceReader { private final BytesRef scratch = new BytesRef(); - BytesRefs(ValueFetcher fetcher, DocIdSetIterator iter) { + BytesRefs(ValueFetcher fetcher, DocIdSetIterator iter, SourceFieldMapper.Mode sourceMode) { super(fetcher, iter); } @@ -241,7 +252,7 @@ public String toString() { private static class Geometries extends BlockSourceReader { - Geometries(ValueFetcher fetcher, DocIdSetIterator iter) { + Geometries(ValueFetcher fetcher, DocIdSetIterator iter, SourceFieldMapper.Mode sourceMode) { super(fetcher, iter); } @@ -264,8 +275,8 @@ public String toString() { * Load {@code double}s from {@code _source}. */ public static class DoublesBlockLoader extends SourceBlockLoader { - public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { - super(fetcher, lookup); + public DoublesBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) { + super(fetcher, lookup, sourceMode); } @Override @@ -304,8 +315,8 @@ public String toString() { * Load {@code int}s from {@code _source}. */ public static class IntsBlockLoader extends SourceBlockLoader { - public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { - super(fetcher, lookup); + public IntsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) { + super(fetcher, lookup, sourceMode); } @Override @@ -344,8 +355,8 @@ public String toString() { * Load {@code long}s from {@code _source}. */ public static class LongsBlockLoader extends SourceBlockLoader { - public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup) { - super(fetcher, lookup); + public LongsBlockLoader(ValueFetcher fetcher, LeafIteratorLookup lookup, SourceFieldMapper.Mode sourceMode) { + super(fetcher, lookup, sourceMode); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 5aaaf7dce83c9..c2bf9e18bfeec 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -314,7 +314,7 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { BlockSourceReader.LeafIteratorLookup lookup = isIndexed() || isStored() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup); + return new BlockSourceReader.BooleansBlockLoader(fetcher, lookup, blContext.indexSettings().getIndexMappingSourceMode()); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 012f08a1db01d..4b26b70affa45 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -809,7 +809,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup); + var sourceMode = blContext.indexSettings().getIndexMappingSourceMode(); + return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher(blContext.sourcePaths(name())), lookup, sourceMode); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 529ff19bfffd7..1ff9fd2f699c9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -632,18 +632,12 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { if (hasDocValues()) { return new BlockDocValuesReader.BytesRefsFromOrdsBlockLoader(name()); } - if (isSyntheticSource) { - if (false == isStored()) { - throw new IllegalStateException( - "keyword field [" - + name() - + "] is only supported in synthetic _source index if it creates doc values or stored fields" - ); - } + if (isStored()) { return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(name()); } SourceValueFetcher fetcher = sourceValueFetcher(blContext.sourcePaths(name())); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext)); + var sourceMode = blContext.indexSettings().getIndexMappingSourceMode(); + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, sourceBlockLoaderLookup(blContext), sourceMode); } private BlockSourceReader.LeafIteratorLookup sourceBlockLoaderLookup(BlockLoaderContext blContext) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 8cc67cc481b9b..2e815554dc829 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -461,8 +461,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { - return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); + BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + SourceFieldMapper.Mode sourceMode + ) { + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, sourceMode); } }, FLOAT("float", NumericType.FLOAT) { @@ -645,8 +649,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { - return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); + BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + SourceFieldMapper.Mode sourceMode + ) { + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, sourceMode); } }, DOUBLE("double", NumericType.DOUBLE) { @@ -795,8 +803,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { - return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup); + BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + SourceFieldMapper.Mode sourceMode + ) { + return new BlockSourceReader.DoublesBlockLoader(sourceValueFetcher, lookup, sourceMode); } }, BYTE("byte", NumericType.BYTE) { @@ -908,8 +920,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { - return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); + BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + SourceFieldMapper.Mode sourceMode + ) { + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, sourceMode); } private boolean isOutOfRange(Object value) { @@ -1021,8 +1037,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { - return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); + BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + SourceFieldMapper.Mode sourceMode + ) { + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, sourceMode); } private boolean isOutOfRange(Object value) { @@ -1208,8 +1228,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { - return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup); + BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + SourceFieldMapper.Mode sourceMode + ) { + return new BlockSourceReader.IntsBlockLoader(sourceValueFetcher, lookup, sourceMode); } }, LONG("long", NumericType.LONG) { @@ -1355,8 +1379,12 @@ BlockLoader blockLoaderFromDocValues(String fieldName) { } @Override - BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup) { - return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup); + BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + SourceFieldMapper.Mode sourceMode + ) { + return new BlockSourceReader.LongsBlockLoader(sourceValueFetcher, lookup, sourceMode); } private boolean isOutOfRange(Object value) { @@ -1634,7 +1662,11 @@ protected void writeValue(XContentBuilder b, long value) throws IOException { abstract BlockLoader blockLoaderFromDocValues(String fieldName); - abstract BlockLoader blockLoaderFromSource(SourceValueFetcher sourceValueFetcher, BlockSourceReader.LeafIteratorLookup lookup); + abstract BlockLoader blockLoaderFromSource( + SourceValueFetcher sourceValueFetcher, + BlockSourceReader.LeafIteratorLookup lookup, + SourceFieldMapper.Mode sourceMode + ); } public static class NumberFieldType extends SimpleMappedFieldType { @@ -1773,7 +1805,8 @@ public BlockLoader blockLoader(BlockLoaderContext blContext) { BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup); + var sourceMode = blContext.indexSettings().getIndexMappingSourceMode(); + return type.blockLoaderFromSource(sourceValueFetcher(blContext.sourcePaths(name())), lookup, sourceMode); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 2c55fc35db57d..0a3911a73a2fc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -1012,17 +1012,20 @@ protected String delegatingTo() { if (isStored()) { return new BlockStoredFieldsReader.BytesFromStringsBlockLoader(name()); } - if (isSyntheticSource) { + if (isSyntheticSource && syntheticSourceDelegate == null) { /* * When we're in synthetic source mode we don't currently * support text fields that are not stored and are not children * of perfect keyword fields. We'd have to load from the parent - * field and then convert the result to a string. + * field and then convert the result to a string. In this case, + * even if we would synthesize the source, the current field + * would be missing. */ return null; } SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); - return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext)); + var sourceMode = blContext.indexSettings().getIndexMappingSourceMode(); + return new BlockSourceReader.BytesRefsBlockLoader(fetcher, blockReaderDisiLookup(blContext), sourceMode); } /** diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java index 357ada3ad656d..286be8d12570d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BlockSourceReaderTests.java @@ -51,7 +51,7 @@ public void testEmptyArray() throws IOException { private void loadBlock(LeafReaderContext ctx, Consumer test) throws IOException { ValueFetcher valueFetcher = SourceValueFetcher.toString(Set.of("field")); BlockSourceReader.LeafIteratorLookup lookup = BlockSourceReader.lookupFromNorms("field"); - BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup); + BlockLoader loader = new BlockSourceReader.BytesRefsBlockLoader(valueFetcher, lookup, null); assertThat(loader.columnAtATimeReader(ctx), nullValue()); BlockLoader.RowStrideReader reader = loader.rowStrideReader(ctx); assertThat(loader.rowStrideStoredFieldSpec(), equalTo(StoredFieldsSpec.NEEDS_SOURCE)); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 61ffbfe1fee37..74a129e810050 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -1337,12 +1337,15 @@ private BlockLoader getBlockLoader(boolean columnReader) { return mapper.fieldType(loaderFieldName).blockLoader(new MappedFieldType.BlockLoaderContext() { @Override public String indexName() { - throw new UnsupportedOperationException(); + return "test_index"; } @Override public IndexSettings indexSettings() { - throw new UnsupportedOperationException(); + var imd = IndexMetadata.builder(indexName()) + .settings(MapperTestCase.indexSettings(IndexVersion.current(), 1, 1).put(Settings.EMPTY)) + .build(); + return new IndexSettings(imd, Settings.EMPTY); } @Override diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java index ccc3dea78adc8..f6d81af7c14e5 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValueSourceReaderTypeConversionTests.java @@ -26,6 +26,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.NoopCircuitBreaker; @@ -546,7 +547,10 @@ public String indexName() { @Override public IndexSettings indexSettings() { - throw new UnsupportedOperationException(); + var imd = IndexMetadata.builder("test_index") + .settings(ValueSourceReaderTypeConversionTests.indexSettings(IndexVersion.current(), 1, 1).put(Settings.EMPTY)) + .build(); + return new IndexSettings(imd, Settings.EMPTY); } @Override diff --git a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java index 848415c4490fa..c8dd6f87be5fc 100644 --- a/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java +++ b/x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/ValuesSourceReaderOperatorTests.java @@ -24,9 +24,11 @@ import org.apache.lucene.tests.mockfile.HandleLimitFS; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.ByteSizeValue; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BlockFactory; @@ -500,7 +502,10 @@ public String indexName() { @Override public IndexSettings indexSettings() { - throw new UnsupportedOperationException(); + var imd = IndexMetadata.builder("test_index") + .settings(ValueSourceReaderTypeConversionTests.indexSettings(IndexVersion.current(), 1, 1).put(Settings.EMPTY)) + .build(); + return new IndexSettings(imd, Settings.EMPTY); } @Override diff --git a/x-pack/plugin/logsdb/build.gradle b/x-pack/plugin/logsdb/build.gradle index 929d7dad2f5e6..60578f832d153 100644 --- a/x-pack/plugin/logsdb/build.gradle +++ b/x-pack/plugin/logsdb/build.gradle @@ -25,7 +25,7 @@ base { restResources { restApi { - include 'bulk', 'search', '_common', 'indices', 'index', 'cluster', 'data_stream', 'ingest', 'cat', 'capabilities' + include 'bulk', 'search', '_common', 'indices', 'index', 'cluster', 'data_stream', 'ingest', 'cat', 'capabilities', 'esql.query' } } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java index dd7806fc9c8fa..8d7a813b206d8 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java @@ -301,6 +301,21 @@ public void testEsqlTermsAggregation() throws IOException { assertTrue(matchResult.getMessage(), matchResult.isMatch()); } + public void testEsqlTermsAggregationByMethod() throws IOException { + int numberOfDocuments = ESTestCase.randomIntBetween(100, 200); + final List documents = generateDocuments(numberOfDocuments); + + indexDocuments(documents); + + final String query = "FROM $index | STATS count(*) BY method | SORT method | LIMIT " + numberOfDocuments; + final MatchResult matchResult = Matcher.mappings(getContenderMappings(), getBaselineMappings()) + .settings(getContenderSettings(), getBaselineSettings()) + .expected(getEsqlStatsResults(esqlBaseline(query))) + .ignoringSort(true) + .isEqualTo(getEsqlStatsResults(esqlContender(query))); + assertTrue(matchResult.getMessage(), matchResult.isMatch()); + } + public void testFieldCaps() throws IOException { int numberOfDocuments = ESTestCase.randomIntBetween(20, 50); final List documents = generateDocuments(numberOfDocuments); diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/50_esql_synthetic_source_disabled_fields.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/50_esql_synthetic_source_disabled_fields.yml new file mode 100644 index 0000000000000..68597afda6c78 --- /dev/null +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/50_esql_synthetic_source_disabled_fields.yml @@ -0,0 +1,305 @@ +--- +setup: + - requires: + test_runner_features: allowed_warnings_regex + + - do: + indices.create: + index: my-index + body: + settings: + index: + mode: logsdb + mappings: + properties: + "@timestamp": + type: date + host.name: + type: keyword + agent_id: + type: keyword + doc_values: false + store: false + process_id: + type: integer + doc_values: false + store: false + http_method: + type: keyword + doc_values: false + store: false + is_https: + type: boolean + doc_values: false + store: false + location: + type: geo_point + doc_values: false + store: false + message: + type: text + store: false + fields: + raw: + type: keyword + + - do: + bulk: + index: my-index + refresh: true + body: + - { "index": { } } + - { "@timestamp": "2024-02-12T10:30:00Z", "host.name": "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "No, I am your father." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "Do. Or do not. There is no try." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "May the force be with you." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "is_https": true, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "I find your lack of faith disturbing." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "Wars not make one great." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "That's no moon. It's a space station." } + +--- +teardown: + - do: + indices.delete: + index: my-index + +--- +"Simple from": + - do: + esql.query: + body: + query: 'FROM my-index | SORT host.name, @timestamp | LIMIT 1' + + - match: {columns.0.name: "@timestamp"} + - match: {columns.0.type: "date"} + - match: {columns.1.name: "agent_id"} + - match: {columns.1.type: "keyword"} + - match: {columns.2.name: "host.name"} + - match: {columns.2.type: "keyword"} + - match: {columns.3.name: "http_method" } + - match: {columns.3.type: "keyword" } + - match: {columns.4.name: "is_https"} + - match: {columns.4.type: "boolean"} + - match: {columns.5.name: "location"} + - match: {columns.5.type: "geo_point"} + - match: {columns.6.name: "message"} + - match: {columns.6.type: "text"} + - match: {columns.7.name: "message.raw"} + - match: {columns.7.type: "keyword"} + - match: {columns.8.name: "process_id"} + - match: {columns.8.type: "integer"} + + - match: {values.0.0: "2024-02-12T10:31:00.000Z"} + - match: {values.0.1: "yoda"} + - match: {values.0.2: "bar"} + - match: {values.0.3: "PUT"} + - match: {values.0.4: false} + - match: {values.0.5: "POINT (-74.006 40.7128)"} + - match: {values.0.6: "Do. Or do not. There is no try."} + - match: {values.0.7: "Do. Or do not. There is no try."} + - match: {values.0.8: 102} + +--- +"Simple from geo point": + - do: + esql.query: + body: + query: 'FROM my-index | SORT host.name, @timestamp | KEEP location | LIMIT 10' + + - match: {columns.0.name: "location"} + - match: {columns.0.type: "geo_point"} + + - match: {values.0.0: "POINT (-74.006 40.7128)"} + - match: {values.1.0: "POINT (-74.006 40.7128)"} + - match: {values.2.0: "POINT (-74.006 40.7128)"} + - match: {values.3.0: "POINT (-74.006 40.7128)"} + - match: {values.4.0: "POINT (-74.006 40.7128)"} + - match: {values.5.0: "POINT (-74.006 40.7128)"} + +--- +"Simple from number fields": + - do: + esql.query: + body: + query: 'FROM my-index | SORT host.name, @timestamp | KEEP process_id | LIMIT 10' + + - match: {columns.0.name: "process_id"} + - match: {columns.0.type: "integer"} + + - match: {values.0.0: 102} + - match: {values.1.0: 102} + - match: {values.2.0: 104} + - match: {values.3.0: 101} + - match: {values.4.0: 103} + - match: {values.5.0: 105} + +--- +"Simple from keyword fields": + - do: + esql.query: + body: + query: 'FROM my-index | SORT host.name, @timestamp | KEEP agent_id, http_method | LIMIT 10' + + - match: {columns.0.name: "agent_id"} + - match: {columns.0.type: "keyword"} + - match: {columns.1.name: "http_method"} + - match: {columns.1.type: "keyword"} + + - match: {values.0.0: "yoda"} + - match: {values.0.1: "PUT"} + - match: {values.1.0: "darth-vader"} + - match: {values.1.1: "POST"} + - match: {values.2.0: "yoda"} + - match: {values.2.1: "POST"} + - match: {values.3.0: "darth-vader"} + - match: {values.3.1: "GET"} + - match: {values.4.0: "obi-wan"} + - match: {values.4.1: "GET"} + - match: {values.5.0: "obi-wan"} + - match: {values.5.1: "GET"} + +--- +"Simple from boolean fields": + - do: + esql.query: + body: + query: 'FROM my-index | SORT host.name, @timestamp | KEEP is_https | LIMIT 10' + + - match: {columns.0.name: "is_https"} + - match: {columns.0.type: "boolean"} + + - match: {values.0.0: false} + - match: {values.1.0: true} + - match: {values.2.0: false} + - match: {values.3.0: false} + - match: {values.4.0: false} + - match: {values.5.0: false} + +--- +"Simple from text fields": + - do: + esql.query: + body: + query: 'FROM my-index | SORT host.name, @timestamp | KEEP message | LIMIT 10' + + - match: {columns.0.name: "message"} + - match: {columns.0.type: "text"} + + - match: {values.0.0: "Do. Or do not. There is no try."} + - match: {values.1.0: "I find your lack of faith disturbing."} + - match: {values.2.0: "Wars not make one great."} + - match: {values.3.0: "No, I am your father."} + - match: {values.4.0: "May the force be with you."} + - match: {values.5.0: "That's no moon. It's a space station."} + +--- +"message field without keyword multi-field": + - do: + indices.create: + index: my-index2 + body: + settings: + index: + mode: logsdb + mappings: + properties: + "@timestamp": + type: date + host.name: + type: keyword + agent_id: + type: keyword + doc_values: false + store: false + process_id: + type: integer + doc_values: false + store: false + http_method: + type: keyword + doc_values: false + store: false + is_https: + type: boolean + doc_values: false + store: false + location: + type: geo_point + doc_values: false + store: false + message: + type: text + store: false + + - do: + bulk: + index: my-index2 + refresh: true + body: + - { "index": { } } + - { "@timestamp": "2024-02-12T10:30:00Z", "host.name": "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "No, I am your father." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "Do. Or do not. There is no try." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "May the force be with you." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "is_https": true, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "I find your lack of faith disturbing." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "Wars not make one great." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "is_https": false, "location": { "lat": 40.7128, "lon": -74.0060 }, "message": "That's no moon. It's a space station." } + + - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + esql.query: + body: + query: 'FROM my-index2 | SORT host.name, @timestamp | LIMIT 1' + + - match: {columns.0.name: "@timestamp"} + - match: {columns.0.type: "date"} + - match: {columns.1.name: "agent_id"} + - match: {columns.1.type: "keyword"} + - match: {columns.2.name: "host.name"} + - match: {columns.2.type: "keyword"} + - match: {columns.3.name: "http_method" } + - match: {columns.3.type: "keyword" } + - match: {columns.4.name: "is_https"} + - match: {columns.4.type: "boolean"} + - match: {columns.5.name: "location"} + - match: {columns.5.type: "geo_point"} + - match: {columns.6.name: "message"} + - match: {columns.6.type: "text"} + - match: {columns.7.name: "process_id"} + - match: {columns.7.type: "integer"} + + - match: {values.0.0: "2024-02-12T10:31:00.000Z"} + - match: {values.0.1: "yoda"} + - match: {values.0.2: "bar"} + - match: {values.0.3: "PUT"} + - match: {values.0.4: false} + - match: {values.0.5: "POINT (-74.006 40.7128)"} + - match: {values.0.6: null} # null is expected, because text fields aren't stored in ignored source + - match: {values.0.7: 102} + + - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + esql.query: + body: + query: 'FROM my-index2 | SORT host.name, @timestamp | KEEP message | LIMIT 10' + + - match: {columns.0.name: "message"} + - match: {columns.0.type: "text"} + + # null is expected, because text fields aren't stored in ignored source + - match: {values.0.0: null} + - match: {values.1.0: null} + - match: {values.2.0: null} + - match: {values.3.0: null} + - match: {values.4.0: null} + - match: {values.5.0: null} diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/51_esql_synthetic_source.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/51_esql_synthetic_source.yml new file mode 100644 index 0000000000000..7e305bda4ef4e --- /dev/null +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/51_esql_synthetic_source.yml @@ -0,0 +1,177 @@ +--- +setup: + - do: + indices.create: + index: my-index + body: + settings: + index: + mode: logsdb + mappings: + properties: + "@timestamp": + type: date + host.name: + type: keyword + agent_id: + type: keyword + process_id: + type: integer + http_method: + type: keyword + is_https: + type: boolean + location: + type: geo_point + message: + type: text + + - do: + bulk: + index: my-index + refresh: true + body: + - { "index": { } } + - { "@timestamp": "2024-02-12T10:30:00Z", "host.name": "foo", "agent_id": "darth-vader", "process_id": 101, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "No, I am your father." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:31:00Z", "host.name": "bar", "agent_id": "yoda", "process_id": 102, "http_method": "PUT", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "Do. Or do not. There is no try." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:32:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 103, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "May the force be with you." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:33:00Z", "host.name": "baz", "agent_id": "darth-vader", "process_id": 102, "http_method": "POST", "is_https": true, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "I find your lack of faith disturbing." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:34:00Z", "host.name": "baz", "agent_id": "yoda", "process_id": 104, "http_method": "POST", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "Wars not make one great." } + - { "index": { } } + - { "@timestamp": "2024-02-12T10:35:00Z", "host.name": "foo", "agent_id": "obi-wan", "process_id": 105, "http_method": "GET", "is_https": false, "location": {"lat" : 40.7128, "lon" : -74.0060}, "message": "That's no moon. It's a space station." } + +--- +teardown: + - do: + indices.delete: + index: my-index + +--- +"Simple from": + - do: + esql.query: + body: + query: 'FROM my-index | SORT host.name, @timestamp | LIMIT 1' + + - match: {columns.0.name: "@timestamp"} + - match: {columns.0.type: "date"} + - match: {columns.1.name: "agent_id"} + - match: {columns.1.type: "keyword"} + - match: {columns.2.name: "host.name"} + - match: {columns.2.type: "keyword"} + - match: {columns.3.name: "http_method" } + - match: {columns.3.type: "keyword" } + - match: {columns.4.name: "is_https"} + - match: {columns.4.type: "boolean"} + - match: {columns.5.name: "location"} + - match: {columns.5.type: "geo_point"} + - match: {columns.6.name: "message"} + - match: {columns.6.type: "text"} + - match: {columns.7.name: "process_id"} + - match: {columns.7.type: "integer"} + + - match: {values.0.0: "2024-02-12T10:31:00.000Z"} + - match: {values.0.1: "yoda"} + - match: {values.0.2: "bar"} + - match: {values.0.3: "PUT"} + - match: {values.0.4: false} + - match: {values.0.5: "POINT (-74.00600004941225 40.712799984030426)"} + - match: {values.0.6: "Do. Or do not. There is no try."} + - match: {values.0.7: 102} + +--- +"Simple from geo point": + - do: + esql.query: + body: + query: 'FROM my-index | SORT host.name, @timestamp | KEEP location | LIMIT 10' + + - match: {columns.0.name: "location"} + - match: {columns.0.type: "geo_point"} + + - match: {values.0.0: "POINT (-74.00600004941225 40.712799984030426)"} + - match: {values.1.0: "POINT (-74.00600004941225 40.712799984030426)"} + - match: {values.2.0: "POINT (-74.00600004941225 40.712799984030426)"} + - match: {values.3.0: "POINT (-74.00600004941225 40.712799984030426)"} + - match: {values.4.0: "POINT (-74.00600004941225 40.712799984030426)"} + - match: {values.5.0: "POINT (-74.00600004941225 40.712799984030426)"} + +--- +"Simple from number fields": + - do: + esql.query: + body: + query: 'FROM my-index | SORT host.name, @timestamp | KEEP process_id | LIMIT 10' + + - match: {columns.0.name: "process_id"} + - match: {columns.0.type: "integer"} + + - match: {values.0.0: 102} + - match: {values.1.0: 102} + - match: {values.2.0: 104} + - match: {values.3.0: 101} + - match: {values.4.0: 103} + - match: {values.5.0: 105} + +--- +"Simple from keyword fields": + - do: + esql.query: + body: + query: 'FROM my-index | SORT host.name, @timestamp | KEEP agent_id, http_method | LIMIT 10' + + - match: {columns.0.name: "agent_id"} + - match: {columns.0.type: "keyword"} + - match: {columns.1.name: "http_method"} + - match: {columns.1.type: "keyword"} + + - match: {values.0.0: "yoda"} + - match: {values.0.1: "PUT"} + - match: {values.1.0: "darth-vader"} + - match: {values.1.1: "POST"} + - match: {values.2.0: "yoda"} + - match: {values.2.1: "POST"} + - match: {values.3.0: "darth-vader"} + - match: {values.3.1: "GET"} + - match: {values.4.0: "obi-wan"} + - match: {values.4.1: "GET"} + - match: {values.5.0: "obi-wan"} + - match: {values.5.1: "GET"} + +--- +"Simple from boolean fields": + - do: + esql.query: + body: + query: 'FROM my-index | SORT host.name, @timestamp | KEEP is_https | LIMIT 10' + + - match: {columns.0.name: "is_https"} + - match: {columns.0.type: "boolean"} + + - match: {values.0.0: false} + - match: {values.1.0: true} + - match: {values.2.0: false} + - match: {values.3.0: false} + - match: {values.4.0: false} + - match: {values.5.0: false} + +--- +"Simple from text fields": + - do: + esql.query: + body: + query: 'FROM my-index | SORT host.name, @timestamp | KEEP message | LIMIT 10' + + - match: {columns.0.name: "message"} + - match: {columns.0.type: "text"} + + - match: {values.0.0: "Do. Or do not. There is no try."} + - match: {values.1.0: "I find your lack of faith disturbing."} + - match: {values.2.0: "Wars not make one great."} + - match: {values.3.0: "No, I am your father."} + - match: {values.4.0: "May the force be with you."} + - match: {values.5.0: "That's no moon. It's a space station."} diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index 5b04225cee105..303b94ec655dc 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -339,7 +339,8 @@ protected Object parseSourceValue(Object value) { BlockSourceReader.LeafIteratorLookup lookup = isStored() || isIndexed() ? BlockSourceReader.lookupFromFieldNames(blContext.fieldNames(), name()) : BlockSourceReader.lookupMatchingAll(); - return new BlockSourceReader.LongsBlockLoader(valueFetcher, lookup); + var sourceMode = blContext.indexSettings().getIndexMappingSourceMode(); + return new BlockSourceReader.LongsBlockLoader(valueFetcher, lookup, sourceMode); } @Override