From 5e0701f026258e9d7c5021267ba09e776b29c2af Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Thu, 28 Jul 2022 10:34:05 -0700 Subject: [PATCH] Add source fallback for keyword fields using operation (#88735) This change adds an operation parameter to FieldDataContext that allows us to specialize the field data that are returned from fielddataBuilder in MappedFieldType. Keyword, integer, and geo point field types now support source fallback where we build a doc values wrapper using source if doc values doesn't exist for this field under the operation SCRIPT. This allows us to have source fallback in scripting for the scripting fields API. --- .../script/ScriptScoreBenchmark.java | 6 +- docs/changelog/88735.yaml | 5 + .../expression/ExpressionScriptEngine.java | 2 +- .../ExpressionFieldScriptTests.java | 2 +- .../ExpressionNumberSortScriptTests.java | 2 +- .../ExpressionTermsSetQueryTests.java | 2 +- .../test/painless/50_script_doc_values.yml | 269 +++++++++++++++++- .../mapper/extras/TokenCountFieldMapper.java | 2 +- .../join/query/HasChildQueryBuilder.java | 2 +- .../join/query/HasParentQueryBuilder.java | 2 +- .../percolator/PercolateQueryBuilder.java | 12 +- .../percolator/QueryBuilderStoreTests.java | 4 +- .../index/mapper/size/SizeFieldMapper.java | 2 +- .../index/fielddata/FieldDataContext.java | 16 +- .../fielddata/IndexFieldDataService.java | 1 - .../SourceValueFetcherIndexFieldData.java | 152 ++++++++++ ...lueFetcherMultiGeoPointIndexFieldData.java | 115 ++++++++ ...alueFetcherSortedBinaryIndexFieldData.java | 127 +++++++++ ...lueFetcherSortedNumericIndexFieldData.java | 151 ++++++++++ .../mapper/AbstractGeometryFieldMapper.java | 13 + .../index/mapper/AbstractScriptFieldType.java | 2 +- .../index/mapper/ArraySourceValueFetcher.java | 9 + .../index/mapper/DocumentParser.java | 5 +- .../index/mapper/GeoPointFieldMapper.java | 28 +- .../index/mapper/KeywordFieldMapper.java | 42 ++- .../index/mapper/MappedFieldType.java | 9 + .../index/mapper/NumberFieldMapper.java | 48 +++- .../index/mapper/SourceValueFetcher.java | 2 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 2 +- .../index/mapper/VersionFieldMapper.java | 2 +- .../index/query/SearchExecutionContext.java | 22 +- .../index/query/TermsSetQueryBuilder.java | 2 +- .../functionscore/DecayFunctionBuilder.java | 6 +- .../FieldValueFactorFunctionBuilder.java | 3 +- .../RandomScoreFunctionBuilder.java | 7 +- .../support/AggregationContext.java | 2 +- .../fetch/subphase/FetchDocValuesPhase.java | 2 +- .../search/lookup/LeafDocLookup.java | 24 +- .../search/lookup/SearchLookup.java | 14 +- .../search/slice/SliceBuilder.java | 2 +- .../search/sort/FieldSortBuilder.java | 4 +- .../search/sort/GeoDistanceSortBuilder.java | 2 +- .../index/IndexSortSettingsTests.java | 9 +- .../fielddata/AbstractFieldDataTestCase.java | 2 +- .../index/fielddata/FilterFieldDataTests.java | 8 +- .../fielddata/IndexFieldDataServiceTests.java | 2 +- .../AbstractScriptFieldTypeTestCase.java | 10 +- .../index/mapper/BooleanFieldScriptTests.java | 2 +- .../mapper/CompositeRuntimeFieldTests.java | 4 +- .../index/mapper/DateFieldScriptTests.java | 4 +- .../index/mapper/DoubleFieldScriptTests.java | 4 +- .../mapper/GeoPointFieldScriptTests.java | 2 +- .../index/mapper/IgnoredFieldMapperTests.java | 5 +- .../index/mapper/IndexFieldMapperTests.java | 2 +- .../index/mapper/IpFieldScriptTests.java | 4 +- .../index/mapper/LongFieldScriptTests.java | 4 +- .../mapper/PlaceHolderFieldMapperTests.java | 2 +- .../mapper/ProvidedIdFieldMapperTests.java | 5 +- .../index/mapper/RoutingFieldMapperTests.java | 5 +- .../index/mapper/StringFieldScriptTests.java | 8 +- .../index/mapper/TextFieldMapperTests.java | 2 - .../index/mapper/VersionFieldMapperTests.java | 2 +- .../query/SearchExecutionContextTests.java | 6 +- .../script/CompositeFieldScriptTests.java | 6 +- ...dNumericDocValuesLongFieldScriptTests.java | 2 +- ...tedSetDocValuesStringFieldScriptTests.java | 2 +- .../bucket/range/RangeAggregatorTests.java | 2 +- .../bucket/terms/TermsAggregatorTests.java | 2 +- .../fetch/subphase/FieldFetcherTests.java | 9 +- .../search/lookup/LeafDocLookupTests.java | 6 +- .../search/slice/SliceBuilderTests.java | 2 +- .../index/mapper/MapperServiceTestCase.java | 8 +- .../index/mapper/MapperTestCase.java | 19 +- .../aggregations/AggregatorTestCase.java | 11 +- .../AggregateDoubleMetricFieldTypeTests.java | 4 +- .../xpack/rollup/v2/FieldValueFetcher.java | 2 +- 76 files changed, 1159 insertions(+), 134 deletions(-) create mode 100644 docs/changelog/88735.yaml create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index 2fc5929cb955d..23d342079cdb4 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -57,6 +57,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; /** @@ -83,10 +84,11 @@ public class ScriptScoreBenchmark { Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null)) ); private final IndexFieldDataCache fieldDataCache = new IndexFieldDataCache.None(); + private final Map> sourcePaths = Map.of("n", Set.of("n")); private final CircuitBreakerService breakerService = new NoneCircuitBreakerService(); private final SearchLookup lookup = new SearchLookup( fieldTypes::get, - (mft, lookup) -> mft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark")).build(fieldDataCache, breakerService) + (mft, lookup, fdo) -> mft.fielddataBuilder(FieldDataContext.noRuntimeFields("benchmark")).build(fieldDataCache, breakerService) ); @Param({ "expression", "metal", "painless_cast", "painless_def" }) @@ -152,7 +154,7 @@ private Query scriptScoreQuery(ScoreScript.Factory factory) { private ScoreScript.Factory bareMetalScript() { return (params, lookup) -> { MappedFieldType type = fieldTypes.get("n"); - IndexNumericFieldData ifd = (IndexNumericFieldData) lookup.getForField(type); + IndexNumericFieldData ifd = (IndexNumericFieldData) lookup.getForField(type, MappedFieldType.FielddataOperation.SEARCH); return new ScoreScript.LeafFactory() { @Override public ScoreScript newInstance(DocReader docReader) throws IOException { diff --git a/docs/changelog/88735.yaml b/docs/changelog/88735.yaml new file mode 100644 index 0000000000000..91c8d1c3f4417 --- /dev/null +++ b/docs/changelog/88735.yaml @@ -0,0 +1,5 @@ +pr: 88735 +summary: Add source fallback for keyword fields using operation +area: Mapping +type: enhancement +issues: [] diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java index 0d2f71ade6e51..a6c62b10b9f81 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java @@ -460,7 +460,7 @@ private static DoubleValuesSource getDocValueSource(String variable, SearchLooku throw new ParseException("Field [" + fieldname + "] does not exist in mappings", 5); } - IndexFieldData fieldData = lookup.getForField(fieldType); + IndexFieldData fieldData = lookup.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); final DoubleValuesSource valueSource; if (fieldType instanceof GeoPointFieldType) { // geo diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java index b8ce67ef13d66..68c58d8ee7de8 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java @@ -49,7 +49,7 @@ public void setUp() throws Exception { when(fieldData.load(any())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(field -> field.equals("field") ? fieldType : null, (ignored, _lookup) -> fieldData); + lookup = new SearchLookup(field -> field.equals("field") ? fieldType : null, (ignored, _lookup, fdt) -> fieldData); } private FieldScript.LeafFactory compile(String expression) { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java index a3f1a750053c8..01aec0e0e3b24 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java @@ -50,7 +50,7 @@ public void setUp() throws Exception { when(fieldData.load(any())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(field -> field.equals("field") ? fieldType : null, (ignored, _lookup) -> fieldData); + lookup = new SearchLookup(field -> field.equals("field") ? fieldType : null, (ignored, _lookup, fdt) -> fieldData); } private NumberSortScript.LeafFactory compile(String expression) { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java index faa08db55138b..60a1d6e64551b 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java @@ -49,7 +49,7 @@ public void setUp() throws Exception { when(fieldData.load(any())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(field -> field.equals("field") ? fieldType : null, (ignored, _lookup) -> fieldData); + lookup = new SearchLookup(field -> field.equals("field") ? fieldType : null, (ignored, _lookup, fdt) -> fieldData); } private TermsSetQueryScript.LeafFactory compile(String expression) { diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index 63f01c4f53b36..4ba1b270cd995 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -15,14 +15,23 @@ setup: type: date_nanos geo_point: type: geo_point + geo_point_no_doc_values: + type: geo_point + doc_values: false ip: type: ip keyword: type: keyword + keyword_no_doc_values: + type: keyword + doc_values: false long: type: long integer: type: integer + integer_no_doc_values: + type: integer + doc_values: false short: type: short byte: @@ -55,10 +64,13 @@ setup: date: 2017-01-01T12:11:12 nanos: 2015-01-01T12:10:30.123456789Z geo_point: 41.12,-71.34 + geo_point_no_doc_values: 41.12,-71.34 ip: 192.168.0.19 keyword: not split at all + keyword_no_doc_values: no doc values long: 12348732141234 integer: 134134566 + integer_no_doc_values: 134134566 short: 1324 byte: 12 double: 3.14159265358979 @@ -85,9 +97,12 @@ setup: date: [2017-01-01T12:11:12, 2018-01-01T12:11:12] nanos: [2015-01-01T12:10:30.123456789Z, 2015-01-01T12:10:30.987654321Z] geo_point: [[-71.34,41.12],[60.32,21.25]] + geo_point_no_doc_values: [[60.32,21.25],[-71.34,41.12]] keyword: ["one string", "another string"] + keyword_no_doc_values: ["no doc values 1", "no doc values 0", "no doc values 2"] long: [1152921504606846976, 576460752303423488] integer: [5, 17, 29] + integer_no_doc_values: [17, 29, 5] short: [6, 18, 30, 45] byte: [16, 32, 64, 8, 4] double: [3.141592653588, 2.141592653587] @@ -211,7 +226,6 @@ setup: source: "field('boolean').size()" - match: { hits.hits.0.fields.field.0: 0 } - --- "date": - skip: @@ -693,6 +707,104 @@ setup: - match: { hits.hits.0.fields.width.0: 0.0 } - match: { hits.hits.0.fields.height.0: 0.0 } +--- +"geo_point_no_doc_values": + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "doc['geo_point_no_doc_values'].get(0)" + - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "doc['geo_point_no_doc_values'].value" + - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "field('geo_point_no_doc_values').get(new GeoPoint())" + - match: { hits.hits.0.fields.field.0.lat: 41.1199999647215 } + - match: { hits.hits.0.fields.field.0.lon: -71.34000004269183 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('geo_point_no_doc_values', new GeoPoint())" + - match: { hits.hits.0.fields.field.0.lat: 41.1199999647215 } + - match: { hits.hits.0.fields.field.0.lon: -71.34000004269183 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "3" } } + script_fields: + field: + script: + source: "field('geo_point_no_doc_values').get(new GeoPoint())" + - match: { hits.hits.0.fields.field.0.lat: 21.249999990686774 } + - match: { hits.hits.0.fields.field.0.lon: 60.319999968633056 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "3" } } + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('geo_point_no_doc_values', new GeoPoint())" + - match: { hits.hits.0.fields.field.0.lat: 21.249999990686774 } + - match: { hits.hits.0.fields.field.0.lon: 60.319999968633056 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "3" } } + script_fields: + field: + script: + source: "field('geo_point_no_doc_values').get(1, new GeoPoint())" + - match: { hits.hits.0.fields.field.0.lat: 41.1199999647215 } + - match: { hits.hits.0.fields.field.0.lon: -71.34000004269183 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "2" } } + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('geo_point_no_doc_values', new GeoPoint(1.0, 2.0))" + - match: { hits.hits.0.fields.field.0.lat: 1.0 } + - match: { hits.hits.0.fields.field.0.lon: 2.0 } + --- "ip": - do: @@ -863,6 +975,85 @@ setup: - match: { hits.hits.1.fields.field.0: "" } - match: { hits.hits.2.fields.field.0: "another string, one string" } +--- +"keyword_no_doc_values": + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "doc['keyword_no_doc_values'].get(0)" + - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "doc['keyword_no_doc_values'].value" + - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "field('keyword_no_doc_values').get('missing')" + - match: { hits.hits.0.fields.field.0: "no doc values" } + - match: { hits.hits.1.fields.field.0: "missing" } + - match: { hits.hits.2.fields.field.0: "no doc values 0" } # doc values are sorted + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('keyword_no_doc_values', 'missing')" + # same as `field('keyword').get('missing')` + - match: { hits.hits.0.fields.field.0: "no doc values" } + - match: { hits.hits.1.fields.field.0: "missing" } + - match: { hits.hits.2.fields.field.0: "no doc values 0" } + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "field('keyword_no_doc_values').get(1, 'dne')" + - match: { hits.hits.0.fields.field.0: "dne" } + - match: { hits.hits.1.fields.field.0: "dne" } + - match: { hits.hits.2.fields.field.0: "no doc values 1" } # doc values are sorted + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "String.join(', ', field('keyword_no_doc_values'))" + - match: { hits.hits.0.fields.field.0: "no doc values" } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "no doc values 0, no doc values 1, no doc values 2" } + --- "long": - skip: @@ -964,6 +1155,82 @@ setup: - match: { hits.hits.1.fields.field.0: 0 } - match: { hits.hits.2.fields.field.0: 54 } +--- +"integer_no_doc_values": + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "doc['integer_no_doc_values'].get(0)" + - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "doc['integer_no_doc_values'].value" + - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "field('integer_no_doc_values').get(-1)" + - match: { hits.hits.0.fields.field.0: 134134566 } + - match: { hits.hits.1.fields.field.0: -1 } + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('integer_no_doc_values', -1)" + - match: { hits.hits.0.fields.field.0: 134134566 } + - match: { hits.hits.1.fields.field.0: -1 } + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "field('integer_no_doc_values').get(1, -3)" + - match: { hits.hits.0.fields.field.0: -3 } + - match: { hits.hits.1.fields.field.0: -3 } + - match: { hits.hits.2.fields.field.0: 17 } + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "int total = 0; for (int i : field('integer_no_doc_values')) { total += i; } total + field('integer_no_doc_values').size();" + - match: { hits.hits.0.fields.field.0: 134134567 } + - match: { hits.hits.1.fields.field.0: 0 } + - match: { hits.hits.2.fields.field.0: 54 } + --- "short": - do: diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java index f5c69329e3599..585b2ad44720e 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java @@ -122,7 +122,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (hasDocValues() == false) { return (lookup, ignoredValues) -> List.of(); } - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); } } diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java index d877db6a7ff3c..a8845b2731abb 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java @@ -344,7 +344,7 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { Query childFilter = joiner.filter(type); Query filteredQuery = Queries.filtered(query.toQuery(context), childFilter); MappedFieldType ft = context.getFieldType(parentJoinField); - final SortedSetOrdinalsIndexFieldData fieldData = context.getForField(ft); + final SortedSetOrdinalsIndexFieldData fieldData = context.getForField(ft, MappedFieldType.FielddataOperation.SEARCH); return new LateParsingQuery( parentFilter, filteredQuery, diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java index 64c3e627df75f..9c297236a8e0b 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java @@ -181,7 +181,7 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { Query innerQuery = Queries.filtered(query.toQuery(context), parentFilter); Query childFilter = joiner.childrenFilter(parentType); MappedFieldType fieldType = context.getFieldType(joiner.childJoinField(parentType)); - final SortedSetOrdinalsIndexFieldData fieldData = context.getForField(fieldType); + final SortedSetOrdinalsIndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); return new HasChildQueryBuilder.LateParsingQuery( childFilter, innerQuery, diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index c574dc49ddc82..97fcc404d4e47 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -641,9 +641,17 @@ public BitSetProducer bitsetFilter(Query query) { @Override @SuppressWarnings("unchecked") - public > IFD getForField(MappedFieldType fieldType) { + public > IFD getForField( + MappedFieldType fieldType, + MappedFieldType.FielddataOperation fielddataOperation + ) { IndexFieldData.Builder builder = fieldType.fielddataBuilder( - new FieldDataContext(delegate.getFullyQualifiedIndex().getName(), delegate::lookup) + new FieldDataContext( + delegate.getFullyQualifiedIndex().getName(), + delegate::lookup, + this::sourcePath, + fielddataOperation + ) ); IndexFieldDataCache cache = new IndexFieldDataCache.None(); CircuitBreakerService circuitBreaker = new NoneCircuitBreakerService(); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java index e2d3ffe259e47..5a2162fa80f50 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.index.mapper.BinaryFieldMapper; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.TestDocumentParserContext; import org.elasticsearch.index.query.SearchExecutionContext; @@ -61,6 +62,7 @@ public void testStoringQueryBuilders() throws IOException { IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer()); config.setMergePolicy(NoMergePolicy.INSTANCE); BinaryFieldMapper fieldMapper = PercolatorFieldMapper.Builder.createQueryBuilderFieldBuilder(MapperBuilderContext.ROOT); + MappedFieldType.FielddataOperation fielddataOperation = MappedFieldType.FielddataOperation.SEARCH; Version version = Version.CURRENT; try (IndexWriter indexWriter = new IndexWriter(directory, config)) { @@ -76,7 +78,7 @@ public void testStoringQueryBuilders() throws IOException { when(searchExecutionContext.indexVersionCreated()).thenReturn(version); when(searchExecutionContext.getWriteableRegistry()).thenReturn(writableRegistry()); when(searchExecutionContext.getParserConfig()).thenReturn(parserConfig()); - when(searchExecutionContext.getForField(fieldMapper.fieldType())).thenReturn( + when(searchExecutionContext.getForField(fieldMapper.fieldType(), fielddataOperation)).thenReturn( new BytesBinaryIndexFieldData(fieldMapper.name(), CoreValuesSourceType.KEYWORD) ); when(searchExecutionContext.getFieldType(Mockito.anyString())).thenAnswer(invocation -> { diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index 385efcb0ebaee..34603c2cd3843 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -57,7 +57,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (hasDocValues() == false) { return (lookup, ignoredValues) -> List.of(); } - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java b/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java index 4b219faadaf39..3cfab4e599015 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java @@ -8,8 +8,11 @@ package org.elasticsearch.index.fielddata; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.search.lookup.SearchLookup; +import java.util.Set; +import java.util.function.Function; import java.util.function.Supplier; /** @@ -17,8 +20,15 @@ * * @param fullyQualifiedIndexName the index name with any remote index information added * @param lookupSupplier a supplier for a SearchLookup to be used by runtime scripts + * @param sourcePathsLookup a function to get source paths for a specific field + * @param fielddataOperation the operation used to determine data structures to generate fielddata from */ -public record FieldDataContext(String fullyQualifiedIndexName, Supplier lookupSupplier) { +public record FieldDataContext( + String fullyQualifiedIndexName, + Supplier lookupSupplier, + Function> sourcePathsLookup, + MappedFieldType.FielddataOperation fielddataOperation +) { /** * A context to use when runtime fields are not available @@ -30,7 +40,9 @@ public record FieldDataContext(String fullyQualifiedIndexName, Supplier { throw new UnsupportedOperationException("Runtime fields not supported for [" + reason + "]"); } + () -> { throw new UnsupportedOperationException("Runtime fields not supported for [" + reason + "]"); }, + Set::of, + MappedFieldType.FielddataOperation.SEARCH ); } } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index 7c4be67f1bd5c..d520d647eb2c2 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -100,7 +100,6 @@ public synchronized void clearField(final String fieldName) { public > IFD getForField(MappedFieldType fieldType, FieldDataContext fieldDataContext) { final String fieldName = fieldType.name(); IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldDataContext); - IndexFieldDataCache cache; synchronized (this) { cache = fieldDataCaches.get(fieldName); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java new file mode 100644 index 0000000000000..8d24977a9c22f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.SortField; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.script.field.ToScriptFieldFactory; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +public abstract class SourceValueFetcherIndexFieldData + implements + IndexFieldData> { + + public abstract static class Builder implements IndexFieldData.Builder { + + protected final String fieldName; + protected final ValuesSourceType valuesSourceType; + protected final ValueFetcher valueFetcher; + protected final SourceLookup sourceLookup; + protected final ToScriptFieldFactory toScriptFieldFactory; + + public Builder( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + this.fieldName = fieldName; + this.valuesSourceType = valuesSourceType; + this.valueFetcher = valueFetcher; + this.sourceLookup = sourceLookup; + this.toScriptFieldFactory = toScriptFieldFactory; + } + } + + protected final String fieldName; + protected final ValuesSourceType valuesSourceType; + protected final ValueFetcher valueFetcher; + protected final SourceLookup sourceLookup; + protected final ToScriptFieldFactory toScriptFieldFactory; + + protected SourceValueFetcherIndexFieldData( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + this.fieldName = fieldName; + this.valuesSourceType = valuesSourceType; + this.valueFetcher = valueFetcher; + this.sourceLookup = sourceLookup; + this.toScriptFieldFactory = toScriptFieldFactory; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return valuesSourceType; + } + + @Override + public SourceValueFetcherLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { + throw new IllegalArgumentException("not supported for source fallback"); + } + + @Override + public BucketedSort newBucketedSort( + BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra + ) { + throw new IllegalArgumentException("not supported for source fallback"); + } + + public abstract static class SourceValueFetcherLeafFieldData implements LeafFieldData { + + protected final ToScriptFieldFactory toScriptFieldFactory; + protected final LeafReaderContext leafReaderContext; + + protected final ValueFetcher valueFetcher; + protected final SourceLookup sourceLookup; + + public SourceValueFetcherLeafFieldData( + ToScriptFieldFactory toScriptFieldFactory, + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + this.toScriptFieldFactory = toScriptFieldFactory; + this.leafReaderContext = leafReaderContext; + this.valueFetcher = valueFetcher; + this.sourceLookup = sourceLookup; + } + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public void close() { + + } + + @Override + public SortedBinaryDocValues getBytesValues() { + throw new IllegalArgumentException("not supported for source fallback"); + } + } + + /** + * Marker interface to indicate these doc values are generated + * on-the-fly from a {@code ValueFetcher}. + */ + public interface ValueFetcherDocValues { + // marker interface + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java new file mode 100644 index 0000000000000..9c54be22a4f27 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.ToScriptFieldFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; + +public class SourceValueFetcherMultiGeoPointIndexFieldData extends SourceValueFetcherIndexFieldData { + + public static class Builder extends SourceValueFetcherIndexFieldData.Builder { + + public Builder( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory); + } + + @Override + public SourceValueFetcherMultiGeoPointIndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { + return new SourceValueFetcherMultiGeoPointIndexFieldData( + fieldName, + valuesSourceType, + valueFetcher, + sourceLookup, + toScriptFieldFactory + ); + } + } + + protected SourceValueFetcherMultiGeoPointIndexFieldData( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory); + } + + @Override + public SourceValueFetcherMultiGeoPointLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + return new SourceValueFetcherMultiGeoPointLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup); + } + + public static class SourceValueFetcherMultiGeoPointLeafFieldData extends + SourceValueFetcherIndexFieldData.SourceValueFetcherLeafFieldData { + + public SourceValueFetcherMultiGeoPointLeafFieldData( + ToScriptFieldFactory toScriptFieldFactory, + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceLookup); + } + + @Override + public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { + return toScriptFieldFactory.getScriptFieldFactory( + new MultiGeoPointValues(new SourceValueFetcherMultiGeoPointDocValues(leafReaderContext, valueFetcher, sourceLookup)), + name + ); + } + } + + public static class SourceValueFetcherMultiGeoPointDocValues extends + SourceValueFetcherSortedNumericIndexFieldData.SourceValueFetcherSortedNumericDocValues { + + public SourceValueFetcherMultiGeoPointDocValues( + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + super(leafReaderContext, valueFetcher, sourceLookup); + } + + @Override + @SuppressWarnings("unchecked") + public boolean advanceExact(int doc) throws IOException { + sourceLookup.setSegmentAndDocument(leafReaderContext, doc); + values = new TreeSet<>(); + + for (Object value : valueFetcher.fetchValues(sourceLookup, Collections.emptyList())) { + List coordinates = ((Map>) value).get("coordinates"); + values.add(new GeoPoint((double) coordinates.get(1), (double) coordinates.get(0)).getEncoded()); + } + + iterator = values.iterator(); + + return true; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java new file mode 100644 index 0000000000000..501430149a0ce --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java @@ -0,0 +1,127 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.ToScriptFieldFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.SortedSet; +import java.util.TreeSet; + +public class SourceValueFetcherSortedBinaryIndexFieldData extends SourceValueFetcherIndexFieldData { + + public static class Builder extends SourceValueFetcherIndexFieldData.Builder { + + public Builder( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory); + } + + @Override + public SourceValueFetcherSortedBinaryIndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { + return new SourceValueFetcherSortedBinaryIndexFieldData( + fieldName, + valuesSourceType, + valueFetcher, + sourceLookup, + toScriptFieldFactory + ); + } + } + + protected SourceValueFetcherSortedBinaryIndexFieldData( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory); + } + + @Override + public SourceValueFetcherSortedBinaryLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + return new SourceValueFetcherSortedBinaryLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup); + } + + public static class SourceValueFetcherSortedBinaryLeafFieldData extends SourceValueFetcherLeafFieldData { + + public SourceValueFetcherSortedBinaryLeafFieldData( + ToScriptFieldFactory toScriptFieldFactory, + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceLookup); + } + + @Override + public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { + return toScriptFieldFactory.getScriptFieldFactory( + new SourceValueFetcherSortedBinaryDocValues(leafReaderContext, valueFetcher, sourceLookup), + name + ); + } + } + + public static class SourceValueFetcherSortedBinaryDocValues extends SortedBinaryDocValues implements ValueFetcherDocValues { + + private final LeafReaderContext leafReaderContext; + + private final ValueFetcher valueFetcher; + private final SourceLookup sourceLookup; + + private SortedSet values; + private Iterator iterator; + + public SourceValueFetcherSortedBinaryDocValues( + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + this.leafReaderContext = leafReaderContext; + this.valueFetcher = valueFetcher; + this.sourceLookup = sourceLookup; + } + + @Override + public boolean advanceExact(int doc) throws IOException { + sourceLookup.setSegmentAndDocument(leafReaderContext, doc); + values = new TreeSet<>(valueFetcher.fetchValues(sourceLookup, Collections.emptyList())); + iterator = values.iterator(); + + return true; + } + + @Override + public int docValueCount() { + return values.size(); + } + + @Override + public BytesRef nextValue() throws IOException { + assert iterator.hasNext(); + return new BytesRef(iterator.next().toString()); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java new file mode 100644 index 0000000000000..671c547155cdd --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.ToScriptFieldFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.TreeSet; + +public class SourceValueFetcherSortedNumericIndexFieldData extends SourceValueFetcherIndexFieldData { + + public static class Builder extends SourceValueFetcherIndexFieldData.Builder { + + public Builder( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory); + } + + @Override + public SourceValueFetcherSortedNumericIndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { + return new SourceValueFetcherSortedNumericIndexFieldData( + fieldName, + valuesSourceType, + valueFetcher, + sourceLookup, + toScriptFieldFactory + ); + } + } + + protected SourceValueFetcherSortedNumericIndexFieldData( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory); + } + + @Override + public SourceValueFetcherSortedNumericLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + return new SourceValueFetcherSortedNumericLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup); + } + + public static class SourceValueFetcherSortedNumericLeafFieldData extends SourceValueFetcherLeafFieldData { + + public SourceValueFetcherSortedNumericLeafFieldData( + ToScriptFieldFactory toScriptFieldFactory, + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceLookup); + } + + @Override + public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { + return toScriptFieldFactory.getScriptFieldFactory( + new SourceValueFetcherSortedNumericDocValues(leafReaderContext, valueFetcher, sourceLookup), + name + ); + } + } + + public static class SourceValueFetcherSortedNumericDocValues extends SortedNumericDocValues implements ValueFetcherDocValues { + + protected final LeafReaderContext leafReaderContext; + + protected final ValueFetcher valueFetcher; + protected final SourceLookup sourceLookup; + + protected TreeSet values; + protected Iterator iterator; + + public SourceValueFetcherSortedNumericDocValues( + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + this.leafReaderContext = leafReaderContext; + this.valueFetcher = valueFetcher; + this.sourceLookup = sourceLookup; + } + + @Override + public boolean advanceExact(int doc) throws IOException { + sourceLookup.setSegmentAndDocument(leafReaderContext, doc); + values = new TreeSet<>(); + + for (Object value : valueFetcher.fetchValues(sourceLookup, Collections.emptyList())) { + values.add(((Number) value).longValue()); + } + + iterator = values.iterator(); + + return true; + } + + @Override + public int docValueCount() { + return values.size(); + } + + @Override + public long nextValue() throws IOException { + assert iterator.hasNext(); + return iterator.next(); + } + + @Override + public int docID() { + throw new UnsupportedOperationException("not supported for source fallback"); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException("not supported for source fallback"); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException("not supported for source fallback"); + } + + @Override + public long cost() { + throw new UnsupportedOperationException("not supported for source fallback"); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 04a6d62576073..87b6abb217bc9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; @@ -123,6 +124,18 @@ protected Object parseSourceValue(Object value) { } }; } + + public ValueFetcher valueFetcher(Set sourcePaths, Object nullValue, String format) { + Function, List> formatter = getFormatter(format != null ? format : GeometryFormatterFactory.GEOJSON); + return new ArraySourceValueFetcher(sourcePaths, nullValue) { + @Override + protected Object parseSourceValue(Object value) { + final List values = new ArrayList<>(); + geometryParser.fetchFromSource(value, values::add); + return formatter.apply(values); + } + }; + } } private final Explicit ignoreMalformed; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java index c966ecacaa38d..187d148387dff 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java @@ -173,7 +173,7 @@ protected final void applyScriptContext(SearchExecutionContext context) { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java index 809dc8d3e3e04..6d138250ca3b6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java @@ -43,6 +43,15 @@ public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context, this.nullValue = nullValue; } + /** + * @param sourcePaths The paths to pull source values from + * @param nullValue An optional substitute value if the _source value is `null` + */ + public ArraySourceValueFetcher(Set sourcePaths, Object nullValue) { + this.sourcePaths = sourcePaths; + this.nullValue = nullValue; + } + @Override public List fetchValues(SourceLookup lookup, List ignoredValues) { List values = new ArrayList<>(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 43d0a6cc06f18..d79f07d9cac25 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -142,8 +142,9 @@ private static void executeIndexTimeScripts(DocumentParserContext context) { } SearchLookup searchLookup = new SearchLookup( context.mappingLookup().indexTimeLookup()::get, - (ft, lookup) -> ft.fielddataBuilder(new FieldDataContext(context.indexSettings().getIndex().getName(), lookup)) - .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) + (ft, lookup, fto) -> ft.fielddataBuilder( + new FieldDataContext(context.indexSettings().getIndex().getName(), lookup, context.mappingLookup()::sourcePaths, fto) + ).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) ); // field scripts can be called both by the loop at the end of this method and via // the document reader, so to ensure that we don't run them multiple times we diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index e550acdcf6237..2951162275399 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -30,6 +30,7 @@ import org.elasticsearch.geometry.Point; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.SourceValueFetcherMultiGeoPointIndexFieldData; import org.elasticsearch.index.fielddata.plain.AbstractLatLonPointIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.script.GeoPointFieldScript; @@ -52,6 +53,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.Function; /** @@ -369,8 +371,30 @@ private boolean isPointGeometry(LatLonGeometry[] geometries) { @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { - failIfNoDocValues(); - return new AbstractLatLonPointIndexFieldData.Builder(name(), CoreValuesSourceType.GEOPOINT, GeoPointDocValuesField::new); + FielddataOperation operation = fieldDataContext.fielddataOperation(); + + if (operation == FielddataOperation.SEARCH) { + failIfNoDocValues(); + } + + if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) { + return new AbstractLatLonPointIndexFieldData.Builder(name(), CoreValuesSourceType.GEOPOINT, GeoPointDocValuesField::new); + } + + if (operation == FielddataOperation.SCRIPT) { + SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); + Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); + + return new SourceValueFetcherMultiGeoPointIndexFieldData.Builder( + name(), + CoreValuesSourceType.GEOPOINT, + valueFetcher(sourcePaths, null, null), + searchLookup.source(), + GeoPointDocValuesField::new + ); + } + + throw new IllegalStateException("unknown field data type [" + operation.name() + "]"); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 3da17851000d5..32bfda76d6d2c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -48,6 +48,7 @@ import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; @@ -77,6 +78,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Set; import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH; import static org.elasticsearch.core.Strings.format; @@ -682,12 +684,34 @@ NamedAnalyzer normalizer() { @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { - failIfNoDocValues(); - return new SortedSetOrdinalsIndexFieldData.Builder( - name(), - CoreValuesSourceType.KEYWORD, - (dv, n) -> new KeywordDocValuesField(FieldData.toString(dv), n) - ); + FielddataOperation operation = fieldDataContext.fielddataOperation(); + + if (operation == FielddataOperation.SEARCH) { + failIfNoDocValues(); + } + + if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) { + return new SortedSetOrdinalsIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + (dv, n) -> new KeywordDocValuesField(FieldData.toString(dv), n) + ); + } + + if (operation == FielddataOperation.SCRIPT) { + SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); + Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); + + return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + sourceValueFetcher(sourcePaths), + searchLookup.source(), + KeywordDocValuesField::new + ); + } + + throw new IllegalStateException("unknown field data type [" + operation.name() + "]"); } @Override @@ -698,7 +722,11 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return new SourceValueFetcher(name(), context, nullValue) { + return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + } + + private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { + return new SourceValueFetcher(sourcePaths, nullValue) { @Override protected String parseSourceValue(Object value) { String keywordValue = value.toString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 0d5ec776a900d..06fa0a97fa881 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -80,6 +80,15 @@ public MappedFieldType( this.meta = Objects.requireNonNull(meta); } + /** + * Operation to specify what data structures are used to retrieve + * field data from and generate a representation of doc values. + */ + public enum FielddataOperation { + SEARCH, + SCRIPT + } + /** * Return a fielddata builder for this field * diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 5dd897be93aa8..7c4b4384ddb7e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -36,6 +36,7 @@ import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; +import org.elasticsearch.index.fielddata.SourceValueFetcherSortedNumericIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedDoublesIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.mapper.TimeSeriesParams.MetricType; @@ -54,6 +55,7 @@ import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParser.Token; @@ -66,6 +68,7 @@ import java.util.EnumSet; import java.util.Map; import java.util.Objects; +import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; @@ -918,6 +921,21 @@ public IndexFieldData.Builder getFieldDataBuilder(String name) { return new SortedNumericIndexFieldData.Builder(name, numericType(), IntegerDocValuesField::new); } + @Override + public IndexFieldData.Builder getValueFetcherFieldDataBuilder( + String name, + SourceLookup sourceLookup, + ValueFetcher valueFetcher + ) { + return new SourceValueFetcherSortedNumericIndexFieldData.Builder( + name, + numericType().getValuesSourceType(), + valueFetcher, + sourceLookup, + IntegerDocValuesField::new + ); + } + @Override SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) { return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName); @@ -1253,6 +1271,10 @@ public static Query longRangeQuery( public abstract IndexFieldData.Builder getFieldDataBuilder(String name); + public IndexFieldData.Builder getValueFetcherFieldDataBuilder(String name, SourceLookup sourceLookup, ValueFetcher valueFetcher) { + throw new UnsupportedOperationException("not supported for source fallback"); + } + /** * Adjusts a value to the value it would have been had it been parsed by that mapper * and then cast up to a double. This is meant to be an entry point to manipulate values @@ -1403,8 +1425,24 @@ public Function pointReaderIfPossible() { @Override public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext) { - failIfNoDocValues(); - return type.getFieldDataBuilder(name()); + FielddataOperation operation = fieldDataContext.fielddataOperation(); + + if (fieldDataContext.fielddataOperation() == FielddataOperation.SEARCH) { + failIfNoDocValues(); + } + + if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) { + return type.getFieldDataBuilder(name()); + } + + if (operation == FielddataOperation.SCRIPT) { + SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); + Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); + + return type.getValueFetcherFieldDataBuilder(name(), searchLookup.source(), sourceValueFetcher(sourcePaths)); + } + + throw new IllegalStateException("unknown field data type [" + operation.name() + "]"); } @Override @@ -1423,7 +1461,11 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return new SourceValueFetcher(name(), context, nullValue) { + return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + } + + private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { + return new SourceValueFetcher(sourcePaths, nullValue) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java index f10eb1bbd1ed4..be2df55b6b7a8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java @@ -46,7 +46,7 @@ public SourceValueFetcher(String fieldName, SearchExecutionContext context, Obje * @param sourcePaths The paths to pull source values from * @param nullValue An optional substitute value if the _source value is `null` */ - private SourceValueFetcher(Set sourcePaths, Object nullValue) { + public SourceValueFetcher(Set sourcePaths, Object nullValue) { this.sourcePaths = sourcePaths; this.nullValue = nullValue; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index e563746ba99d7..2826928274241 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -103,7 +103,7 @@ public String typeName() { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java index 239e191d5676e..88ed2d9224d88 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java @@ -51,7 +51,7 @@ public Query termQuery(Object value, SearchExecutionContext context) { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 3eaabca96c9ab..f1514a959a31a 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -37,6 +37,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MappedFieldType.FielddataOperation; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperParsingException; @@ -91,7 +92,7 @@ public class SearchExecutionContext extends QueryRewriteContext { private final MappingLookup mappingLookup; private final SimilarityService similarityService; private final BitsetFilterCache bitsetFilterCache; - private final BiFunction> indexFieldDataService; + private final BiFunction> indexFieldDataLookup; private SearchLookup lookup = null; private final int shardId; @@ -170,7 +171,7 @@ public SearchExecutionContext(SearchExecutionContext source) { source.shardRequestIndex, source.indexSettings, source.bitsetFilterCache, - source.indexFieldDataService, + source.indexFieldDataLookup, source.mapperService, source.mappingLookup, source.similarityService, @@ -218,7 +219,7 @@ private SearchExecutionContext( this.mapperService = mapperService; this.mappingLookup = mappingLookup; this.bitsetFilterCache = bitsetFilterCache; - this.indexFieldDataService = indexFieldDataLookup; + this.indexFieldDataLookup = indexFieldDataLookup; this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields(); this.nestedScope = new NestedScope(); this.scriptService = scriptService; @@ -278,10 +279,15 @@ public boolean allowExpensiveQueries() { } @SuppressWarnings("unchecked") - public > IFD getForField(MappedFieldType fieldType) { - return (IFD) indexFieldDataService.apply( + public > IFD getForField(MappedFieldType fieldType, FielddataOperation fielddataOperation) { + return (IFD) indexFieldDataLookup.apply( fieldType, - new FieldDataContext(fullyQualifiedIndex.getName(), () -> this.lookup().forkAndTrackFieldReferences(fieldType.name())) + new FieldDataContext( + fullyQualifiedIndex.getName(), + () -> this.lookup().forkAndTrackFieldReferences(fieldType.name()), + this::sourcePath, + fielddataOperation + ) ); } @@ -477,9 +483,9 @@ public SearchLookup lookup() { if (this.lookup == null) { this.lookup = new SearchLookup( this::getFieldType, - (fieldType, searchLookup) -> indexFieldDataService.apply( + (fieldType, searchLookup, fielddataOperation) -> indexFieldDataLookup.apply( fieldType, - new FieldDataContext(fullyQualifiedIndex.getName(), searchLookup) + new FieldDataContext(fullyQualifiedIndex.getName(), searchLookup, this::sourcePath, fielddataOperation) ) ); } diff --git a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java index 37bceefb9a7fd..d6d31b03a8f93 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java @@ -266,7 +266,7 @@ private LongValuesSource createValuesSource(SearchExecutionContext context) { throw new QueryShardException(context, "failed to find minimum_should_match field [" + minimumShouldMatchField + "]"); } - IndexNumericFieldData fieldData = context.getForField(msmFieldType); + IndexNumericFieldData fieldData = context.getForField(msmFieldType, MappedFieldType.FielddataOperation.SEARCH); longValuesSource = new FieldValuesSource(fieldData); } else if (minimumShouldMatchScript != null) { TermsSetQueryScript.Factory factory = context.compile(minimumShouldMatchScript, TermsSetQueryScript.CONTEXT); diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java index 568886f3804f1..efc6f6729f822 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java @@ -260,7 +260,7 @@ private AbstractDistanceScoreFunction parseNumberVariable( DecayFunctionBuilder.ORIGIN ); } - IndexNumericFieldData numericFieldData = context.getForField(fieldType); + IndexNumericFieldData numericFieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); return new NumericFieldDataScoreFunction(origin, scale, decay, offset, getDecayFunction(), numericFieldData, mode); } @@ -300,7 +300,7 @@ private AbstractDistanceScoreFunction parseGeoVariable( } double scale = DistanceUnit.DEFAULT.parse(scaleString, DistanceUnit.DEFAULT); double offset = DistanceUnit.DEFAULT.parse(offsetString, DistanceUnit.DEFAULT); - IndexGeoPointFieldData indexFieldData = context.getForField(fieldType); + IndexGeoPointFieldData indexFieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); return new GeoFieldDataScoreFunction(origin, scale, decay, offset, getDecayFunction(), indexFieldData, mode); } @@ -350,7 +350,7 @@ private AbstractDistanceScoreFunction parseDateVariable( double scale = val.getMillis(); val = TimeValue.parseTimeValue(offsetString, TimeValue.timeValueHours(24), DecayFunctionParser.class.getSimpleName() + ".offset"); double offset = val.getMillis(); - IndexNumericFieldData numericFieldData = context.getForField(dateFieldType); + IndexNumericFieldData numericFieldData = context.getForField(dateFieldType, MappedFieldType.FielddataOperation.SEARCH); return new NumericFieldDataScoreFunction(origin, scale, decay, offset, getDecayFunction(), numericFieldData, mode); } diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java index 3431349332873..50c765d4eba97 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.lucene.search.function.FieldValueFactorFunction; import org.elasticsearch.common.lucene.search.function.ScoreFunction; import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -140,7 +141,7 @@ public Version getMinimalSupportedVersion() { protected ScoreFunction doToFunction(SearchExecutionContext context) { IndexNumericFieldData fieldData = null; if (context.isFieldMapped(field)) { - fieldData = context.getForField(context.getFieldType(field)); + fieldData = context.getForField(context.getFieldType(field), MappedFieldType.FielddataOperation.SEARCH); } else { if (missing == null) { throw new ElasticsearchException("Unable to find a field mapper for field [" + field + "]. No 'missing' value defined."); diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java index 53e09336abfe9..1cbe92ea4c458 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.lucene.search.function.RandomScoreFunction; import org.elasticsearch.common.lucene.search.function.ScoreFunction; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -166,7 +167,11 @@ protected ScoreFunction doToFunction(SearchExecutionContext context) { ); } int seed = this.seed == null ? hash(context.nowInMillis()) : this.seed; - return new RandomScoreFunction(seed, salt, context.getForField(context.getFieldType(fieldName))); + return new RandomScoreFunction( + seed, + salt, + context.getForField(context.getFieldType(fieldName), MappedFieldType.FielddataOperation.SEARCH) + ); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java index 9170afc1af03a..df5ede180ad30 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java @@ -435,7 +435,7 @@ public Analyzer buildCustomAnalyzer( @Override protected IndexFieldData buildFieldData(MappedFieldType ft) { - return context.getForField(ft); + return context.getForField(ft, MappedFieldType.FielddataOperation.SEARCH); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index 8515bff1049d9..07d8a2a872984 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -46,7 +46,7 @@ public FetchSubPhaseProcessor getProcessor(FetchContext context) { } ValueFetcher fetcher = new DocValueFetcher( ft.docValueFormat(fieldAndFormat.format, null), - context.searchLookup().getForField(ft) + context.searchLookup().getForField(ft, MappedFieldType.FielddataOperation.SEARCH) ); fields.add(new DocValueField(fieldAndFormat.field, fetcher)); } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java index 8a71978db46ce..93fbfc1a5b43c 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.util.Maps; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SourceValueFetcherIndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; import org.elasticsearch.script.field.Field; @@ -22,12 +23,13 @@ import java.util.Collection; import java.util.Map; import java.util.Set; +import java.util.function.BiFunction; import java.util.function.Function; public class LeafDocLookup implements Map> { private final Function fieldTypeLookup; - private final Function> fieldDataLookup; + private final BiFunction> fieldDataLookup; private final LeafReaderContext reader; private int docId = -1; @@ -36,7 +38,7 @@ public class LeafDocLookup implements Map> { LeafDocLookup( Function fieldTypeLookup, - Function> fieldDataLookup, + BiFunction> fieldDataLookup, LeafReaderContext reader ) { this.fieldTypeLookup = fieldTypeLookup; @@ -48,9 +50,15 @@ public void setDocument(int docId) { this.docId = docId; } - public DocValuesScriptFieldFactory getScriptFieldFactory(String fieldName) { + protected DocValuesScriptFieldFactory getScriptFieldFactory(String fieldName, MappedFieldType.FielddataOperation options) { DocValuesScriptFieldFactory factory = localCacheScriptFieldData.get(fieldName); + // do not use cached source fallback fields for old style doc access + if (options == MappedFieldType.FielddataOperation.SEARCH + && factory instanceof SourceValueFetcherIndexFieldData.ValueFetcherDocValues) { + factory = null; + } + if (factory == null) { final MappedFieldType fieldType = fieldTypeLookup.apply(fieldName); @@ -63,7 +71,7 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String fieldName) { factory = AccessController.doPrivileged(new PrivilegedAction() { @Override public DocValuesScriptFieldFactory run() { - return fieldDataLookup.apply(fieldType).load(reader).getScriptFieldFactory(fieldName); + return fieldDataLookup.apply(fieldType, options).load(reader).getScriptFieldFactory(fieldName); } }); @@ -80,20 +88,18 @@ public DocValuesScriptFieldFactory run() { } public Field getScriptField(String fieldName) { - return getScriptFieldFactory(fieldName).toScriptField(); + return getScriptFieldFactory(fieldName, MappedFieldType.FielddataOperation.SCRIPT).toScriptField(); } @Override public ScriptDocValues get(Object key) { - String fieldName = key.toString(); - return getScriptFieldFactory(fieldName).toScriptDocValues(); + return getScriptFieldFactory(key.toString(), MappedFieldType.FielddataOperation.SEARCH).toScriptDocValues(); } @Override public boolean containsKey(Object key) { String fieldName = key.toString(); - DocValuesScriptFieldFactory docValuesFieldFactory = localCacheScriptFieldData.get(fieldName); - return docValuesFieldFactory != null || fieldTypeLookup.apply(fieldName) != null; + return localCacheScriptFieldData.get(fieldName) != null || fieldTypeLookup.apply(fieldName) != null; } @Override diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index f2e552f2cf3f6..3b1e53d3061b7 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.lookup; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; @@ -16,7 +17,6 @@ import java.util.LinkedHashSet; import java.util.Objects; import java.util.Set; -import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -41,7 +41,11 @@ public class SearchLookup { private final Set fieldChain; private final SourceLookup sourceLookup; private final Function fieldTypeLookup; - private final BiFunction, IndexFieldData> fieldDataLookup; + private final TriFunction< + MappedFieldType, + Supplier, + MappedFieldType.FielddataOperation, + IndexFieldData> fieldDataLookup; /** * Create the top level field lookup for a search request. Provides a way to look up fields from doc_values, @@ -49,7 +53,7 @@ public class SearchLookup { */ public SearchLookup( Function fieldTypeLookup, - BiFunction, IndexFieldData> fieldDataLookup + TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup ) { this.fieldTypeLookup = fieldTypeLookup; this.fieldChain = Collections.emptySet(); @@ -105,8 +109,8 @@ public MappedFieldType fieldType(String fieldName) { return fieldTypeLookup.apply(fieldName); } - public IndexFieldData getForField(MappedFieldType fieldType) { - return fieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name())); + public IndexFieldData getForField(MappedFieldType fieldType, MappedFieldType.FielddataOperation options) { + return fieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name()), options); } public SourceLookup source() { diff --git a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java index 30a9e5b6d5ad6..895aa5bed5bff 100644 --- a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java @@ -260,7 +260,7 @@ private Query createSliceQuery(int id, int max, SearchExecutionContext context, if (type.hasDocValues() == false) { throw new IllegalArgumentException("cannot load numeric doc values on " + field); } else { - IndexFieldData ifm = context.getForField(type); + IndexFieldData ifm = context.getForField(type, MappedFieldType.FielddataOperation.SEARCH); if (ifm instanceof IndexNumericFieldData == false) { throw new IllegalArgumentException("cannot load numeric doc values on " + field); } diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index 46c88b8915782..19326a214a940 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -362,7 +362,7 @@ public SortFieldAndFormat build(SearchExecutionContext context) throws IOExcepti fieldType = resolveUnmappedType(context); } - IndexFieldData fieldData = context.getForField(fieldType); + IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); if (fieldData instanceof IndexNumericFieldData == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) { throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields"); @@ -462,7 +462,7 @@ public BucketedSort buildBucketedSort(SearchExecutionContext context, BigArrays fieldType = resolveUnmappedType(context); } - IndexFieldData fieldData = context.getForField(fieldType); + IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); if (fieldData instanceof IndexNumericFieldData == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) { throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields"); diff --git a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java index cd9478e7a8a68..4d842da11163d 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java @@ -616,7 +616,7 @@ private IndexGeoPointFieldData fieldData(SearchExecutionContext context) { throw new IllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort"); } } - return context.getForField(fieldType); + return context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); } private Nested nested(SearchExecutionContext context) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index a5d2fe6bf3cf2..6cbfb53c3f584 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -32,6 +32,7 @@ import java.util.Collections; import java.util.Map; +import java.util.Set; import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta; import static org.hamcrest.Matchers.arrayWithSize; @@ -215,6 +216,12 @@ private Sort buildIndexSort(IndexSettings indexSettings, Map indexFieldDataService.getForField(ft, new FieldDataContext("test", s))); + return config.buildIndexSort( + lookup::get, + (ft, s) -> indexFieldDataService.getForField( + ft, + new FieldDataContext("test", s, Set::of, MappedFieldType.FielddataOperation.SEARCH) + ) + ); } } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 0007525f4b357..b9542dd4be315 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -157,7 +157,7 @@ public > IFD getForField(String type, String field } else { throw new UnsupportedOperationException(type); } - return searchExecutionContext.getForField(fieldType); + return searchExecutionContext.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); } @Before diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java index 0bee4ffdf8313..f7fddd9111d0c 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java @@ -56,7 +56,7 @@ public void testFilterByFrequency() throws Exception { .fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0) .build(builderContext) .fieldType(); - IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft); + IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataOperation.SEARCH); for (LeafReaderContext context : contexts) { LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); @@ -71,7 +71,7 @@ public void testFilterByFrequency() throws Exception { .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, 201, 100) .build(builderContext) .fieldType(); - IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft); + IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataOperation.SEARCH); for (LeafReaderContext context : contexts) { LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); @@ -86,7 +86,7 @@ public void testFilterByFrequency() throws Exception { .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101) .build(builderContext) .fieldType(); - IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft); + IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataOperation.SEARCH); for (LeafReaderContext context : contexts) { LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); @@ -102,7 +102,7 @@ public void testFilterByFrequency() throws Exception { .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101) .build(builderContext) .fieldType(); - IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft); + IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataOperation.SEARCH); for (LeafReaderContext context : contexts) { LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index b0517c86a9baa..282e5009d6d0b 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -136,7 +136,7 @@ public void testGetForFieldRuntimeField() { return (IndexFieldData.Builder) (cache, breakerService) -> null; }); SearchLookup searchLookup = new SearchLookup(null, null); - ifdService.getForField(ft, new FieldDataContext("qualified", () -> searchLookup)); + ifdService.getForField(ft, new FieldDataContext("qualified", () -> searchLookup, null, MappedFieldType.FielddataOperation.SEARCH)); assertSame(searchLookup, searchLookupSetOnce.get().get()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index 3e0f860c2f6c1..7ef32966aa131 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -187,7 +187,7 @@ protected static SearchExecutionContext mockContext() { } protected static FieldDataContext mockFielddataContext() { - return new FieldDataContext("test", mockContext()::lookup); + return new FieldDataContext("test", mockContext()::lookup, mockContext()::sourcePath, MappedFieldType.FielddataOperation.SCRIPT); } protected static SearchExecutionContext mockContext(boolean allowExpensiveQueries) { @@ -210,12 +210,14 @@ protected static SearchExecutionContext mockContext(boolean allowExpensiveQuerie when(context.allowExpensiveQueries()).thenReturn(allowExpensiveQueries); SearchLookup lookup = new SearchLookup( context::getFieldType, - (mft, lookupSupplier) -> mft.fielddataBuilder(new FieldDataContext("test", lookupSupplier)).build(null, null) + (mft, lookupSupplier, fdo) -> mft.fielddataBuilder(new FieldDataContext("test", lookupSupplier, context::sourcePath, fdo)) + .build(null, null) ); when(context.lookup()).thenReturn(lookup); - when(context.getForField(any())).then(args -> { + when(context.getForField(any(), any())).then(args -> { MappedFieldType ft = args.getArgument(0); - return ft.fielddataBuilder(new FieldDataContext("test", context::lookup)) + MappedFieldType.FielddataOperation fdo = args.getArgument(1); + return ft.fielddataBuilder(new FieldDataContext("test", context::lookup, context::sourcePath, fdo)) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); }); return context; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java index 8e1d25abfbcfc..44b66a9e89bd2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java @@ -57,7 +57,7 @@ public void testTooManyValues() throws IOException { BooleanFieldScript script = new BooleanFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java index 0885c628538a6..9477c34ddb886 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java @@ -334,7 +334,9 @@ public void testParseDocumentSubFieldAccess() throws IOException { withLuceneIndex(mapperService, iw -> iw.addDocuments(Arrays.asList(doc1.rootDoc(), doc2.rootDoc())), reader -> { SearchLookup searchLookup = new SearchLookup( mapperService::fieldType, - (mft, lookupSupplier) -> mft.fielddataBuilder(new FieldDataContext("test", lookupSupplier)).build(null, null) + (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( + new FieldDataContext("test", lookupSupplier, mapperService.mappingLookup()::sourcePaths, fdo) + ).build(null, null) ); LeafSearchLookup leafSearchLookup = searchLookup.getLeafSearchLookup(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java index d4f8f43db2941..07383b2979633 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java @@ -66,7 +66,7 @@ public void testTooManyValues() throws IOException { DateFieldScript script = new DateFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null), DateFormatter.forPattern(randomDateFormatterPattern()).withLocale(randomLocale(random())), reader.leaves().get(0) ) { @@ -102,7 +102,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { DateFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null), DateFormatter.forPattern("epoch_millis") ); DateFieldScript dateFieldScript = leafFactory.newInstance(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java index 6dbb109709a73..0b45c56581d70 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java @@ -64,7 +64,7 @@ public void testTooManyValues() throws IOException { DoubleFieldScript script = new DoubleFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null), reader.leaves().get(0) ) { @Override @@ -99,7 +99,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { DoubleFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null) ); DoubleFieldScript doubleFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java index 76ebd29e762b1..93e32df4d10f9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java @@ -59,7 +59,7 @@ public void testTooManyValues() throws IOException { GeoPointFieldScript script = new GeoPointFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java index 9a5b0ee44ccf6..3949d590b82eb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java @@ -50,7 +50,10 @@ public void testFetchIgnoredFieldValue() throws IOException { mapperService, iw -> { iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field("field", "value"))).rootDoc()); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchLookup lookup = new SearchLookup( + mapperService::fieldType, + fieldDataLookup(mapperService.mappingLookup()::sourcePaths) + ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.lookup()).thenReturn(lookup); IgnoredFieldMapper.IgnoredFieldType ft = (IgnoredFieldMapper.IgnoredFieldType) mapperService.fieldType("_ignored"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java index 0b20c730d4933..767118aec5544 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java @@ -47,7 +47,7 @@ public void testFetchFieldValue() throws IOException { iw.addDocument(mapperService.documentMapper().parse(source).rootDoc()); }, iw -> { IndexFieldMapper.IndexFieldType ft = (IndexFieldMapper.IndexFieldType) mapperService.fieldType("_index"); - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(mapperService.mappingLookup()::sourcePaths)); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); IndexSearcher searcher = newSearcher(iw); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java index 86ea18b746814..c735284108334 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java @@ -65,7 +65,7 @@ public void testTooManyValues() throws IOException { IpFieldScript script = new IpFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null), reader.leaves().get(0) ) { @Override @@ -100,7 +100,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { IpFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null) ); IpFieldScript ipFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java index 4c657e14e8412..859374ea8de6c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java @@ -64,7 +64,7 @@ public void testTooManyValues() throws IOException { LongFieldScript script = new LongFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null), reader.leaves().get(0) ) { @Override @@ -99,7 +99,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { LongFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null) ); LongFieldScript longFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java index 915268b1896a9..36cf482507b2f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java @@ -61,7 +61,7 @@ public void testFetchValue() throws Exception { .rootDoc() ); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(mapperService.mappingLookup()::sourcePaths)); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); FieldFetcher fieldFetcher = FieldFetcher.create( searchExecutionContext, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java index 1f0b8a7bffc3c..551ad396e06e9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java @@ -75,7 +75,10 @@ public void testFetchIdFieldValue() throws IOException { mapperService, iw -> { iw.addDocument(mapperService.documentMapper().parse(source(id, b -> b.field("field", "value"), null)).rootDoc()); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchLookup lookup = new SearchLookup( + mapperService::fieldType, + fieldDataLookup(mapperService.mappingLookup()::sourcePaths) + ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.lookup()).thenReturn(lookup); ProvidedIdFieldMapper.IdFieldType ft = (ProvidedIdFieldMapper.IdFieldType) mapperService.fieldType("_id"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java index 3e6747102305a..85eb59df7c44a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java @@ -68,7 +68,10 @@ public void testFetchRoutingFieldValue() throws IOException { mapperService, iw -> { iw.addDocument(mapperService.documentMapper().parse(source("1", b -> {}, "abcd")).rootDoc()); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchLookup lookup = new SearchLookup( + mapperService::fieldType, + fieldDataLookup(mapperService.mappingLookup()::sourcePaths) + ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.lookup()).thenReturn(lookup); RoutingFieldMapper.RoutingFieldType ft = (RoutingFieldMapper.RoutingFieldType) mapperService.fieldType("_routing"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java index 4f00772f20a3b..a01234c04af77 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java @@ -63,7 +63,7 @@ public void testTooManyValues() throws IOException { StringFieldScript script = new StringFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null), reader.leaves().get(0) ) { @Override @@ -89,7 +89,7 @@ public void testTooManyChars() throws IOException { StringFieldScript script = new StringFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null), reader.leaves().get(0) ) { @Override @@ -129,7 +129,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { StringFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null) ); StringFieldScript stringFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = stringFieldScript.resultsForDoc(0); @@ -158,7 +158,7 @@ public final void testFromSourceDoesNotEnforceCharsLimit() throws IOException { StringFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null) ); StringFieldScript stringFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = stringFieldScript.resultsForDoc(0); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index c2760466d79e7..3950e64a1fc60 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -536,8 +536,6 @@ public void testFielddata() throws IOException { MapperService enabledMapper = createMapperService(fieldMapping(b -> b.field("type", "text").field("fielddata", true))); enabledMapper.fieldType("field").fielddataBuilder(FieldDataContext.noRuntimeFields("test")); // no exception - // this time - e = expectThrows( MapperParsingException.class, () -> createMapperService(fieldMapping(b -> b.field("type", "text").field("index", false).field("fielddata", true))) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java index b292a57375e59..54714ee863b62 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java @@ -50,7 +50,7 @@ public void testFetchFieldValue() throws IOException { iw.addDocument(parsedDoc.rootDoc()); }, iw -> { VersionFieldMapper.VersionFieldType ft = (VersionFieldMapper.VersionFieldType) mapperService.fieldType("_version"); - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(mapperService.mappingLookup()::sourcePaths)); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); IndexSearcher searcher = newSearcher(iw); diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 8142276dd4a61..897d6e1e29149 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -149,7 +149,7 @@ public void testClusterAlias() throws IOException { IndexFieldMapper mapper = new IndexFieldMapper(); - IndexFieldData forField = context.getForField(mapper.fieldType()); + IndexFieldData forField = context.getForField(mapper.fieldType(), MappedFieldType.FielddataOperation.SEARCH); String expected = clusterAlias == null ? context.getIndexSettings().getIndexMetadata().getIndex().getName() : clusterAlias + ":" + context.getIndexSettings().getIndex().getName(); @@ -598,9 +598,9 @@ private static List collect(String field, SearchExecutionContext searchE MappedFieldType fieldType = searchExecutionContext.getFieldType(field); IndexFieldData indexFieldData; if (randomBoolean()) { - indexFieldData = searchExecutionContext.getForField(fieldType); + indexFieldData = searchExecutionContext.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); } else { - indexFieldData = searchExecutionContext.lookup().getForField(fieldType); + indexFieldData = searchExecutionContext.lookup().getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); } searcher.search(query, new Collector() { @Override diff --git a/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java index 7c477cbcbe27a..fa7bcf308409f 100644 --- a/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java @@ -31,7 +31,7 @@ public void testTooManyValues() throws IOException { CompositeFieldScript script = new CompositeFieldScript( "composite", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null), reader.leaves().get(0) ) { @Override @@ -62,7 +62,7 @@ public void testTooManyChars() throws IOException { CompositeFieldScript script = new CompositeFieldScript( "composite", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null), reader.leaves().get(0) ) { @Override @@ -75,7 +75,7 @@ public void execute() { StringFieldScript stringFieldScript = new StringFieldScript( "composite.leaf", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java index a44b1aba76de3..3845659750138 100644 --- a/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java @@ -39,7 +39,7 @@ public void testValuesLimitIsNotEnforced() throws IOException { try (DirectoryReader reader = iw.getReader()) { SortedNumericDocValuesLongFieldScript docValues = new SortedNumericDocValuesLongFieldScript( "test", - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null), reader.leaves().get(0) ); List values = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java index 002ad85ddbdb8..8a36c68b2d652 100644 --- a/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java @@ -40,7 +40,7 @@ public void testValuesLimitIsNotEnforced() throws IOException { try (DirectoryReader reader = iw.getReader()) { SortedSetDocValuesStringFieldScript docValues = new SortedSetDocValuesStringFieldScript( "test", - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null), reader.leaves().get(0) ); List values = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java index d131311ff1a25..9859809f46fac 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java @@ -635,7 +635,7 @@ public void testOverlappingRanges() throws IOException { */ public void testRuntimeFieldTopLevelQueryNotOptimized() throws IOException { long totalDocs = (long) RangeAggregator.DOCS_PER_RANGE_TO_USE_FILTERS * 4; - SearchLookup lookup = new SearchLookup(s -> null, (ft, l) -> null); + SearchLookup lookup = new SearchLookup(s -> null, (ft, l, ftd) -> null); StringFieldScript.LeafFactory scriptFactory = ctx -> new StringFieldScript("dummy", Map.of(), lookup, ctx) { @Override public void execute() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index 5bddf4967a0cf..10a03c06cd598 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -2044,7 +2044,7 @@ public void testWithFilterAndPreciseSize() throws IOException { */ public void testRuntimeFieldTopLevelNotOptimized() throws IOException { long totalDocs = 500; - SearchLookup lookup = new SearchLookup(s -> null, (ft, l) -> null); + SearchLookup lookup = new SearchLookup(s -> null, (ft, l, ftd) -> null); StringFieldScript.LeafFactory scriptFactory = ctx -> new StringFieldScript("dummy", Map.of(), lookup, ctx) { @Override public void execute() { diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 502c220d6e9fd..d9a08a3426bda 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -234,7 +234,10 @@ public void testMetadataFields() throws IOException { new FieldAndFormat("_ignored", null) ); FieldFetcher fieldFetcher = FieldFetcher.create( - newSearchExecutionContext(mapperService, (ft, fdc) -> fieldDataLookup().apply(ft, fdc.lookupSupplier())), + newSearchExecutionContext( + mapperService, + (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup()).apply(ft, fdc.lookupSupplier(), fdc.fielddataOperation()) + ), fieldList ); IndexSearcher searcher = newSearcher(iw); @@ -1122,7 +1125,7 @@ public void testFetchRuntimeFieldWithSourceDisabled() throws IOException { MapperService mapperService = createMapperService(mapping); SearchExecutionContext searchExecutionContext = newSearchExecutionContext( mapperService, - (ft, fdc) -> fieldDataLookup().apply(ft, fdc.lookupSupplier()) + (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup()).apply(ft, fdc.lookupSupplier(), fdc.fielddataOperation()) ); withLuceneIndex(mapperService, iw -> iw.addDocument(new LuceneDocument()), iw -> { FieldFetcher fieldFetcher = FieldFetcher.create(searchExecutionContext, fieldAndFormatList("runtime_field", null, false)); @@ -1151,7 +1154,7 @@ public void testFetchMetadataFieldWithSourceDisabled() throws IOException { MapperService mapperService = createMapperService(mapping); SearchExecutionContext searchExecutionContext = newSearchExecutionContext( mapperService, - (ft, fdc) -> fieldDataLookup().apply(ft, fdc.lookupSupplier()) + (ft, fdc) -> fieldDataLookup(fdc.sourcePathsLookup()).apply(ft, fdc.lookupSupplier(), fdc.fielddataOperation()) ); withLuceneIndex(mapperService, iw -> { ParsedDocument parsedDocument = mapperService.documentMapper().parse(source("{}")); diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java index 9367f4646b5cc..7c8921e9123a0 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java @@ -19,7 +19,7 @@ import org.junit.Before; import java.io.IOException; -import java.util.function.Function; +import java.util.function.BiFunction; import static org.mockito.AdditionalAnswers.returnsFirstArg; import static org.mockito.ArgumentMatchers.any; @@ -50,7 +50,7 @@ public void setUp() throws Exception { docLookup = new LeafDocLookup( field -> field.equals("field") ? fieldType1 : field.equals("alias") ? fieldType2 : null, - fieldType -> fieldType == fieldType1 ? fieldData1 : fieldType == fieldType2 ? fieldData2 : null, + (fieldType, fielddataType) -> fieldType == fieldType1 ? fieldData1 : fieldType == fieldType2 ? fieldData2 : null, null ); } @@ -77,7 +77,7 @@ public void testFlattenedField() throws IOException { MappedFieldType fieldType1 = fieldType.getChildFieldType("key1"); MappedFieldType fieldType2 = fieldType.getChildFieldType("key2"); - Function> fieldDataSupplier = ft -> { + BiFunction> fieldDataSupplier = (ft, fdt) -> { FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) ft; return keyedFieldType.key().equals("key1") ? fieldData1 : fieldData2; }; diff --git a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java index ee16d22a90f25..4bf7b4d766911 100644 --- a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java @@ -171,7 +171,7 @@ public Query existsQuery(SearchExecutionContext context) { when(context.getIndexSettings()).thenReturn(indexSettings); if (dvType != null) { IndexNumericFieldData fd = mock(IndexNumericFieldData.class); - when(context.getForField(fieldType)).thenReturn(fd); + when(context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH)).thenReturn(fd); } return context; diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 9cdff6438faf5..45d55fa86f57d 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -21,6 +21,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -77,7 +78,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.BiFunction; import java.util.function.BooleanSupplier; import java.util.function.Function; import java.util.function.Supplier; @@ -644,8 +644,10 @@ public void onRemoval(ShardId shardId, Accountable accountable) { ); } - protected BiFunction, IndexFieldData> fieldDataLookup() { - return (mft, lookupSource) -> mft.fielddataBuilder(new FieldDataContext("test", lookupSource)) + protected TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup( + Function> sourcePathsLookup + ) { + return (mft, lookupSource, fdo) -> mft.fielddataBuilder(new FieldDataContext("test", lookupSource, sourcePathsLookup, fdo)) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 7cc221bc2c608..387fa339e65c4 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -322,8 +322,11 @@ protected final List fetchFromDocValues(MapperService mapperService, MappedFi mapperService, iw -> { iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field(ft.name(), sourceValue))).rootDoc()); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); - ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft)); + SearchLookup lookup = new SearchLookup( + mapperService::fieldType, + fieldDataLookup(mapperService.mappingLookup()::sourcePaths) + ); + ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft, MappedFieldType.FielddataOperation.SEARCH)); IndexSearcher searcher = newSearcher(iw); LeafReaderContext context = searcher.getIndexReader().leaves().get(0); lookup.source().setSegmentAndDocument(context, 0); @@ -573,6 +576,7 @@ protected void registerDimensionChecks(ParameterChecker checker) throws IOExcept */ protected void assertFetch(MapperService mapperService, String field, Object value, String format) throws IOException { MappedFieldType ft = mapperService.fieldType(field); + MappedFieldType.FielddataOperation fdt = MappedFieldType.FielddataOperation.SEARCH; SourceToParse source = source(b -> b.field(ft.name(), value)); ValueFetcher docValueFetcher = new DocValueFetcher( ft.docValueFormat(format, null), @@ -582,8 +586,12 @@ protected void assertFetch(MapperService mapperService, String field, Object val SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); - when(searchExecutionContext.getForField(ft)).thenAnswer( - inv -> fieldDataLookup().apply(ft, () -> { throw new UnsupportedOperationException(); }) + when(searchExecutionContext.getForField(ft, fdt)).thenAnswer( + inv -> fieldDataLookup(mapperService.mappingLookup()::sourcePaths).apply( + ft, + () -> { throw new UnsupportedOperationException(); }, + fdt + ) ); ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); @@ -659,7 +667,6 @@ public final void testIndexTimeFieldData() throws IOException { .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) .load(ctx) .getScriptFieldFactory("test"); - docValuesFieldSource.setNextDocId(0); DocumentLeafReader reader = new DocumentLeafReader(doc.rootDoc(), Collections.emptyMap()); @@ -698,7 +705,7 @@ public final void testIndexTimeStoredFieldsAccess() throws IOException { SourceToParse source = source(this::writeField); ParsedDocument doc = mapperService.documentMapper().parse(source); - SearchLookup lookup = new SearchLookup(f -> fieldType, (f, s) -> { throw new UnsupportedOperationException(); }); + SearchLookup lookup = new SearchLookup(f -> fieldType, (f, s, t) -> { throw new UnsupportedOperationException(); }); withLuceneIndex(mapperService, iw -> iw.addDocument(doc.rootDoc()), ir -> { diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index dc184b54356ef..4b3010c282f95 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -291,10 +291,15 @@ protected AggregationContext createAggregationContext( .map(ft -> new FieldAliasMapper(ft.name() + "-alias", ft.name() + "-alias", ft.name())) .collect(toList()) ); - BiFunction> fieldDataBuilder = (fieldType, context) -> fieldType - .fielddataBuilder(new FieldDataContext(indexSettings.getIndex().getName(), context.lookupSupplier())) - .build(new IndexFieldDataCache.None(), breakerService); + .fielddataBuilder( + new FieldDataContext( + indexSettings.getIndex().getName(), + context.lookupSupplier(), + context.sourcePathsLookup(), + context.fielddataOperation() + ) + ).build(new IndexFieldDataCache.None(), breakerService); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() { @Override public void onRemoval(ShardId shardId, Accountable accountable) {} diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java index 2700af1005aca..09b2e04b9201c 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java @@ -122,7 +122,9 @@ public void testUsedInScript() throws IOException { when(searchExecutionContext.allowExpensiveQueries()).thenReturn(true); SearchLookup lookup = new SearchLookup( searchExecutionContext::getFieldType, - (mft, lookupSupplier) -> mft.fielddataBuilder(new FieldDataContext("test", lookupSupplier)).build(null, null) + (mft, lookupSupplier, fdo) -> mft.fielddataBuilder( + new FieldDataContext("test", lookupSupplier, searchExecutionContext::sourcePath, fdo) + ).build(null, null) ); when(searchExecutionContext.lookup()).thenReturn(lookup); IndexSearcher searcher = newSearcher(reader); diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java index e2073803f56bb..9911eae3b5491 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java @@ -119,7 +119,7 @@ private static List build(SearchExecutionContext context, Str if (fieldType == null) { throw new IllegalArgumentException("Unknown field: [" + field + "]"); } - IndexFieldData fieldData = context.getForField(fieldType); + IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); fetchers.add(new FieldValueFetcher(field, fieldType, fieldData, getValidator(field, validTypes))); } return Collections.unmodifiableList(fetchers);