From 922753545f2a9fe3ed8d8e88e0a13bd94e5b7c5a Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Thu, 16 Jun 2022 10:47:18 -0700 Subject: [PATCH 01/23] Update SearchLookup to include source paths for each field. --- .../benchmark/script/ScriptScoreBenchmark.java | 5 ++++- .../expression/ExpressionFieldScriptTests.java | 7 ++++++- .../expression/ExpressionNumberSortScriptTests.java | 7 ++++++- .../expression/ExpressionTermsSetQueryTests.java | 7 ++++++- .../elasticsearch/index/mapper/DocumentParser.java | 3 ++- .../index/query/SearchExecutionContext.java | 3 ++- .../elasticsearch/search/lookup/SearchLookup.java | 10 +++++++++- .../index/fielddata/IndexFieldDataServiceTests.java | 2 +- .../mapper/AbstractScriptFieldTypeTestCase.java | 3 ++- .../index/mapper/BooleanFieldScriptTests.java | 2 +- .../index/mapper/CompositeRuntimeFieldTests.java | 3 ++- .../index/mapper/DateFieldScriptTests.java | 2 +- .../index/mapper/DoubleFieldScriptTests.java | 2 +- .../index/mapper/GeoPointFieldScriptTests.java | 2 +- .../index/mapper/IgnoredFieldMapperTests.java | 6 +++++- .../index/mapper/IndexFieldMapperTests.java | 2 +- .../index/mapper/IpFieldScriptTests.java | 2 +- .../index/mapper/LongFieldScriptTests.java | 2 +- .../index/mapper/PlaceHolderFieldMapperTests.java | 2 +- .../index/mapper/ProvidedIdFieldMapperTests.java | 6 +++++- .../index/mapper/RoutingFieldMapperTests.java | 6 +++++- .../index/mapper/StringFieldScriptTests.java | 4 ++-- .../index/mapper/VersionFieldMapperTests.java | 2 +- .../bucket/range/RangeAggregatorTests.java | 2 +- .../bucket/terms/TermsAggregatorTests.java | 2 +- .../aggregations/support/ScriptValuesTests.java | 2 +- .../search/query/ScriptScoreQueryTests.java | 2 +- ...GeoPointScriptFieldDistanceFeatureQueryTests.java | 2 +- .../LongScriptFieldDistanceFeatureQueryTests.java | 2 +- .../elasticsearch/index/mapper/MapperTestCase.java | 12 ++++++++++-- .../mapper/AggregateDoubleMetricFieldTypeTests.java | 3 ++- 31 files changed, 84 insertions(+), 33 deletions(-) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index 92d1cbdf96cef..3c76b4cae6627 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -56,6 +56,7 @@ import java.nio.file.Path; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.TimeUnit; /** @@ -82,10 +83,12 @@ public class ScriptScoreBenchmark { Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null)) ); private final IndexFieldDataCache fieldDataCache = new IndexFieldDataCache.None(); + private final Map> sourcePaths = Map.of("n", Set.of("n")); private final CircuitBreakerService breakerService = new NoneCircuitBreakerService(); private SearchLookup lookup = new SearchLookup( fieldTypes::get, - (mft, lookup) -> mft.fielddataBuilder("test", lookup).build(fieldDataCache, breakerService) + (mft, lookup) -> mft.fielddataBuilder("test", lookup).build(fieldDataCache, breakerService), + sourcePaths::get ); @Param({ "expression", "metal", "painless_cast", "painless_def" }) diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java index b8ce67ef13d66..9b225285b029e 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.text.ParseException; import java.util.Collections; +import java.util.Set; import static org.hamcrest.Matchers.equalTo; import static org.mockito.ArgumentMatchers.any; @@ -49,7 +50,11 @@ public void setUp() throws Exception { when(fieldData.load(any())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(field -> field.equals("field") ? fieldType : null, (ignored, _lookup) -> fieldData); + lookup = new SearchLookup( + field -> field.equals("field") ? fieldType : null, + (ignored, _lookup) -> fieldData, + field -> field.equals("field") ? Set.of("field") : null + ); } private FieldScript.LeafFactory compile(String expression) { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java index a3f1a750053c8..7624d56021e10 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.text.ParseException; import java.util.Collections; +import java.util.Set; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; @@ -50,7 +51,11 @@ public void setUp() throws Exception { when(fieldData.load(any())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(field -> field.equals("field") ? fieldType : null, (ignored, _lookup) -> fieldData); + lookup = new SearchLookup( + field -> field.equals("field") ? fieldType : null, + (ignored, _lookup) -> fieldData, + field -> field.equals("field") ? Set.of("field") : null + ); } private NumberSortScript.LeafFactory compile(String expression) { diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java index faa08db55138b..d44c5c2ae676f 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.text.ParseException; import java.util.Collections; +import java.util.Set; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; @@ -49,7 +50,11 @@ public void setUp() throws Exception { when(fieldData.load(any())).thenReturn(atomicFieldData); service = new ExpressionScriptEngine(); - lookup = new SearchLookup(field -> field.equals("field") ? fieldType : null, (ignored, _lookup) -> fieldData); + lookup = new SearchLookup( + field -> field.equals("field") ? fieldType : null, + (ignored, _lookup) -> fieldData, + field -> field.equals("field") ? Set.of("field") : null + ); } private TermsSetQueryScript.LeafFactory compile(String expression) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 067061443d885..8bc41cbaa2258 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -142,7 +142,8 @@ private static void executeIndexTimeScripts(DocumentParserContext context) { SearchLookup searchLookup = new SearchLookup( context.mappingLookup().indexTimeLookup()::get, (ft, lookup) -> ft.fielddataBuilder(context.indexSettings().getIndex().getName(), lookup) - .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), + context.mappingLookup()::sourcePaths ); // field scripts can be called both by the loop at the end of this method and via // the document reader, so to ensure that we don't run them multiple times we diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index c3c0a6b24ff75..9e3ca706f318d 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -478,7 +478,8 @@ public SearchLookup lookup() { if (this.lookup == null) { this.lookup = new SearchLookup( this::getFieldType, - (fieldType, searchLookup) -> indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup) + (fieldType, searchLookup) -> indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup), + this::sourcePath ); } return this.lookup; diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index f2e552f2cf3f6..e30851cc171b8 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -42,6 +42,7 @@ public class SearchLookup { private final SourceLookup sourceLookup; private final Function fieldTypeLookup; private final BiFunction, IndexFieldData> fieldDataLookup; + private final Function> sourcePathsLookup; /** * Create the top level field lookup for a search request. Provides a way to look up fields from doc_values, @@ -49,12 +50,14 @@ public class SearchLookup { */ public SearchLookup( Function fieldTypeLookup, - BiFunction, IndexFieldData> fieldDataLookup + BiFunction, IndexFieldData> fieldDataLookup, + Function> sourcePathsLookup ) { this.fieldTypeLookup = fieldTypeLookup; this.fieldChain = Collections.emptySet(); this.sourceLookup = new SourceLookup(); this.fieldDataLookup = fieldDataLookup; + this.sourcePathsLookup = sourcePathsLookup; } /** @@ -69,6 +72,7 @@ private SearchLookup(SearchLookup searchLookup, Set fieldChain) { this.sourceLookup = searchLookup.sourceLookup; this.fieldTypeLookup = searchLookup.fieldTypeLookup; this.fieldDataLookup = searchLookup.fieldDataLookup; + this.sourcePathsLookup = searchLookup.sourcePathsLookup; } /** @@ -109,6 +113,10 @@ public IndexFieldData getForField(MappedFieldType fieldType) { return fieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name())); } + public Set sourcePaths(String fieldName) { + return sourcePathsLookup.apply(fieldName); + } + public SourceLookup source() { return sourceLookup; } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 83ecc8f7552e2..d1a8c2abf992b 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -135,7 +135,7 @@ public void testGetForFieldRuntimeField() { searchLookupSetOnce.set(searchLookup); return (IndexFieldData.Builder) (cache, breakerService) -> null; }); - SearchLookup searchLookup = new SearchLookup(null, null); + SearchLookup searchLookup = new SearchLookup(null, null, null); ifdService.getForField(ft, "qualified", () -> searchLookup); assertSame(searchLookup, searchLookupSetOnce.get().get()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index d5d8b1b4d273c..24d4ebf31fe85 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -202,7 +202,8 @@ protected static SearchExecutionContext mockContext(boolean allowExpensiveQuerie when(context.allowExpensiveQueries()).thenReturn(allowExpensiveQueries); SearchLookup lookup = new SearchLookup( context::getFieldType, - (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null) + (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null), + context::sourcePath ); when(context.lookup()).thenReturn(lookup); return context; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java index 67c15548fcf13..e875f3718a24e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java @@ -57,7 +57,7 @@ public void testTooManyValues() throws IOException { BooleanFieldScript script = new BooleanFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java index 019023e0043c3..78e3a9921db0e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java @@ -333,7 +333,8 @@ public void testParseDocumentSubFieldAccess() throws IOException { withLuceneIndex(mapperService, iw -> iw.addDocuments(Arrays.asList(doc1.rootDoc(), doc2.rootDoc())), reader -> { SearchLookup searchLookup = new SearchLookup( mapperService::fieldType, - (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null) + (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null), + mapperService.mappingLookup()::sourcePaths ); LeafSearchLookup leafSearchLookup = searchLookup.getLeafSearchLookup(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java index 48bf8a465a878..8a2d9572e94f7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java @@ -61,7 +61,7 @@ public void testTooManyValues() throws IOException { DateFieldScript script = new DateFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), DateFormatter.forPattern(randomDateFormatterPattern()).withLocale(randomLocale(random())), reader.leaves().get(0) ) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java index 4235770f4ba41..6e7766298c558 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java @@ -59,7 +59,7 @@ public void testTooManyValues() throws IOException { DoubleFieldScript script = new DoubleFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java index 2cd554eda0f30..8ba165ca6e025 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java @@ -59,7 +59,7 @@ public void testTooManyValues() throws IOException { GeoPointFieldScript script = new GeoPointFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java index 9a5b0ee44ccf6..49c663db35d2e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java @@ -50,7 +50,11 @@ public void testFetchIgnoredFieldValue() throws IOException { mapperService, iw -> { iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field("field", "value"))).rootDoc()); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchLookup lookup = new SearchLookup( + mapperService::fieldType, + fieldDataLookup(), + mapperService.mappingLookup()::sourcePaths + ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.lookup()).thenReturn(lookup); IgnoredFieldMapper.IgnoredFieldType ft = (IgnoredFieldMapper.IgnoredFieldType) mapperService.fieldType("_ignored"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java index 0b20c730d4933..4efdf9e57aca9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java @@ -47,7 +47,7 @@ public void testFetchFieldValue() throws IOException { iw.addDocument(mapperService.documentMapper().parse(source).rootDoc()); }, iw -> { IndexFieldMapper.IndexFieldType ft = (IndexFieldMapper.IndexFieldType) mapperService.fieldType("_index"); - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(), mapperService.mappingLookup()::sourcePaths); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); IndexSearcher searcher = newSearcher(iw); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java index 640378f3fd06d..0db9fac21a20c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java @@ -59,7 +59,7 @@ public void testTooManyValues() throws IOException { IpFieldScript script = new IpFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java index 06f148fbc92c0..8d3730b571780 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java @@ -59,7 +59,7 @@ public void testTooManyValues() throws IOException { LongFieldScript script = new LongFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java index 915268b1896a9..955c48b342bb2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java @@ -61,7 +61,7 @@ public void testFetchValue() throws Exception { .rootDoc() ); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(), mapperService.mappingLookup()::sourcePaths); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); FieldFetcher fieldFetcher = FieldFetcher.create( searchExecutionContext, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java index 8181e5c8f23a0..52a0703244fc1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java @@ -74,7 +74,11 @@ public void testFetchIdFieldValue() throws IOException { mapperService, iw -> { iw.addDocument(mapperService.documentMapper().parse(source(id, b -> b.field("field", "value"), null)).rootDoc()); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchLookup lookup = new SearchLookup( + mapperService::fieldType, + fieldDataLookup(), + mapperService.mappingLookup()::sourcePaths + ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.lookup()).thenReturn(lookup); ProvidedIdFieldMapper.IdFieldType ft = (ProvidedIdFieldMapper.IdFieldType) mapperService.fieldType("_id"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java index 3e6747102305a..bd644731baf9e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java @@ -68,7 +68,11 @@ public void testFetchRoutingFieldValue() throws IOException { mapperService, iw -> { iw.addDocument(mapperService.documentMapper().parse(source("1", b -> {}, "abcd")).rootDoc()); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchLookup lookup = new SearchLookup( + mapperService::fieldType, + fieldDataLookup(), + mapperService.mappingLookup()::sourcePaths + ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.lookup()).thenReturn(lookup); RoutingFieldMapper.RoutingFieldType ft = (RoutingFieldMapper.RoutingFieldType) mapperService.fieldType("_routing"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java index c7e8e9482d6bd..03314b3735bd9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java @@ -59,7 +59,7 @@ public void testTooManyValues() throws IOException { StringFieldScript script = new StringFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override @@ -85,7 +85,7 @@ public void testTooManyChars() throws IOException { StringFieldScript script = new StringFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java index b292a57375e59..ab6895f3f26fd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java @@ -50,7 +50,7 @@ public void testFetchFieldValue() throws IOException { iw.addDocument(parsedDoc.rootDoc()); }, iw -> { VersionFieldMapper.VersionFieldType ft = (VersionFieldMapper.VersionFieldType) mapperService.fieldType("_version"); - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(), mapperService.mappingLookup()::sourcePaths); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); IndexSearcher searcher = newSearcher(iw); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java index d131311ff1a25..3f68060e7b733 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java @@ -635,7 +635,7 @@ public void testOverlappingRanges() throws IOException { */ public void testRuntimeFieldTopLevelQueryNotOptimized() throws IOException { long totalDocs = (long) RangeAggregator.DOCS_PER_RANGE_TO_USE_FILTERS * 4; - SearchLookup lookup = new SearchLookup(s -> null, (ft, l) -> null); + SearchLookup lookup = new SearchLookup(s -> null, (ft, l) -> null, s -> null); StringFieldScript.LeafFactory scriptFactory = ctx -> new StringFieldScript("dummy", Map.of(), lookup, ctx) { @Override public void execute() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index fe25e18553af0..5fbd0fcb20be2 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -2043,7 +2043,7 @@ public void testWithFilterAndPreciseSize() throws IOException { */ public void testRuntimeFieldTopLevelNotOptimized() throws IOException { long totalDocs = 500; - SearchLookup lookup = new SearchLookup(s -> null, (ft, l) -> null); + SearchLookup lookup = new SearchLookup(s -> null, (ft, l) -> null, s -> null); StringFieldScript.LeafFactory scriptFactory = ctx -> new StringFieldScript("dummy", Map.of(), lookup, ctx) { @Override public void execute() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java index cd9d8dfb5840b..6e90fbdd54790 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java @@ -36,7 +36,7 @@ private static class FakeAggregationScript extends AggregationScript { int index; FakeAggregationScript(Object[][] values) { - super(Collections.emptyMap(), new SearchLookup(null, null) { + super(Collections.emptyMap(), new SearchLookup(null, null, null) { @Override public LeafSearchLookup getLeafSearchLookup(LeafReaderContext context) { diff --git a/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java b/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java index 5555b413dfecc..74b2000024e8b 100644 --- a/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java @@ -49,7 +49,7 @@ public class ScriptScoreQueryTests extends ESTestCase { private DirectoryReader reader; private IndexSearcher searcher; private LeafReaderContext leafReaderContext; - private final SearchLookup lookup = new SearchLookup(null, null); + private final SearchLookup lookup = new SearchLookup(null, null, null); @Before public void initSearcher() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java index d06b2c8ae7ff1..ed6fb67ab58eb 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java @@ -81,7 +81,7 @@ public void testMatches() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"location\": [-3.56, -45.98]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - SearchLookup searchLookup = new SearchLookup(null, null); + SearchLookup searchLookup = new SearchLookup(null, null, null); Function leafFactory = ctx -> new GeoPointFieldScript( "test", Map.of(), diff --git a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java index 8127ff8d3a9aa..cc42359c32d20 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java @@ -68,7 +68,7 @@ public void testMatches() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181351]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - SearchLookup searchLookup = new SearchLookup(null, null); + SearchLookup searchLookup = new SearchLookup(null, null, null); Function leafFactory = ctx -> new DateFieldScript( "test", Map.of(), diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 02a9053084b8a..baa22affde3fb 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -340,7 +340,11 @@ protected final List fetchFromDocValues(MapperService mapperService, MappedFi mapperService, iw -> { iw.addDocument(mapperService.documentMapper().parse(source(b -> b.field(ft.name(), sourceValue))).rootDoc()); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup()); + SearchLookup lookup = new SearchLookup( + mapperService::fieldType, + fieldDataLookup(), + mapperService.mappingLookup()::sourcePaths + ); ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft)); IndexSearcher searcher = newSearcher(iw); LeafReaderContext context = searcher.getIndexReader().leaves().get(0); @@ -715,7 +719,11 @@ public final void testIndexTimeStoredFieldsAccess() throws IOException { SourceToParse source = source(this::writeField); ParsedDocument doc = mapperService.documentMapper().parse(source); - SearchLookup lookup = new SearchLookup(f -> fieldType, (f, s) -> { throw new UnsupportedOperationException(); }); + SearchLookup lookup = new SearchLookup( + f -> fieldType, + (f, s) -> { throw new UnsupportedOperationException(); }, + f -> f.equals("field") ? Set.of("field") : null + ); withLuceneIndex(mapperService, iw -> iw.addDocument(doc.rootDoc()), ir -> { diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java index 6ccb2cac33daf..bad5ae6ea62b3 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java @@ -121,7 +121,8 @@ public void testUsedInScript() throws IOException { when(searchExecutionContext.allowExpensiveQueries()).thenReturn(true); SearchLookup lookup = new SearchLookup( searchExecutionContext::getFieldType, - (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null) + (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null), + searchExecutionContext::sourcePath ); when(searchExecutionContext.lookup()).thenReturn(lookup); IndexSearcher searcher = newSearcher(reader); From ad63384b38b4f2d9632bc4396865a04f59dde5ec Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Thu, 16 Jun 2022 12:45:06 -0700 Subject: [PATCH 02/23] Add source fallback for keyword field. --- .../test/painless/50_script_doc_values.yml | 83 ++++++++++++++++++- .../index/mapper/KeywordFieldMapper.java | 20 +++-- .../script/AbstractFieldScript.java | 11 +++ .../script/StringFieldScript.java | 17 ++++ 4 files changed, 124 insertions(+), 7 deletions(-) diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index 63f01c4f53b36..e41b6194da23f 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -19,6 +19,9 @@ setup: type: ip keyword: type: keyword + keyword_no_doc_values: + type: keyword + doc_values: false long: type: long integer: @@ -57,6 +60,7 @@ setup: geo_point: 41.12,-71.34 ip: 192.168.0.19 keyword: not split at all + keyword_no_doc_values: no doc values long: 12348732141234 integer: 134134566 short: 1324 @@ -86,6 +90,7 @@ setup: nanos: [2015-01-01T12:10:30.123456789Z, 2015-01-01T12:10:30.987654321Z] geo_point: [[-71.34,41.12],[60.32,21.25]] keyword: ["one string", "another string"] + keyword_no_doc_values: ["no doc values 0", "no doc values 1", "no doc values 2"] long: [1152921504606846976, 576460752303423488] integer: [5, 17, 29] short: [6, 18, 30, 45] @@ -211,7 +216,6 @@ setup: source: "field('boolean').size()" - match: { hits.hits.0.fields.field.0: 0 } - --- "date": - skip: @@ -863,6 +867,83 @@ setup: - match: { hits.hits.1.fields.field.0: "" } - match: { hits.hits.2.fields.field.0: "another string, one string" } +--- +"keyword_no_doc_values": + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "doc['keyword_no_doc_values'].get(0)" + - match: { hits.hits.0.fields.field.0: "no doc values" } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "doc['keyword_no_doc_values'].value" + - match: { hits.hits.0.fields.field.0: "no doc values" } + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "field('keyword_no_doc_values').get('missing')" + - match: { hits.hits.0.fields.field.0: "no doc values" } + - match: { hits.hits.1.fields.field.0: "missing" } + - match: { hits.hits.2.fields.field.0: "no doc values 0" } # doc values are sorted + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('keyword_no_doc_values', 'missing')" + # same as `field('keyword').get('missing')` + - match: { hits.hits.0.fields.field.0: "no doc values" } + - match: { hits.hits.1.fields.field.0: "missing" } + - match: { hits.hits.2.fields.field.0: "no doc values 0" } + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "field('keyword_no_doc_values').get(1, 'dne')" + - match: { hits.hits.0.fields.field.0: "dne" } + - match: { hits.hits.1.fields.field.0: "dne" } + - match: { hits.hits.2.fields.field.0: "no doc values 1" } # doc values are sorted + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "String.join(', ', field('keyword_no_doc_values'))" + - match: { hits.hits.0.fields.field.0: "no doc values" } + - match: { hits.hits.1.fields.field.0: "" } + - match: { hits.hits.2.fields.field.0: "no doc values 0, no doc values 1, no doc values 2" } + --- "long": - skip: diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 96ce4a2f9fd27..9c7cd942121de 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -47,6 +47,7 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.StringScriptFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; @@ -682,12 +683,19 @@ NamedAnalyzer normalizer() { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - failIfNoDocValues(); - return new SortedSetOrdinalsIndexFieldData.Builder( - name(), - CoreValuesSourceType.KEYWORD, - (dv, n) -> new KeywordDocValuesField(FieldData.toString(dv), n) - ); + if (hasDocValues()) { + return new SortedSetOrdinalsIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + (dv, n) -> new KeywordDocValuesField(FieldData.toString(dv), n) + ); + } else { + return new StringScriptFieldData.Builder( + name(), + StringFieldScript.PARSE_FROM_SOURCE_PATHS.newFactory(name(), Collections.emptyMap(), searchLookup.get()), + KeywordDocValuesField::new + ); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/script/AbstractFieldScript.java b/server/src/main/java/org/elasticsearch/script/AbstractFieldScript.java index d940141bdf198..452f2ee4de7ad 100644 --- a/server/src/main/java/org/elasticsearch/script/AbstractFieldScript.java +++ b/server/src/main/java/org/elasticsearch/script/AbstractFieldScript.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.function.Function; import static org.elasticsearch.core.TimeValue.timeValueMillis; @@ -67,6 +68,7 @@ static ScriptContext newContext(String name, Class factoryClass) { ); protected final String fieldName; + protected final Set sourcePaths; protected final SourceLookup sourceLookup; private final Map params; @@ -75,6 +77,7 @@ public AbstractFieldScript(String fieldName, Map params, SearchL this.fieldName = fieldName; Map docAsMap = docAsMap(); + this.sourcePaths = searchLookup.sourcePaths(fieldName); this.sourceLookup = (SourceLookup) docAsMap.get("_source"); params = new HashMap<>(params); params.put("_source", sourceLookup); @@ -111,6 +114,14 @@ protected final void emitFromSource() { } } + protected final void emitFromSourcePaths() { + for (String sourcePath : sourcePaths) { + for (Object v : extractFromSource(sourcePath)) { + emitFromObject(v); + } + } + } + /** * Check if the we can add another value to the list of values. * @param currentSize the current size of the list diff --git a/server/src/main/java/org/elasticsearch/script/StringFieldScript.java b/server/src/main/java/org/elasticsearch/script/StringFieldScript.java index 0907234e5a119..fc27f4e6030ab 100644 --- a/server/src/main/java/org/elasticsearch/script/StringFieldScript.java +++ b/server/src/main/java/org/elasticsearch/script/StringFieldScript.java @@ -43,6 +43,23 @@ public boolean isResultDeterministic() { } }; + public static final StringFieldScript.Factory PARSE_FROM_SOURCE_PATHS = new Factory() { + @Override + public LeafFactory newFactory(String field, Map params, SearchLookup lookup) { + return ctx -> new StringFieldScript(field, params, lookup, ctx) { + @Override + public void execute() { + emitFromSourcePaths(); + } + }; + } + + @Override + public boolean isResultDeterministic() { + return true; + } + }; + public static Factory leafAdapter(Function parentFactory) { return (leafFieldName, params, searchLookup) -> { CompositeFieldScript.LeafFactory parentLeafFactory = parentFactory.apply(searchLookup); From d110c7d40102f458113adf9a488af64003afd7ff Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 20 Jun 2022 08:39:40 -0700 Subject: [PATCH 03/23] Update some tests for keyword source fallback. --- .../java/org/elasticsearch/index/mapper/MappedFieldType.java | 2 +- .../runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java | 3 ++- .../runtime/LongScriptFieldDistanceFeatureQueryTests.java | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 9fb23f977f6aa..dc94d664f2ebe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -175,7 +175,7 @@ public boolean isAggregatable() { try { fielddataBuilder("", () -> { throw new UnsupportedOperationException("SearchLookup not available"); }); return true; - } catch (IllegalArgumentException e) { + } catch (UnsupportedOperationException | IllegalArgumentException e) { return false; } } diff --git a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java index ed6fb67ab58eb..b1e8433c3b144 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import static org.hamcrest.Matchers.equalTo; @@ -81,7 +82,7 @@ public void testMatches() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"location\": [-3.56, -45.98]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - SearchLookup searchLookup = new SearchLookup(null, null, null); + SearchLookup searchLookup = new SearchLookup(null, null, f -> "location".equals(f) ? Set.of("location") : null); Function leafFactory = ctx -> new GeoPointFieldScript( "test", Map.of(), diff --git a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java index cc42359c32d20..c06aab3816261 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import static org.hamcrest.Matchers.equalTo; @@ -68,7 +69,7 @@ public void testMatches() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181351]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - SearchLookup searchLookup = new SearchLookup(null, null, null); + SearchLookup searchLookup = new SearchLookup(null, null, f -> "timestamp".equals(f) ? Set.of("timestamp") : null); Function leafFactory = ctx -> new DateFieldScript( "test", Map.of(), From 892b27aec2b735f5c873956aa2e6e95b5c53fad5 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Fri, 24 Jun 2022 15:25:53 -0700 Subject: [PATCH 04/23] separate script source fallback into its own method --- .../script/ScriptScoreBenchmark.java | 6 ++ .../search/QueryParserHelperBenchmark.java | 6 ++ .../ExpressionFieldScriptTests.java | 2 + .../ExpressionNumberSortScriptTests.java | 2 + .../ExpressionTermsSetQueryTests.java | 2 + .../test/painless/50_script_doc_values.yml | 6 +- .../org/elasticsearch/index/IndexService.java | 1 + .../fielddata/IndexFieldDataService.java | 50 +++++++++---- .../index/mapper/DocumentParser.java | 9 +++ .../index/mapper/KeywordFieldMapper.java | 31 +++++--- .../index/mapper/MappedFieldType.java | 23 +++++- .../index/query/SearchExecutionContext.java | 22 ++++-- .../script/AbstractFieldScript.java | 11 --- .../script/StringFieldScript.java | 24 +++++++ .../search/lookup/LeafDocLookup.java | 71 ++++++++++++++----- .../search/lookup/SearchLookup.java | 11 ++- .../MetadataCreateIndexServiceTests.java | 1 + .../fielddata/IndexFieldDataServiceTests.java | 2 +- .../AbstractScriptFieldTypeTestCase.java | 2 + .../index/mapper/BooleanFieldScriptTests.java | 2 +- .../mapper/CompositeRuntimeFieldTests.java | 2 + .../index/mapper/DateFieldScriptTests.java | 2 +- .../index/mapper/DateFieldTypeTests.java | 3 + .../index/mapper/DoubleFieldScriptTests.java | 2 +- .../mapper/FieldNamesFieldTypeTests.java | 1 + .../mapper/GeoPointFieldScriptTests.java | 2 +- .../index/mapper/IgnoredFieldMapperTests.java | 1 + .../index/mapper/IndexFieldMapperTests.java | 7 +- .../index/mapper/IndexFieldTypeTests.java | 1 + .../index/mapper/IpFieldScriptTests.java | 2 +- .../index/mapper/LongFieldScriptTests.java | 2 +- .../index/mapper/NumberFieldTypeTests.java | 1 + .../mapper/PlaceHolderFieldMapperTests.java | 7 +- .../mapper/ProvidedIdFieldMapperTests.java | 1 + .../index/mapper/RangeFieldTypeTests.java | 1 + .../index/mapper/RoutingFieldMapperTests.java | 1 + .../index/mapper/StringFieldScriptTests.java | 4 +- .../index/mapper/VersionFieldMapperTests.java | 7 +- .../index/query/RangeQueryRewriteTests.java | 3 + .../query/SearchExecutionContextTests.java | 6 ++ .../bucket/range/RangeAggregatorTests.java | 2 +- .../bucket/terms/TermsAggregatorTests.java | 2 +- .../support/ScriptValuesTests.java | 2 +- .../fetch/subphase/FieldFetcherTests.java | 1 + .../highlight/HighlightBuilderTests.java | 1 + .../search/lookup/LeafDocLookupTests.java | 11 ++- .../search/query/ScriptScoreQueryTests.java | 2 +- .../rescore/QueryRescorerBuilderTests.java | 2 + ...tScriptFieldDistanceFeatureQueryTests.java | 2 +- ...gScriptFieldDistanceFeatureQueryTests.java | 2 +- .../search/sort/AbstractSortTestCase.java | 9 +++ .../AbstractSuggestionBuilderTestCase.java | 2 + .../index/mapper/MapperServiceTestCase.java | 12 ++++ .../index/mapper/MapperTestCase.java | 2 + .../aggregations/AggregatorTestCase.java | 10 ++- .../test/AbstractBuilderTestCase.java | 1 + .../mapper/DataTierFieldTypeTests.java | 2 + .../DocumentSubsetBitsetCacheTests.java | 1 + ...ityIndexReaderWrapperIntegrationTests.java | 2 + .../AggregateDoubleMetricFieldTypeTests.java | 2 + .../job/RollupIndexerIndexingTests.java | 1 + .../rollup/v2/MetricFieldProducerTests.java | 1 + .../mapper/WildcardFieldMapperTests.java | 9 +++ 63 files changed, 341 insertions(+), 80 deletions(-) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index 3c76b4cae6627..861ac6fef5afb 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -25,6 +25,8 @@ import org.elasticsearch.Version; import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.mapper.MappedFieldType; @@ -88,6 +90,10 @@ public class ScriptScoreBenchmark { private SearchLookup lookup = new SearchLookup( fieldTypes::get, (mft, lookup) -> mft.fielddataBuilder("test", lookup).build(fieldDataCache, breakerService), + (mft, lookup) -> { + Tuple sfd = mft.scriptFielddataBuilder("test", lookup); + return new Tuple<>(sfd.v1(), sfd.v2().build(fieldDataCache, breakerService)); + }, sourcePaths::get ); diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java index b4712057cc3de..0a9853a5a3f4a 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java @@ -26,10 +26,12 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.IOUtils; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; @@ -143,6 +145,10 @@ protected SearchExecutionContext buildSearchExecutionContext() { null, (ft, idxName, lookup) -> ft.fielddataBuilder(idxName, lookup) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), + (ft, idxName, lookup) -> { + Tuple sfd = ft.scriptFielddataBuilder(idxName, lookup); + return new Tuple<>(sfd.v1(), sfd.v2().build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService())); + }, mapperService, mapperService.mappingLookup(), similarityService, diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java index 9b225285b029e..1385f66dd0d24 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.script.expression; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; @@ -53,6 +54,7 @@ public void setUp() throws Exception { lookup = new SearchLookup( field -> field.equals("field") ? fieldType : null, (ignored, _lookup) -> fieldData, + (ignored, _lookup) -> new Tuple<>(true, fieldData), field -> field.equals("field") ? Set.of("field") : null ); } diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java index 7624d56021e10..1c6b1fc3a26c7 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.script.expression; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; @@ -54,6 +55,7 @@ public void setUp() throws Exception { lookup = new SearchLookup( field -> field.equals("field") ? fieldType : null, (ignored, _lookup) -> fieldData, + (ignored, _lookup) -> new Tuple<>(true, fieldData), field -> field.equals("field") ? Set.of("field") : null ); } diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java index d44c5c2ae676f..29720b213cead 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.script.expression; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; @@ -53,6 +54,7 @@ public void setUp() throws Exception { lookup = new SearchLookup( field -> field.equals("field") ? fieldType : null, (ignored, _lookup) -> fieldData, + (ignored, _lookup) -> new Tuple<>(true, fieldData), field -> field.equals("field") ? Set.of("field") : null ); } diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index e41b6194da23f..7b088e49c410d 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -870,6 +870,7 @@ setup: --- "keyword_no_doc_values": - do: + catch: bad_request search: rest_total_hits_as_int: true body: @@ -878,9 +879,10 @@ setup: field: script: source: "doc['keyword_no_doc_values'].get(0)" - - match: { hits.hits.0.fields.field.0: "no doc values" } + - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } - do: + catch: bad_request search: rest_total_hits_as_int: true body: @@ -889,7 +891,7 @@ setup: field: script: source: "doc['keyword_no_doc_values'].value" - - match: { hits.hits.0.fields.field.0: "no doc values" } + - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } - do: search: diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 778d0d3928629..60973bc0f274c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -625,6 +625,7 @@ public SearchExecutionContext newSearchExecutionContext( indexSettings, indexCache.bitsetFilterCache(), indexFieldData::getForField, + indexFieldData::getForScriptField, mapperService(), mapperService().mappingLookup(), similarityService(), diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index d46c239806ef8..4747ca98f705c 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -12,6 +12,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MappedFieldType; @@ -107,24 +108,45 @@ public > IFD getForField( ) { final String fieldName = fieldType.name(); IndexFieldData.Builder builder = fieldType.fielddataBuilder(fullyQualifiedIndexName, searchLookup); + IndexFieldDataCache cache = getFieldDataCache(fieldName); + return (IFD) builder.build(cache, circuitBreakerService); + } - IndexFieldDataCache cache; - synchronized (this) { - cache = fieldDataCaches.get(fieldName); - if (cache == null) { - String cacheType = indexSettings.getValue(INDEX_FIELDDATA_CACHE_KEY); - if (FIELDDATA_CACHE_VALUE_NODE.equals(cacheType)) { - cache = indicesFieldDataCache.buildIndexFieldDataCache(listener, index(), fieldName); - } else if ("none".equals(cacheType)) { - cache = new IndexFieldDataCache.None(); - } else { - throw new IllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldName + "]"); - } - fieldDataCaches.put(fieldName, cache); + /** + * Returns fielddata for the provided script field type, given the provided fully qualified index name, + * while also making a {@link SearchLookup} supplier available that is required for runtime fields. + */ + @SuppressWarnings("unchecked") + public > Tuple getForScriptField( + MappedFieldType fieldType, + String fullyQualifiedIndexName, + Supplier searchLookup + ) { + final String fieldName = fieldType.name(); + Tuple scriptFielddataBuilder = fieldType.scriptFielddataBuilder( + fullyQualifiedIndexName, + searchLookup + ); + IndexFieldData.Builder builder = scriptFielddataBuilder.v2(); + IndexFieldDataCache cache = getFieldDataCache(fieldName); + return new Tuple<>(scriptFielddataBuilder.v1(), (IFD) builder.build(cache, circuitBreakerService)); + } + + private synchronized IndexFieldDataCache getFieldDataCache(String fieldName) { + IndexFieldDataCache cache = fieldDataCaches.get(fieldName); + if (cache == null) { + String cacheType = indexSettings.getValue(INDEX_FIELDDATA_CACHE_KEY); + if (FIELDDATA_CACHE_VALUE_NODE.equals(cacheType)) { + cache = indicesFieldDataCache.buildIndexFieldDataCache(listener, index(), fieldName); + } else if ("none".equals(cacheType)) { + cache = new IndexFieldDataCache.None(); + } else { + throw new IllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldName + "]"); } + fieldDataCaches.put(fieldName, cache); } - return (IFD) builder.build(cache, circuitBreakerService); + return cache; } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 8bc41cbaa2258..2ea3f00a091fd 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -17,8 +17,10 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; @@ -143,6 +145,13 @@ private static void executeIndexTimeScripts(DocumentParserContext context) { context.mappingLookup().indexTimeLookup()::get, (ft, lookup) -> ft.fielddataBuilder(context.indexSettings().getIndex().getName(), lookup) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), + (ft, lookup) -> { + Tuple sfd = ft.scriptFielddataBuilder( + context.indexSettings().getIndex().getName(), + lookup + ); + return new Tuple<>(sfd.v1(), sfd.v2().build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService())); + }, context.mappingLookup()::sourcePaths ); // field scripts can be called both by the loop at the end of this method and via diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 9c7cd942121de..8f23eb3d24d64 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -43,6 +43,7 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; @@ -683,19 +684,31 @@ NamedAnalyzer normalizer() { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + failIfNoDocValues(); + return new SortedSetOrdinalsIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + (dv, n) -> new KeywordDocValuesField(FieldData.toString(dv), n) + ); + } + + @Override + public Tuple scriptFielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup + ) { if (hasDocValues()) { - return new SortedSetOrdinalsIndexFieldData.Builder( - name(), - CoreValuesSourceType.KEYWORD, - (dv, n) -> new KeywordDocValuesField(FieldData.toString(dv), n) - ); - } else { - return new StringScriptFieldData.Builder( + return new Tuple<>(true, fielddataBuilder(fullyQualifiedIndexName, searchLookup)); + } + + return new Tuple<>( + false, + new StringScriptFieldData.Builder( name(), StringFieldScript.PARSE_FROM_SOURCE_PATHS.newFactory(name(), Collections.emptyMap(), searchLookup.get()), KeywordDocValuesField::new - ); - } + ) + ); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index dc94d664f2ebe..fcbd009114a74 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.query.DistanceFeatureQueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; @@ -95,6 +96,26 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S throw new IllegalArgumentException("Fielddata is not supported on field [" + name() + "] of type [" + typeName() + "]"); } + /** + * A specialized version of {@link MappedFieldType#fielddataBuilder(String, Supplier)} for scripting. Allows specific + * fields types to customize which data structure(s) values are fetched from for a field such as using + * source when doc values are not available or defaulting to source for unparsed value access when + * it makes sense to do so. + * + * @param fullyQualifiedIndexName the name of the index this field-data is build for + * @param searchLookup a {@link SearchLookup} supplier to allow for accessing other fields values in the context of runtime fields + * + * @return returns {@code true} if field data was used to generate values otherwise {@code false}, along with the + * {@link IndexFieldData.Builder} as part of a {@link Tuple}. This is required to maintain the original + * behavior of the old-style accessors for scripting like {@code doc['field'].value} + */ + public Tuple scriptFielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup + ) { + return Tuple.tuple(true, fielddataBuilder(fullyQualifiedIndexName, searchLookup)); + } + /** * Create a helper class to fetch field values during the {@link FetchFieldsPhase}. * @@ -175,7 +196,7 @@ public boolean isAggregatable() { try { fielddataBuilder("", () -> { throw new UnsupportedOperationException("SearchLookup not available"); }); return true; - } catch (UnsupportedOperationException | IllegalArgumentException e) { + } catch (IllegalArgumentException e) { return false; } } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 9e3ca706f318d..1a69715238998 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; @@ -91,7 +92,12 @@ public class SearchExecutionContext extends QueryRewriteContext { private final MappingLookup mappingLookup; private final SimilarityService similarityService; private final BitsetFilterCache bitsetFilterCache; - private final TriFunction, IndexFieldData> indexFieldDataService; + private final TriFunction, IndexFieldData> indexFieldDataLookup; + private final TriFunction< + MappedFieldType, + String, + Supplier, + Tuple>> scriptIndexFieldDataLookup; private SearchLookup lookup = null; private final int shardId; @@ -122,6 +128,7 @@ public SearchExecutionContext( IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, TriFunction, IndexFieldData> indexFieldDataLookup, + TriFunction, Tuple>> scriptIndexFieldDataLookup, MapperService mapperService, MappingLookup mappingLookup, SimilarityService similarityService, @@ -143,6 +150,7 @@ public SearchExecutionContext( indexSettings, bitsetFilterCache, indexFieldDataLookup, + scriptIndexFieldDataLookup, mapperService, mappingLookup, similarityService, @@ -170,7 +178,8 @@ public SearchExecutionContext(SearchExecutionContext source) { source.shardRequestIndex, source.indexSettings, source.bitsetFilterCache, - source.indexFieldDataService, + source.indexFieldDataLookup, + source.scriptIndexFieldDataLookup, source.mapperService, source.mappingLookup, source.similarityService, @@ -195,6 +204,7 @@ private SearchExecutionContext( IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, TriFunction, IndexFieldData> indexFieldDataLookup, + TriFunction, Tuple>> scriptIndexFieldDataLookup, MapperService mapperService, MappingLookup mappingLookup, SimilarityService similarityService, @@ -218,7 +228,8 @@ private SearchExecutionContext( this.mapperService = mapperService; this.mappingLookup = mappingLookup; this.bitsetFilterCache = bitsetFilterCache; - this.indexFieldDataService = indexFieldDataLookup; + this.indexFieldDataLookup = indexFieldDataLookup; + this.scriptIndexFieldDataLookup = scriptIndexFieldDataLookup; this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields(); this.nestedScope = new NestedScope(); this.scriptService = scriptService; @@ -279,7 +290,7 @@ public boolean allowExpensiveQueries() { @SuppressWarnings("unchecked") public > IFD getForField(MappedFieldType fieldType) { - return (IFD) indexFieldDataService.apply( + return (IFD) indexFieldDataLookup.apply( fieldType, fullyQualifiedIndex.getName(), () -> this.lookup().forkAndTrackFieldReferences(fieldType.name()) @@ -478,7 +489,8 @@ public SearchLookup lookup() { if (this.lookup == null) { this.lookup = new SearchLookup( this::getFieldType, - (fieldType, searchLookup) -> indexFieldDataService.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup), + (fieldType, searchLookup) -> indexFieldDataLookup.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup), + (fieldType, searchLookup) -> scriptIndexFieldDataLookup.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup), this::sourcePath ); } diff --git a/server/src/main/java/org/elasticsearch/script/AbstractFieldScript.java b/server/src/main/java/org/elasticsearch/script/AbstractFieldScript.java index 452f2ee4de7ad..d940141bdf198 100644 --- a/server/src/main/java/org/elasticsearch/script/AbstractFieldScript.java +++ b/server/src/main/java/org/elasticsearch/script/AbstractFieldScript.java @@ -17,7 +17,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Set; import java.util.function.Function; import static org.elasticsearch.core.TimeValue.timeValueMillis; @@ -68,7 +67,6 @@ static ScriptContext newContext(String name, Class factoryClass) { ); protected final String fieldName; - protected final Set sourcePaths; protected final SourceLookup sourceLookup; private final Map params; @@ -77,7 +75,6 @@ public AbstractFieldScript(String fieldName, Map params, SearchL this.fieldName = fieldName; Map docAsMap = docAsMap(); - this.sourcePaths = searchLookup.sourcePaths(fieldName); this.sourceLookup = (SourceLookup) docAsMap.get("_source"); params = new HashMap<>(params); params.put("_source", sourceLookup); @@ -114,14 +111,6 @@ protected final void emitFromSource() { } } - protected final void emitFromSourcePaths() { - for (String sourcePath : sourcePaths) { - for (Object v : extractFromSource(sourcePath)) { - emitFromObject(v); - } - } - } - /** * Check if the we can add another value to the list of values. * @param currentSize the current size of the list diff --git a/server/src/main/java/org/elasticsearch/script/StringFieldScript.java b/server/src/main/java/org/elasticsearch/script/StringFieldScript.java index fc27f4e6030ab..e307d787fff22 100644 --- a/server/src/main/java/org/elasticsearch/script/StringFieldScript.java +++ b/server/src/main/java/org/elasticsearch/script/StringFieldScript.java @@ -15,6 +15,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; @@ -43,6 +44,12 @@ public boolean isResultDeterministic() { } }; + /** + * PARSE_FROM_SOURCE_PATHS is required for source fallback for the scripting fields API. + * A runtime field is used to generate values directly from source when doc values are not + * available. This differs from PARSE_FROM_SOURCE by using source paths that account for multi-field + * relationships and copy-to relationships. See {@link org.elasticsearch.index.mapper.MappingLookup#sourcePaths(String)}. + */ public static final StringFieldScript.Factory PARSE_FROM_SOURCE_PATHS = new Factory() { @Override public LeafFactory newFactory(String field, Map params, SearchLookup lookup) { @@ -91,11 +98,14 @@ public interface LeafFactory { StringFieldScript newInstance(LeafReaderContext ctx); } + protected final Set sourcePaths; + private final List results = new ArrayList<>(); private long chars; public StringFieldScript(String fieldName, Map params, SearchLookup searchLookup, LeafReaderContext ctx) { super(fieldName, params, searchLookup, ctx); + this.sourcePaths = searchLookup.sourcePaths(fieldName); } /** @@ -116,6 +126,20 @@ public final void runForDoc(int docId, Consumer consumer) { resultsForDoc(docId).forEach(consumer); } + /** + * emitFromSourcePaths is used for the script fields API to handle source fallback + * when doc values are not available. This differs from {@link #emitFromSourcePaths()} + * by handling multiple source paths for possible mutli-field relationships and copy-to relationships. + * See {@link org.elasticsearch.index.mapper.MappingLookup#sourcePaths(String)}. + */ + protected void emitFromSourcePaths() { + for (String sourcePath : sourcePaths) { + for (Object v : extractFromSource(sourcePath)) { + emitFromObject(v); + } + } + } + @Override protected void emitFromObject(Object v) { if (v != null) { diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java index 8a71978db46ce..d24391d130ffe 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java @@ -10,6 +10,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.util.Maps; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.MappedFieldType; @@ -28,19 +29,22 @@ public class LeafDocLookup implements Map> { private final Function fieldTypeLookup; private final Function> fieldDataLookup; + private final Function>> scriptFieldDataLookup; private final LeafReaderContext reader; private int docId = -1; - private final Map localCacheScriptFieldData = Maps.newMapWithExpectedSize(4); + private final Map, DocValuesScriptFieldFactory> localCacheScriptFieldData = Maps.newMapWithExpectedSize(4); LeafDocLookup( Function fieldTypeLookup, Function> fieldDataLookup, + Function>> scriptFieldDataLookup, LeafReaderContext reader ) { this.fieldTypeLookup = fieldTypeLookup; this.fieldDataLookup = fieldDataLookup; + this.scriptFieldDataLookup = scriptFieldDataLookup; this.reader = reader; } @@ -48,8 +52,12 @@ public void setDocument(int docId) { this.docId = docId; } - public DocValuesScriptFieldFactory getScriptFieldFactory(String fieldName) { - DocValuesScriptFieldFactory factory = localCacheScriptFieldData.get(fieldName); + public Field getScriptField(String fieldName) { + DocValuesScriptFieldFactory factory = localCacheScriptFieldData.get(new Tuple<>(fieldName, false)); + + if (factory == null) { + factory = localCacheScriptFieldData.get(new Tuple<>(fieldName, true)); + } if (factory == null) { final MappedFieldType fieldType = fieldTypeLookup.apply(fieldName); @@ -60,14 +68,19 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String fieldName) { // Load the field data on behalf of the script. Otherwise, it would require // additional permissions to deal with pagedbytes/ramusagestimator/etc. - factory = AccessController.doPrivileged(new PrivilegedAction() { - @Override - public DocValuesScriptFieldFactory run() { - return fieldDataLookup.apply(fieldType).load(reader).getScriptFieldFactory(fieldName); + Tuple fdt = AccessController.doPrivileged( + new PrivilegedAction>() { + @Override + public Tuple run() { + Tuple> fdt = scriptFieldDataLookup.apply(fieldType); + DocValuesScriptFieldFactory dvf = fdt.v2().load(reader).getScriptFieldFactory(fieldName); + return new Tuple<>(fdt.v1(), dvf); + } } - }); + ); - localCacheScriptFieldData.put(fieldName, factory); + factory = fdt.v2(); + localCacheScriptFieldData.put(new Tuple<>(fieldName, fdt.v1()), factory); } try { @@ -76,24 +89,48 @@ public DocValuesScriptFieldFactory run() { throw ExceptionsHelper.convertToElastic(ioe); } - return factory; - } - - public Field getScriptField(String fieldName) { - return getScriptFieldFactory(fieldName).toScriptField(); + return factory.toScriptField(); } @Override public ScriptDocValues get(Object key) { String fieldName = key.toString(); - return getScriptFieldFactory(fieldName).toScriptDocValues(); + DocValuesScriptFieldFactory factory = localCacheScriptFieldData.get(new Tuple<>(fieldName, true)); + + if (factory == null) { + final MappedFieldType fieldType = fieldTypeLookup.apply(fieldName); + + if (fieldType == null) { + throw new IllegalArgumentException("No field found for [" + fieldName + "] in mapping"); + } + + // Load the field data on behalf of the script. Otherwise, it would require + // additional permissions to deal with pagedbytes/ramusagestimator/etc. + factory = AccessController.doPrivileged(new PrivilegedAction() { + @Override + public DocValuesScriptFieldFactory run() { + return fieldDataLookup.apply(fieldType).load(reader).getScriptFieldFactory(fieldName); + } + }); + + localCacheScriptFieldData.put(new Tuple<>(fieldName, true), factory); + } + + try { + factory.setNextDocId(docId); + } catch (IOException ioe) { + throw ExceptionsHelper.convertToElastic(ioe); + } + + return factory.toScriptDocValues(); } @Override public boolean containsKey(Object key) { String fieldName = key.toString(); - DocValuesScriptFieldFactory docValuesFieldFactory = localCacheScriptFieldData.get(fieldName); - return docValuesFieldFactory != null || fieldTypeLookup.apply(fieldName) != null; + return localCacheScriptFieldData.get(new Tuple<>(fieldName, true)) != null + || localCacheScriptFieldData.get(new Tuple<>(fieldName, false)) != null + || fieldTypeLookup.apply(fieldName) != null; } @Override diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index e30851cc171b8..989667965dd0f 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.lookup; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; @@ -42,6 +43,7 @@ public class SearchLookup { private final SourceLookup sourceLookup; private final Function fieldTypeLookup; private final BiFunction, IndexFieldData> fieldDataLookup; + private final BiFunction, Tuple>> scriptFieldDataLookup; private final Function> sourcePathsLookup; /** @@ -51,12 +53,14 @@ public class SearchLookup { public SearchLookup( Function fieldTypeLookup, BiFunction, IndexFieldData> fieldDataLookup, + BiFunction, Tuple>> scriptFieldDataLookup, Function> sourcePathsLookup ) { this.fieldTypeLookup = fieldTypeLookup; this.fieldChain = Collections.emptySet(); this.sourceLookup = new SourceLookup(); this.fieldDataLookup = fieldDataLookup; + this.scriptFieldDataLookup = scriptFieldDataLookup; this.sourcePathsLookup = sourcePathsLookup; } @@ -72,6 +76,7 @@ private SearchLookup(SearchLookup searchLookup, Set fieldChain) { this.sourceLookup = searchLookup.sourceLookup; this.fieldTypeLookup = searchLookup.fieldTypeLookup; this.fieldDataLookup = searchLookup.fieldDataLookup; + this.scriptFieldDataLookup = searchLookup.scriptFieldDataLookup; this.sourcePathsLookup = searchLookup.sourcePathsLookup; } @@ -99,7 +104,7 @@ public final SearchLookup forkAndTrackFieldReferences(String field) { public LeafSearchLookup getLeafSearchLookup(LeafReaderContext context) { return new LeafSearchLookup( context, - new LeafDocLookup(fieldTypeLookup, this::getForField, context), + new LeafDocLookup(fieldTypeLookup, this::getForField, this::getForScriptField, context), sourceLookup, new LeafStoredFieldsLookup(fieldTypeLookup, (doc, visitor) -> context.reader().document(doc, visitor)) ); @@ -113,6 +118,10 @@ public IndexFieldData getForField(MappedFieldType fieldType) { return fieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name())); } + public Tuple> getForScriptField(MappedFieldType fieldType) { + return scriptFieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name())); + } + public Set sourcePaths(String fieldName) { return sourcePathsLookup.apply(fieldName); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index cd075234c8e01..d9fc8778953f0 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -128,6 +128,7 @@ public void setupCreateIndexRequestAndAliasValidator() { null, null, null, + null, parserConfig(), writableRegistry(), null, diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index d1a8c2abf992b..084246a8bd0dc 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -135,7 +135,7 @@ public void testGetForFieldRuntimeField() { searchLookupSetOnce.set(searchLookup); return (IndexFieldData.Builder) (cache, breakerService) -> null; }); - SearchLookup searchLookup = new SearchLookup(null, null, null); + SearchLookup searchLookup = new SearchLookup(null, null, null, null); ifdService.getForField(ft, "qualified", () -> searchLookup); assertSame(searchLookup, searchLookupSetOnce.get().get()); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index 24d4ebf31fe85..71a4f0261881a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.script.BooleanFieldScript; import org.elasticsearch.script.DateFieldScript; @@ -203,6 +204,7 @@ protected static SearchExecutionContext mockContext(boolean allowExpensiveQuerie SearchLookup lookup = new SearchLookup( context::getFieldType, (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null), + (mft, lookupSupplier) -> new Tuple<>(true, mft.fielddataBuilder("test", lookupSupplier).build(null, null)), context::sourcePath ); when(context.lookup()).thenReturn(lookup); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java index e875f3718a24e..8b7359be47682 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java @@ -57,7 +57,7 @@ public void testTooManyValues() throws IOException { BooleanFieldScript script = new BooleanFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java index 78e3a9921db0e..40ab45c3ea3ce 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Tuple; import org.elasticsearch.script.CompositeFieldScript; import org.elasticsearch.script.LongFieldScript; import org.elasticsearch.script.Script; @@ -334,6 +335,7 @@ public void testParseDocumentSubFieldAccess() throws IOException { SearchLookup searchLookup = new SearchLookup( mapperService::fieldType, (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null), + (mft, lookupSupplier) -> new Tuple<>(true, mft.fielddataBuilder("test", lookupSupplier).build(null, null)), mapperService.mappingLookup()::sourcePaths ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java index 8a2d9572e94f7..b175bdf6b9b57 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java @@ -61,7 +61,7 @@ public void testTooManyValues() throws IOException { DateFieldScript script = new DateFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), DateFormatter.forPattern(randomDateFormatterPattern()).withLocale(randomLocale(random())), reader.leaves().get(0) ) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 28606d9e99840..6c5c1ab4e7634 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -183,6 +183,7 @@ public void testTermQuery() { null, null, null, + null, parserConfig(), writableRegistry(), null, @@ -238,6 +239,7 @@ public void testRangeQuery() throws IOException { null, null, null, + null, parserConfig(), writableRegistry(), null, @@ -316,6 +318,7 @@ public void testRangeQueryWithIndexSort() { null, null, null, + null, parserConfig(), writableRegistry(), null, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java index 6e7766298c558..99dc512243adf 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java @@ -59,7 +59,7 @@ public void testTooManyValues() throws IOException { DoubleFieldScript script = new DoubleFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java index df8e09684f4fc..e096851b3be44 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java @@ -43,6 +43,7 @@ public void testTermQuery() { null, null, null, + null, mappingLookup, null, null, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java index 8ba165ca6e025..790d1a70c86c4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java @@ -59,7 +59,7 @@ public void testTooManyValues() throws IOException { GeoPointFieldScript script = new GeoPointFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java index 49c663db35d2e..f783b78f8f35f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java @@ -53,6 +53,7 @@ public void testFetchIgnoredFieldValue() throws IOException { SearchLookup lookup = new SearchLookup( mapperService::fieldType, fieldDataLookup(), + scriptFieldDataLookup(), mapperService.mappingLookup()::sourcePaths ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java index 4efdf9e57aca9..153072edef7cd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java @@ -47,7 +47,12 @@ public void testFetchFieldValue() throws IOException { iw.addDocument(mapperService.documentMapper().parse(source).rootDoc()); }, iw -> { IndexFieldMapper.IndexFieldType ft = (IndexFieldMapper.IndexFieldType) mapperService.fieldType("_index"); - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(), mapperService.mappingLookup()::sourcePaths); + SearchLookup lookup = new SearchLookup( + mapperService::fieldType, + fieldDataLookup(), + scriptFieldDataLookup(), + mapperService.mappingLookup()::sourcePaths + ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); IndexSearcher searcher = newSearcher(iw); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java index dd90380e02af8..0dcd652d2cc0b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java @@ -70,6 +70,7 @@ private SearchExecutionContext createContext() { null, null, null, + null, parserConfig(), writableRegistry(), null, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java index 0db9fac21a20c..b9a075be95c69 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java @@ -59,7 +59,7 @@ public void testTooManyValues() throws IOException { IpFieldScript script = new IpFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java index 8d3730b571780..df16172ef9bd7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java @@ -59,7 +59,7 @@ public void testTooManyValues() throws IOException { LongFieldScript script = new LongFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index eb5dfcfeb8ac8..c138da8a62201 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -614,6 +614,7 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS null, null, null, + null, parserConfig(), writableRegistry(), null, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java index 955c48b342bb2..a1d025637f284 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java @@ -61,7 +61,12 @@ public void testFetchValue() throws Exception { .rootDoc() ); }, iw -> { - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(), mapperService.mappingLookup()::sourcePaths); + SearchLookup lookup = new SearchLookup( + mapperService::fieldType, + fieldDataLookup(), + scriptFieldDataLookup(), + mapperService.mappingLookup()::sourcePaths + ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); FieldFetcher fieldFetcher = FieldFetcher.create( searchExecutionContext, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java index 52a0703244fc1..ccb6f8eb69e07 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java @@ -77,6 +77,7 @@ public void testFetchIdFieldValue() throws IOException { SearchLookup lookup = new SearchLookup( mapperService::fieldType, fieldDataLookup(), + scriptFieldDataLookup(), mapperService.mappingLookup()::sourcePaths ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index 0f79f2e01c1b6..9636cc8d5a944 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -212,6 +212,7 @@ private SearchExecutionContext createContext() { null, null, null, + null, parserConfig(), writableRegistry(), null, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java index bd644731baf9e..228ccddde051f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java @@ -71,6 +71,7 @@ public void testFetchRoutingFieldValue() throws IOException { SearchLookup lookup = new SearchLookup( mapperService::fieldType, fieldDataLookup(), + scriptFieldDataLookup(), mapperService.mappingLookup()::sourcePaths ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java index 03314b3735bd9..feb43db1399b8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java @@ -59,7 +59,7 @@ public void testTooManyValues() throws IOException { StringFieldScript script = new StringFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override @@ -85,7 +85,7 @@ public void testTooManyChars() throws IOException { StringFieldScript script = new StringFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java index ab6895f3f26fd..15295b0ac4f57 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java @@ -50,7 +50,12 @@ public void testFetchFieldValue() throws IOException { iw.addDocument(parsedDoc.rootDoc()); }, iw -> { VersionFieldMapper.VersionFieldType ft = (VersionFieldMapper.VersionFieldType) mapperService.fieldType("_version"); - SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(), mapperService.mappingLookup()::sourcePaths); + SearchLookup lookup = new SearchLookup( + mapperService::fieldType, + fieldDataLookup(), + scriptFieldDataLookup(), + mapperService.mappingLookup()::sourcePaths + ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); IndexSearcher searcher = newSearcher(iw); diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java index 6e40b0c3316ee..721568c62ceae 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java @@ -35,6 +35,7 @@ public void testRewriteMissingField() throws Exception { indexService.getIndexSettings(), null, null, + null, indexService.mapperService(), indexService.mapperService().mappingLookup(), null, @@ -75,6 +76,7 @@ public void testRewriteMissingReader() throws Exception { indexService.getIndexSettings(), null, null, + null, indexService.mapperService(), indexService.mapperService().mappingLookup(), null, @@ -117,6 +119,7 @@ public void testRewriteEmptyReader() throws Exception { indexService.getIndexSettings(), null, null, + null, indexService.mapperService(), indexService.mapperService().mappingLookup(), null, diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 4f87376311452..82ffea0faefd6 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -184,6 +185,7 @@ public void testIndexSortedOnField() { null, null, null, + null, XContentParserConfiguration.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), null, @@ -427,6 +429,10 @@ private static SearchExecutionContext createSearchExecutionContext( indexSettings, null, (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), + (mappedFieldType, idxName, searchLookup) -> { + Tuple sfd = mappedFieldType.scriptFielddataBuilder(idxName, searchLookup); + return new Tuple<>(sfd.v1(), sfd.v2().build(null, null)); + }, mapperService, mappingLookup, null, diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java index 3f68060e7b733..fa9a8633b2394 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java @@ -635,7 +635,7 @@ public void testOverlappingRanges() throws IOException { */ public void testRuntimeFieldTopLevelQueryNotOptimized() throws IOException { long totalDocs = (long) RangeAggregator.DOCS_PER_RANGE_TO_USE_FILTERS * 4; - SearchLookup lookup = new SearchLookup(s -> null, (ft, l) -> null, s -> null); + SearchLookup lookup = new SearchLookup(s -> null, (ft, l) -> null, (ft, l) -> null, s -> null); StringFieldScript.LeafFactory scriptFactory = ctx -> new StringFieldScript("dummy", Map.of(), lookup, ctx) { @Override public void execute() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index 5fbd0fcb20be2..ba53789a8a3ce 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -2043,7 +2043,7 @@ public void testWithFilterAndPreciseSize() throws IOException { */ public void testRuntimeFieldTopLevelNotOptimized() throws IOException { long totalDocs = 500; - SearchLookup lookup = new SearchLookup(s -> null, (ft, l) -> null, s -> null); + SearchLookup lookup = new SearchLookup(s -> null, (ft, l) -> null, (ft, l) -> null, s -> null); StringFieldScript.LeafFactory scriptFactory = ctx -> new StringFieldScript("dummy", Map.of(), lookup, ctx) { @Override public void execute() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java index 6e90fbdd54790..cd3dc6816fa10 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java @@ -36,7 +36,7 @@ private static class FakeAggregationScript extends AggregationScript { int index; FakeAggregationScript(Object[][] values) { - super(Collections.emptyMap(), new SearchLookup(null, null, null) { + super(Collections.emptyMap(), new SearchLookup(null, null, null, null) { @Override public LeafSearchLookup getLeafSearchLookup(LeafReaderContext context) { diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 6a596bf5af490..3c30a905d3f45 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -1255,6 +1255,7 @@ private static SearchExecutionContext newSearchExecutionContext( indexSettings, null, indexFieldDataLookup, + null, mapperService, mapperService.mappingLookup(), null, diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index 42f95718c522a..370272cb687cf 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -303,6 +303,7 @@ public void testBuildSearchContextHighlight() throws IOException { null, null, null, + null, parserConfig(), namedWriteableRegistry, null, diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java index 9367f4646b5cc..48dcfb9e51b76 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java @@ -7,6 +7,7 @@ */ package org.elasticsearch.search.lookup; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; @@ -51,6 +52,9 @@ public void setUp() throws Exception { docLookup = new LeafDocLookup( field -> field.equals("field") ? fieldType1 : field.equals("alias") ? fieldType2 : null, fieldType -> fieldType == fieldType1 ? fieldData1 : fieldType == fieldType2 ? fieldData2 : null, + fieldType -> fieldType == fieldType1 ? new Tuple<>(true, fieldData1) + : fieldType == fieldType2 ? new Tuple<>(true, fieldData2) + : null, null ); } @@ -82,6 +86,11 @@ public void testFlattenedField() throws IOException { return keyedFieldType.key().equals("key1") ? fieldData1 : fieldData2; }; + Function>> scriptFieldDataSupplier = ft -> { + FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) ft; + return keyedFieldType.key().equals("key1") ? new Tuple<>(true, fieldData1) : new Tuple<>(true, fieldData2); + }; + LeafDocLookup docLookup = new LeafDocLookup(field -> { if (field.equals("flattened.key1")) { return fieldType1; @@ -90,7 +99,7 @@ public void testFlattenedField() throws IOException { return fieldType2; } return null; - }, fieldDataSupplier, null); + }, fieldDataSupplier, scriptFieldDataSupplier, null); assertEquals(docValues1, docLookup.get("flattened.key1")); assertEquals(docValues2, docLookup.get("flattened.key2")); diff --git a/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java b/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java index 74b2000024e8b..c99a770053b73 100644 --- a/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java @@ -49,7 +49,7 @@ public class ScriptScoreQueryTests extends ESTestCase { private DirectoryReader reader; private IndexSearcher searcher; private LeafReaderContext leafReaderContext; - private final SearchLookup lookup = new SearchLookup(null, null, null); + private final SearchLookup lookup = new SearchLookup(null, null, null, null); @Before public void initSearcher() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java index 7685119e7b660..d046d6b60d6c1 100644 --- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java @@ -141,6 +141,7 @@ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, null, null, null, + null, parserConfig(), namedWriteableRegistry, null, @@ -203,6 +204,7 @@ public void testRewritingKeepsSettings() throws IOException { null, null, null, + null, parserConfig(), namedWriteableRegistry, null, diff --git a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java index b1e8433c3b144..f39f9e5eb1a74 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java @@ -82,7 +82,7 @@ public void testMatches() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"location\": [-3.56, -45.98]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - SearchLookup searchLookup = new SearchLookup(null, null, f -> "location".equals(f) ? Set.of("location") : null); + SearchLookup searchLookup = new SearchLookup(null, null, null, f -> "location".equals(f) ? Set.of("location") : null); Function leafFactory = ctx -> new GeoPointFieldScript( "test", Map.of(), diff --git a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java index c06aab3816261..e48e9d9e212a1 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java @@ -69,7 +69,7 @@ public void testMatches() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181351]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - SearchLookup searchLookup = new SearchLookup(null, null, f -> "timestamp".equals(f) ? Set.of("timestamp") : null); + SearchLookup searchLookup = new SearchLookup(null, null, null, f -> "timestamp".equals(f) ? Set.of("timestamp") : null); Function leafFactory = ctx -> new DateFieldScript( "test", Map.of(), diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 9c16d5fc275b3..0d030d94fd51e 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; @@ -196,6 +197,13 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null); }; + TriFunction, Tuple>> scriptIndexFieldDataLookup = ( + fieldType, + fieldIndexName, + searchLookup) -> { + Tuple sfd = fieldType.scriptFielddataBuilder(fieldIndexName, searchLookup); + return new Tuple<>(sfd.v1(), sfd.v2().build(new IndexFieldDataCache.None(), null)); + }; NestedLookup nestedLookup = NestedLookup.build( List.of(new NestedObjectMapper.Builder("path", Version.CURRENT).build(MapperBuilderContext.ROOT)) ); @@ -205,6 +213,7 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea idxSettings, bitsetFilterCache, indexFieldDataLookup, + scriptIndexFieldDataLookup, null, null, null, diff --git a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java index 35b1fa8001d7b..24d8a4cec82be 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -179,6 +179,7 @@ public NamedAnalyzer get(Object key) { idxSettings, null, null, + null, mapperService, lookup, null, @@ -235,6 +236,7 @@ public void testBuildWithUnmappedField() { idxSettings, null, null, + null, mock(MapperService.class), MappingLookup.EMPTY, null, diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index d79a82232cafb..b4cf9725e6c28 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -31,6 +31,7 @@ import org.elasticsearch.common.util.MockPageCacheRecycler; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -626,6 +627,10 @@ public void onRemoval(ShardId shardId, Accountable accountable) { }), (ft, idxName, lookup) -> ft.fielddataBuilder(idxName, lookup) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), + (ft, idxName, lookup) -> { + Tuple sfd = ft.scriptFielddataBuilder(idxName, lookup); + return new Tuple<>(sfd.v1(), sfd.v2().build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService())); + }, mapperService, mapperService.mappingLookup(), similarityService, @@ -648,6 +653,13 @@ protected BiFunction, IndexFieldData> .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } + protected BiFunction, Tuple>> scriptFieldDataLookup() { + return (mft, lookupSource) -> new Tuple<>( + true, + mft.fielddataBuilder("test", lookupSource).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) + ); + } + protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer build) throws IOException { try (Directory directory = newDirectory()) { RandomIndexWriter iw = new RandomIndexWriter(random(), directory); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index baa22affde3fb..ae06df817a858 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -343,6 +343,7 @@ protected final List fetchFromDocValues(MapperService mapperService, MappedFi SearchLookup lookup = new SearchLookup( mapperService::fieldType, fieldDataLookup(), + scriptFieldDataLookup(), mapperService.mappingLookup()::sourcePaths ); ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft)); @@ -722,6 +723,7 @@ public final void testIndexTimeStoredFieldsAccess() throws IOException { SearchLookup lookup = new SearchLookup( f -> fieldType, (f, s) -> { throw new UnsupportedOperationException(); }, + (f, s) -> { throw new UnsupportedOperationException(); }, f -> f.equals("field") ? Set.of("field") : null ); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 5fd1014accf97..a671e0f07924e 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -60,6 +60,7 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; @@ -293,12 +294,18 @@ protected AggregationContext createAggregationContext( .map(ft -> new FieldAliasMapper(ft.name() + "-alias", ft.name() + "-alias", ft.name())) .collect(toList()) ); - TriFunction, IndexFieldData> fieldDataBuilder = ( fieldType, s, searchLookup) -> fieldType.fielddataBuilder(indexSettings.getIndex().getName(), searchLookup) .build(new IndexFieldDataCache.None(), breakerService); + TriFunction, Tuple>> scriptFieldDataBuilder = ( + fieldType, + s, + searchLookup) -> { + Tuple sfd = fieldType.scriptFielddataBuilder(indexSettings.getIndex().getName(), searchLookup); + return new Tuple<>(sfd.v1(), sfd.v2().build(new IndexFieldDataCache.None(), breakerService)); + }; BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() { @Override public void onRemoval(ShardId shardId, Accountable accountable) {} @@ -312,6 +319,7 @@ public void onCache(ShardId shardId, Accountable accountable) {} indexSettings, bitsetFilterCache, fieldDataBuilder, + scriptFieldDataBuilder, null, mappingLookup, null, diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index 660554efc6d2c..523d62c6a7721 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -500,6 +500,7 @@ SearchExecutionContext createShardContext(IndexSearcher searcher) { idxSettings, bitsetFilterCache, indexFieldDataService::getForField, + indexFieldDataService::getForScriptField, mapperService, mapperService.mappingLookup(), similarityService, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java index 9d712d2381384..3f87cdceef1bd 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java @@ -122,6 +122,7 @@ private SearchExecutionContext createContext() { null, null, null, + null, parserConfig(), writableRegistry(), null, @@ -152,6 +153,7 @@ private SearchExecutionContext createContextWithoutSetting() { null, null, null, + null, parserConfig(), writableRegistry(), null, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java index c576c66948373..b6d0168825477 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java @@ -595,6 +595,7 @@ private TestIndexContext testIndex(MappingLookup mappingLookup, Client client) t null, null, null, + null, mappingLookup, null, null, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java index dc5e7475e1c7d..8cf3769c2534b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java @@ -92,6 +92,7 @@ public void testDLS() throws Exception { null, null, null, + null, mappingLookup, null, null, @@ -252,6 +253,7 @@ public void testDLSWithLimitedPermissions() throws Exception { null, null, null, + null, mappingLookup, null, null, diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java index bad5ae6ea62b3..6461423a6f6c5 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java @@ -17,6 +17,7 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.elasticsearch.Version; import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; @@ -122,6 +123,7 @@ public void testUsedInScript() throws IOException { SearchLookup lookup = new SearchLookup( searchExecutionContext::getFieldType, (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null), + (mft, lookupSupplier) -> new Tuple<>(true, mft.fielddataBuilder("test", lookupSupplier).build(null, null)), searchExecutionContext::sourcePath ); when(searchExecutionContext.lookup()).thenReturn(lookup); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index 5c44d181284d0..2156ff48f7e29 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -105,6 +105,7 @@ private void setup() { null, null, null, + null, () -> 0L, null, null, diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/MetricFieldProducerTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/MetricFieldProducerTests.java index e0f97315b7082..f0bbd7f3a289f 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/MetricFieldProducerTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/MetricFieldProducerTests.java @@ -198,6 +198,7 @@ public void testBuildMetricProducers() { null, null, null, + null, () -> 0L, null, null, diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index def4b9fb7abd0..185d1859721d4 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -48,6 +48,7 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; @@ -1082,12 +1083,20 @@ protected final SearchExecutionContext createMockContext() { IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null); }; + TriFunction, Tuple>> scriptIndexFieldDataLookup = ( + fieldType, + fieldIndexName, + searchLookup) -> { + Tuple sfd = fieldType.scriptFielddataBuilder(fieldIndexName, searchLookup); + return new Tuple<>(sfd.v1(), sfd.v2().build(new IndexFieldDataCache.None(), null)); + }; return new SearchExecutionContext( 0, 0, idxSettings, bitsetFilterCache, indexFieldDataLookup, + scriptIndexFieldDataLookup, null, null, null, From 30fc69b6566858f2e9506a29b1607b374ec5c3ba Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 27 Jun 2022 07:06:25 -0700 Subject: [PATCH 05/23] Update docs/changelog/88040.yaml --- docs/changelog/88040.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/88040.yaml diff --git a/docs/changelog/88040.yaml b/docs/changelog/88040.yaml new file mode 100644 index 0000000000000..ac96b7696689c --- /dev/null +++ b/docs/changelog/88040.yaml @@ -0,0 +1,5 @@ +pr: 88040 +summary: Follow Up for Add source fallback for keyword fields +area: "Mapping, Infra/Scripting" +type: enhancement +issues: [] From 27f7d80fb13fc3c1c64f645c1df07e12605c07c0 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 27 Jun 2022 08:03:52 -0700 Subject: [PATCH 06/23] update change log --- docs/changelog/88040.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/88040.yaml b/docs/changelog/88040.yaml index ac96b7696689c..53541b346a95a 100644 --- a/docs/changelog/88040.yaml +++ b/docs/changelog/88040.yaml @@ -1,5 +1,5 @@ pr: 88040 summary: Follow Up for Add source fallback for keyword fields -area: "Mapping, Infra/Scripting" +area: "Mapping" type: enhancement issues: [] From 60d4d0e9fa04f4c158e80d15722451aed0def1a1 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Thu, 7 Jul 2022 13:55:07 -0700 Subject: [PATCH 07/23] Delete docs/changelog/88040.yaml --- docs/changelog/88040.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/88040.yaml diff --git a/docs/changelog/88040.yaml b/docs/changelog/88040.yaml deleted file mode 100644 index 53541b346a95a..0000000000000 --- a/docs/changelog/88040.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 88040 -summary: Follow Up for Add source fallback for keyword fields -area: "Mapping" -type: enhancement -issues: [] From 39cde2211b25a81697fd848f59eee7e1b3401a3b Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 11 Jul 2022 12:22:08 -0700 Subject: [PATCH 08/23] fix tests with master merge --- .../elasticsearch/index/mapper/DateFieldScriptTests.java | 2 +- .../elasticsearch/index/mapper/DoubleFieldScriptTests.java | 2 +- .../org/elasticsearch/index/mapper/IpFieldScriptTests.java | 2 +- .../elasticsearch/index/mapper/LongFieldScriptTests.java | 2 +- .../elasticsearch/index/mapper/StringFieldScriptTests.java | 4 ++-- .../org/elasticsearch/script/CompositeFieldScriptTests.java | 6 +++--- .../field/SortedNumericDocValuesLongFieldScriptTests.java | 2 +- .../field/SortedSetDocValuesStringFieldScriptTests.java | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java index ed01ee400daa2..088fca9ee76f0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java @@ -102,7 +102,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { DateFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), DateFormatter.forPattern("epoch_millis") ); DateFieldScript dateFieldScript = leafFactory.newInstance(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java index 7d79ab3707610..d37a1c635d5b9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java @@ -99,7 +99,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { DoubleFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null) + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null) ); DoubleFieldScript doubleFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java index 900dbe67f9a76..c8d1d9daadbcb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java @@ -100,7 +100,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { IpFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null) + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null) ); IpFieldScript ipFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java index 72fc6ed44f32e..c1062fff2ace7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java @@ -99,7 +99,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { LongFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null) + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null) ); LongFieldScript longFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java index 97065ae546f34..d0a989de5649b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java @@ -129,7 +129,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { StringFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null) + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null) ); StringFieldScript stringFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = stringFieldScript.resultsForDoc(0); @@ -158,7 +158,7 @@ public final void testFromSourceDoesNotEnforceCharsLimit() throws IOException { StringFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null) + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null) ); StringFieldScript stringFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = stringFieldScript.resultsForDoc(0); diff --git a/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java index 7c477cbcbe27a..696dcca5855fb 100644 --- a/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java @@ -31,7 +31,7 @@ public void testTooManyValues() throws IOException { CompositeFieldScript script = new CompositeFieldScript( "composite", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override @@ -62,7 +62,7 @@ public void testTooManyChars() throws IOException { CompositeFieldScript script = new CompositeFieldScript( "composite", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override @@ -75,7 +75,7 @@ public void execute() { StringFieldScript stringFieldScript = new StringFieldScript( "composite.leaf", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java index 0cc14dcfa23c4..24488df1045da 100644 --- a/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java @@ -38,7 +38,7 @@ public void testValuesLimitIsNotEnforced() throws IOException { try (DirectoryReader reader = iw.getReader()) { SortedNumericDocValuesLongFieldScript docValues = new SortedNumericDocValuesLongFieldScript( "test", - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ); List values = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java index 09b86887c54dc..cf5087995d57e 100644 --- a/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java @@ -39,7 +39,7 @@ public void testValuesLimitIsNotEnforced() throws IOException { try (DirectoryReader reader = iw.getReader()) { SortedSetDocValuesStringFieldScript docValues = new SortedSetDocValuesStringFieldScript( "test", - new SearchLookup(field -> null, (ft, lookup) -> null), + new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), reader.leaves().get(0) ); List values = new ArrayList<>(); From 0693ac0bb56ef0b08eee5b5234a077420cb25be2 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 11 Jul 2022 15:25:52 -0700 Subject: [PATCH 09/23] add keyword source fallback for scripting using value fetcher --- .../test/painless/50_script_doc_values.yml | 2 +- .../KeywordValueFetcherIndexFieldData.java | 206 ++++++++++++++++++ .../index/mapper/KeywordFieldMapper.java | 17 +- .../index/mapper/SourceValueFetcher.java | 2 +- .../script/StringFieldScript.java | 41 ---- 5 files changed, 222 insertions(+), 46 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/KeywordValueFetcherIndexFieldData.java diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index 7b088e49c410d..8573b7d3e85ae 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -90,7 +90,7 @@ setup: nanos: [2015-01-01T12:10:30.123456789Z, 2015-01-01T12:10:30.987654321Z] geo_point: [[-71.34,41.12],[60.32,21.25]] keyword: ["one string", "another string"] - keyword_no_doc_values: ["no doc values 0", "no doc values 1", "no doc values 2"] + keyword_no_doc_values: ["no doc values 1", "no doc values 0", "no doc values 2"] long: [1152921504606846976, 576460752303423488] integer: [5, 17, 29] short: [6, 18, 30, 45] diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/KeywordValueFetcherIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/KeywordValueFetcherIndexFieldData.java new file mode 100644 index 0000000000000..39894a90e8141 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/KeywordValueFetcherIndexFieldData.java @@ -0,0 +1,206 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.SortField; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.ToScriptFieldFactory; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.SortedSet; +import java.util.TreeSet; + +public class KeywordValueFetcherIndexFieldData + implements + IndexFieldData { + + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final ValueFetcher valueFetcher; + private final SourceLookup sourceLookup; + private final ToScriptFieldFactory toScriptFieldFactory; + + public Builder( + String name, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + this.name = name; + this.valueFetcher = valueFetcher; + this.sourceLookup = sourceLookup; + this.toScriptFieldFactory = toScriptFieldFactory; + } + + @Override + public KeywordValueFetcherIndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { + return new KeywordValueFetcherIndexFieldData(name, valueFetcher, sourceLookup, toScriptFieldFactory); + } + } + + private final String fieldName; + private final ValueFetcher valueFetcher; + private final SourceLookup sourceLookup; + private final ToScriptFieldFactory toScriptFieldFactory; + + protected KeywordValueFetcherIndexFieldData( + String fieldName, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + this.fieldName = fieldName; + this.valueFetcher = valueFetcher; + this.sourceLookup = sourceLookup; + this.toScriptFieldFactory = toScriptFieldFactory; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.KEYWORD; + } + + @Override + public KeywordValueFetcherLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public KeywordValueFetcherLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + return new KeywordValueFetcherLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup); + + } + + @Override + public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { + final XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); + return new SortField(getFieldName(), source, reverse); + } + + @Override + public BucketedSort newBucketedSort( + BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra + ) { + throw new IllegalArgumentException("only supported on numeric fields"); + } + + public static class KeywordValueFetcherLeafFieldData implements LeafFieldData { + + private final ToScriptFieldFactory toScriptFieldFactory; + private final LeafReaderContext leafReaderContext; + + private final ValueFetcher valueFetcher; + private final SourceLookup sourceLookup; + + public KeywordValueFetcherLeafFieldData( + ToScriptFieldFactory toScriptFieldFactory, + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + this.toScriptFieldFactory = toScriptFieldFactory; + this.leafReaderContext = leafReaderContext; + this.valueFetcher = valueFetcher; + this.sourceLookup = sourceLookup; + } + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public void close() { + + } + + @Override + public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { + return toScriptFieldFactory.getScriptFieldFactory(getBytesValues(), name); + } + + @Override + public SortedBinaryDocValues getBytesValues() { + return new KeywordValueFetcherBinaryDocValues(leafReaderContext, valueFetcher, sourceLookup); + } + } + + public static class KeywordValueFetcherBinaryDocValues extends SortedBinaryDocValues { + + private final LeafReaderContext leafReaderContext; + + private final ValueFetcher valueFetcher; + private final SourceLookup sourceLookup; + + private SortedSet values; + private Iterator iterator; + + public KeywordValueFetcherBinaryDocValues( + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + this.leafReaderContext = leafReaderContext; + this.valueFetcher = valueFetcher; + this.sourceLookup = sourceLookup; + } + + @Override + public boolean advanceExact(int doc) throws IOException { + sourceLookup.setSegmentAndDocument(leafReaderContext, doc); + values = new TreeSet<>(valueFetcher.fetchValues(sourceLookup, Collections.emptyList())); + iterator = values.iterator(); + + return true; + } + + @Override + public int docValueCount() { + return values.size(); + } + + @Override + public BytesRef nextValue() throws IOException { + assert iterator.hasNext(); + return new BytesRef(iterator.next().toString()); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 9ed506356ebc8..79eafd955b6db 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -48,7 +48,7 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.StringScriptFieldData; +import org.elasticsearch.index.fielddata.KeywordValueFetcherIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; @@ -703,9 +703,20 @@ public Tuple scriptFielddataBuilder( return new Tuple<>( false, - new StringScriptFieldData.Builder( + new KeywordValueFetcherIndexFieldData.Builder( name(), - StringFieldScript.PARSE_FROM_SOURCE_PATHS.newFactory(name(), Collections.emptyMap(), searchLookup.get()), + new SourceValueFetcher(searchLookup.get().sourcePaths(name()), nullValue) { + @Override + protected String parseSourceValue(Object value) { + String keywordValue = value.toString(); + if (keywordValue.length() > ignoreAbove) { + return null; + } + + return normalizeValue(normalizer(), name(), keywordValue); + } + }, + searchLookup.get().source(), KeywordDocValuesField::new ) ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java index f10eb1bbd1ed4..be2df55b6b7a8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SourceValueFetcher.java @@ -46,7 +46,7 @@ public SourceValueFetcher(String fieldName, SearchExecutionContext context, Obje * @param sourcePaths The paths to pull source values from * @param nullValue An optional substitute value if the _source value is `null` */ - private SourceValueFetcher(Set sourcePaths, Object nullValue) { + public SourceValueFetcher(Set sourcePaths, Object nullValue) { this.sourcePaths = sourcePaths; this.nullValue = nullValue; } diff --git a/server/src/main/java/org/elasticsearch/script/StringFieldScript.java b/server/src/main/java/org/elasticsearch/script/StringFieldScript.java index 1adcef4f65678..7e366f4b72b18 100644 --- a/server/src/main/java/org/elasticsearch/script/StringFieldScript.java +++ b/server/src/main/java/org/elasticsearch/script/StringFieldScript.java @@ -15,7 +15,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; @@ -44,29 +43,6 @@ public boolean isResultDeterministic() { } }; - /** - * PARSE_FROM_SOURCE_PATHS is required for source fallback for the scripting fields API. - * A runtime field is used to generate values directly from source when doc values are not - * available. This differs from PARSE_FROM_SOURCE by using source paths that account for multi-field - * relationships and copy-to relationships. See {@link org.elasticsearch.index.mapper.MappingLookup#sourcePaths(String)}. - */ - public static final StringFieldScript.Factory PARSE_FROM_SOURCE_PATHS = new Factory() { - @Override - public LeafFactory newFactory(String field, Map params, SearchLookup lookup) { - return ctx -> new StringFieldScript(field, params, lookup, ctx) { - @Override - public void execute() { - emitFromSourcePaths(); - } - }; - } - - @Override - public boolean isResultDeterministic() { - return true; - } - }; - public static Factory leafAdapter(Function parentFactory) { return (leafFieldName, params, searchLookup) -> { CompositeFieldScript.LeafFactory parentLeafFactory = parentFactory.apply(searchLookup); @@ -98,14 +74,11 @@ public interface LeafFactory { StringFieldScript newInstance(LeafReaderContext ctx); } - protected final Set sourcePaths; - private final List results = new ArrayList<>(); private long chars; public StringFieldScript(String fieldName, Map params, SearchLookup searchLookup, LeafReaderContext ctx) { super(fieldName, params, searchLookup, ctx); - this.sourcePaths = searchLookup.sourcePaths(fieldName); } /** @@ -126,20 +99,6 @@ public final void runForDoc(int docId, Consumer consumer) { resultsForDoc(docId).forEach(consumer); } - /** - * emitFromSourcePaths is used for the script fields API to handle source fallback - * when doc values are not available. This differs from {@link #emitFromSourcePaths()} - * by handling multiple source paths for possible mutli-field relationships and copy-to relationships. - * See {@link org.elasticsearch.index.mapper.MappingLookup#sourcePaths(String)}. - */ - protected void emitFromSourcePaths() { - for (String sourcePath : sourcePaths) { - for (Object v : extractFromSource(sourcePath)) { - emitFromObject(v); - } - } - } - @Override protected void emitValueFromCompositeScript(Object value) { if (value != null) { From 9062de74970985de412c2e4836aa349889b04bf9 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 11 Jul 2022 18:04:44 -0700 Subject: [PATCH 10/23] replace tuple return with marker interface --- .../script/ScriptScoreBenchmark.java | 7 +- .../search/QueryParserHelperBenchmark.java | 8 +- .../ExpressionFieldScriptTests.java | 3 +- .../ExpressionNumberSortScriptTests.java | 3 +- .../ExpressionTermsSetQueryTests.java | 3 +- .../fielddata/IndexFieldDataService.java | 11 +-- .../KeywordValueFetcherIndexFieldData.java | 10 ++- .../index/mapper/DocumentParser.java | 11 +-- .../index/mapper/KeywordFieldMapper.java | 39 ++++------ .../index/mapper/MappedFieldType.java | 12 +-- .../index/query/SearchExecutionContext.java | 11 +-- .../search/lookup/LeafDocLookup.java | 74 ++++++------------- .../search/lookup/SearchLookup.java | 7 +- .../AbstractScriptFieldTypeTestCase.java | 3 +- .../mapper/CompositeRuntimeFieldTests.java | 3 +- .../query/SearchExecutionContextTests.java | 6 +- .../search/lookup/LeafDocLookupTests.java | 9 +-- .../search/sort/AbstractSortTestCase.java | 7 +- .../index/mapper/MapperServiceTestCase.java | 15 ++-- .../aggregations/AggregatorTestCase.java | 9 +-- .../AggregateDoubleMetricFieldTypeTests.java | 3 +- .../mapper/WildcardFieldMapperTests.java | 7 +- 22 files changed, 87 insertions(+), 174 deletions(-) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index 861ac6fef5afb..86e1fa292131d 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -25,8 +25,6 @@ import org.elasticsearch.Version; import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.Tuple; -import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.mapper.MappedFieldType; @@ -90,10 +88,7 @@ public class ScriptScoreBenchmark { private SearchLookup lookup = new SearchLookup( fieldTypes::get, (mft, lookup) -> mft.fielddataBuilder("test", lookup).build(fieldDataCache, breakerService), - (mft, lookup) -> { - Tuple sfd = mft.scriptFielddataBuilder("test", lookup); - return new Tuple<>(sfd.v1(), sfd.v2().build(fieldDataCache, breakerService)); - }, + (mft, lookup) -> mft.scriptFielddataBuilder("test", lookup).build(fieldDataCache, breakerService), sourcePaths::get ); diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java index 0a9853a5a3f4a..28fab7639a99f 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java @@ -26,12 +26,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.IOUtils; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; -import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.index.mapper.MapperService; @@ -145,10 +143,8 @@ protected SearchExecutionContext buildSearchExecutionContext() { null, (ft, idxName, lookup) -> ft.fielddataBuilder(idxName, lookup) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), - (ft, idxName, lookup) -> { - Tuple sfd = ft.scriptFielddataBuilder(idxName, lookup); - return new Tuple<>(sfd.v1(), sfd.v2().build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService())); - }, + (ft, idxName, lookup) -> ft.scriptFielddataBuilder(idxName, lookup) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), mapperService, mapperService.mappingLookup(), similarityService, diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java index 1385f66dd0d24..ff0ec76bfb6a1 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.script.expression; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; @@ -54,7 +53,7 @@ public void setUp() throws Exception { lookup = new SearchLookup( field -> field.equals("field") ? fieldType : null, (ignored, _lookup) -> fieldData, - (ignored, _lookup) -> new Tuple<>(true, fieldData), + (ignored, _lookup) -> fieldData, field -> field.equals("field") ? Set.of("field") : null ); } diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java index 1c6b1fc3a26c7..4418d86947a44 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.script.expression; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; @@ -55,7 +54,7 @@ public void setUp() throws Exception { lookup = new SearchLookup( field -> field.equals("field") ? fieldType : null, (ignored, _lookup) -> fieldData, - (ignored, _lookup) -> new Tuple<>(true, fieldData), + (ignored, _lookup) -> fieldData, field -> field.equals("field") ? Set.of("field") : null ); } diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java index 29720b213cead..210d1197499f8 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.script.expression; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.IndexNumericFieldData; import org.elasticsearch.index.fielddata.LeafNumericFieldData; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; @@ -54,7 +53,7 @@ public void setUp() throws Exception { lookup = new SearchLookup( field -> field.equals("field") ? fieldType : null, (ignored, _lookup) -> fieldData, - (ignored, _lookup) -> new Tuple<>(true, fieldData), + (ignored, _lookup) -> fieldData, field -> field.equals("field") ? Set.of("field") : null ); } diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index 87f42b64ef214..72490bfd50765 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -12,7 +12,6 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.MappedFieldType; @@ -115,19 +114,15 @@ public > IFD getForField( * while also making a {@link SearchLookup} supplier available that is required for runtime fields. */ @SuppressWarnings("unchecked") - public > Tuple getForScriptField( + public > IFD getForScriptField( MappedFieldType fieldType, String fullyQualifiedIndexName, Supplier searchLookup ) { final String fieldName = fieldType.name(); - Tuple scriptFielddataBuilder = fieldType.scriptFielddataBuilder( - fullyQualifiedIndexName, - searchLookup - ); - IndexFieldData.Builder builder = scriptFielddataBuilder.v2(); + IndexFieldData.Builder builder = fieldType.scriptFielddataBuilder(fullyQualifiedIndexName, searchLookup); IndexFieldDataCache cache = getFieldDataCache(fieldName); - return new Tuple<>(scriptFielddataBuilder.v1(), (IFD) builder.build(cache, circuitBreakerService)); + return (IFD) builder.build(cache, circuitBreakerService); } private synchronized IndexFieldDataCache getFieldDataCache(String fieldName) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/KeywordValueFetcherIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/KeywordValueFetcherIndexFieldData.java index 39894a90e8141..8613d777abb7f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/KeywordValueFetcherIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/KeywordValueFetcherIndexFieldData.java @@ -163,7 +163,7 @@ public SortedBinaryDocValues getBytesValues() { } } - public static class KeywordValueFetcherBinaryDocValues extends SortedBinaryDocValues { + public static class KeywordValueFetcherBinaryDocValues extends SortedBinaryDocValues implements ValueFetcherDocValues { private final LeafReaderContext leafReaderContext; @@ -203,4 +203,12 @@ public BytesRef nextValue() throws IOException { return new BytesRef(iterator.next().toString()); } } + + /** + * Marker interface to indicate these doc values are generated + * on-the-fly from a {@code ValueFetcher}. + */ + public interface ValueFetcherDocValues { + // marker interface + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 2ea3f00a091fd..5649a638decf9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -17,10 +17,8 @@ import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.IndexAnalyzers; -import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; @@ -145,13 +143,8 @@ private static void executeIndexTimeScripts(DocumentParserContext context) { context.mappingLookup().indexTimeLookup()::get, (ft, lookup) -> ft.fielddataBuilder(context.indexSettings().getIndex().getName(), lookup) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), - (ft, lookup) -> { - Tuple sfd = ft.scriptFielddataBuilder( - context.indexSettings().getIndex().getName(), - lookup - ); - return new Tuple<>(sfd.v1(), sfd.v2().build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService())); - }, + (ft, lookup) -> ft.scriptFielddataBuilder(context.indexSettings().getIndex().getName(), lookup) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), context.mappingLookup()::sourcePaths ); // field scripts can be called both by the loop at the end of this method and via diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 79eafd955b6db..f939b23992f10 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -43,7 +43,6 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; @@ -693,32 +692,26 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S } @Override - public Tuple scriptFielddataBuilder( - String fullyQualifiedIndexName, - Supplier searchLookup - ) { + public IndexFieldData.Builder scriptFielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { if (hasDocValues()) { - return new Tuple<>(true, fielddataBuilder(fullyQualifiedIndexName, searchLookup)); + return fielddataBuilder(fullyQualifiedIndexName, searchLookup); } - return new Tuple<>( - false, - new KeywordValueFetcherIndexFieldData.Builder( - name(), - new SourceValueFetcher(searchLookup.get().sourcePaths(name()), nullValue) { - @Override - protected String parseSourceValue(Object value) { - String keywordValue = value.toString(); - if (keywordValue.length() > ignoreAbove) { - return null; - } - - return normalizeValue(normalizer(), name(), keywordValue); + return new KeywordValueFetcherIndexFieldData.Builder( + name(), + new SourceValueFetcher(searchLookup.get().sourcePaths(name()), nullValue) { + @Override + protected String parseSourceValue(Object value) { + String keywordValue = value.toString(); + if (keywordValue.length() > ignoreAbove) { + return null; } - }, - searchLookup.get().source(), - KeywordDocValuesField::new - ) + + return normalizeValue(normalizer(), name(), keywordValue); + } + }, + searchLookup.get().source(), + KeywordDocValuesField::new ); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index e054b873aa419..26ad58a96cb80 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.query.DistanceFeatureQueryBuilder; import org.elasticsearch.index.query.QueryRewriteContext; @@ -103,16 +102,9 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S * * @param fullyQualifiedIndexName the name of the index this field-data is build for * @param searchLookup a {@link SearchLookup} supplier to allow for accessing other fields values in the context of runtime fields - * - * @return returns {@code true} if field data was used to generate values otherwise {@code false}, along with the - * {@link IndexFieldData.Builder} as part of a {@link Tuple}. This is required to maintain the original - * behavior of the old-style accessors for scripting like {@code doc['field'].value} */ - public Tuple scriptFielddataBuilder( - String fullyQualifiedIndexName, - Supplier searchLookup - ) { - return Tuple.tuple(true, fielddataBuilder(fullyQualifiedIndexName, searchLookup)); + public IndexFieldData.Builder scriptFielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + return fielddataBuilder(fullyQualifiedIndexName, searchLookup); } /** diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 1a69715238998..606470be0c083 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.core.CheckedFunction; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; @@ -93,11 +92,7 @@ public class SearchExecutionContext extends QueryRewriteContext { private final SimilarityService similarityService; private final BitsetFilterCache bitsetFilterCache; private final TriFunction, IndexFieldData> indexFieldDataLookup; - private final TriFunction< - MappedFieldType, - String, - Supplier, - Tuple>> scriptIndexFieldDataLookup; + private final TriFunction, IndexFieldData> scriptIndexFieldDataLookup; private SearchLookup lookup = null; private final int shardId; @@ -128,7 +123,7 @@ public SearchExecutionContext( IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, TriFunction, IndexFieldData> indexFieldDataLookup, - TriFunction, Tuple>> scriptIndexFieldDataLookup, + TriFunction, IndexFieldData> scriptIndexFieldDataLookup, MapperService mapperService, MappingLookup mappingLookup, SimilarityService similarityService, @@ -204,7 +199,7 @@ private SearchExecutionContext( IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, TriFunction, IndexFieldData> indexFieldDataLookup, - TriFunction, Tuple>> scriptIndexFieldDataLookup, + TriFunction, IndexFieldData> scriptIndexFieldDataLookup, MapperService mapperService, MappingLookup mappingLookup, SimilarityService similarityService, diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java index d24391d130ffe..8aa6af9927d24 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java @@ -10,8 +10,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.util.Maps; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.KeywordValueFetcherIndexFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; @@ -29,17 +29,17 @@ public class LeafDocLookup implements Map> { private final Function fieldTypeLookup; private final Function> fieldDataLookup; - private final Function>> scriptFieldDataLookup; + private final Function> scriptFieldDataLookup; private final LeafReaderContext reader; private int docId = -1; - private final Map, DocValuesScriptFieldFactory> localCacheScriptFieldData = Maps.newMapWithExpectedSize(4); + private final Map localCacheScriptFieldData = Maps.newMapWithExpectedSize(4); LeafDocLookup( Function fieldTypeLookup, Function> fieldDataLookup, - Function>> scriptFieldDataLookup, + Function> scriptFieldDataLookup, LeafReaderContext reader ) { this.fieldTypeLookup = fieldTypeLookup; @@ -52,51 +52,14 @@ public void setDocument(int docId) { this.docId = docId; } - public Field getScriptField(String fieldName) { - DocValuesScriptFieldFactory factory = localCacheScriptFieldData.get(new Tuple<>(fieldName, false)); - - if (factory == null) { - factory = localCacheScriptFieldData.get(new Tuple<>(fieldName, true)); - } - - if (factory == null) { - final MappedFieldType fieldType = fieldTypeLookup.apply(fieldName); - - if (fieldType == null) { - throw new IllegalArgumentException("No field found for [" + fieldName + "] in mapping"); - } - - // Load the field data on behalf of the script. Otherwise, it would require - // additional permissions to deal with pagedbytes/ramusagestimator/etc. - Tuple fdt = AccessController.doPrivileged( - new PrivilegedAction>() { - @Override - public Tuple run() { - Tuple> fdt = scriptFieldDataLookup.apply(fieldType); - DocValuesScriptFieldFactory dvf = fdt.v2().load(reader).getScriptFieldFactory(fieldName); - return new Tuple<>(fdt.v1(), dvf); - } - } - ); - - factory = fdt.v2(); - localCacheScriptFieldData.put(new Tuple<>(fieldName, fdt.v1()), factory); - } + protected DocValuesScriptFieldFactory getScriptFieldFactory(String fieldName, Function> lookup) { + DocValuesScriptFieldFactory factory = localCacheScriptFieldData.get(fieldName); - try { - factory.setNextDocId(docId); - } catch (IOException ioe) { - throw ExceptionsHelper.convertToElastic(ioe); + // do not use cached source fallback fields for old style doc access + if (lookup == fieldDataLookup && factory instanceof KeywordValueFetcherIndexFieldData.ValueFetcherDocValues) { + factory = null; } - return factory.toScriptField(); - } - - @Override - public ScriptDocValues get(Object key) { - String fieldName = key.toString(); - DocValuesScriptFieldFactory factory = localCacheScriptFieldData.get(new Tuple<>(fieldName, true)); - if (factory == null) { final MappedFieldType fieldType = fieldTypeLookup.apply(fieldName); @@ -109,11 +72,11 @@ public ScriptDocValues get(Object key) { factory = AccessController.doPrivileged(new PrivilegedAction() { @Override public DocValuesScriptFieldFactory run() { - return fieldDataLookup.apply(fieldType).load(reader).getScriptFieldFactory(fieldName); + return lookup.apply(fieldType).load(reader).getScriptFieldFactory(fieldName); } }); - localCacheScriptFieldData.put(new Tuple<>(fieldName, true), factory); + localCacheScriptFieldData.put(fieldName, factory); } try { @@ -122,15 +85,22 @@ public DocValuesScriptFieldFactory run() { throw ExceptionsHelper.convertToElastic(ioe); } - return factory.toScriptDocValues(); + return factory; + } + + public Field getScriptField(String fieldName) { + return getScriptFieldFactory(fieldName, scriptFieldDataLookup).toScriptField(); + } + + @Override + public ScriptDocValues get(Object key) { + return getScriptFieldFactory(key.toString(), fieldDataLookup).toScriptDocValues(); } @Override public boolean containsKey(Object key) { String fieldName = key.toString(); - return localCacheScriptFieldData.get(new Tuple<>(fieldName, true)) != null - || localCacheScriptFieldData.get(new Tuple<>(fieldName, false)) != null - || fieldTypeLookup.apply(fieldName) != null; + return localCacheScriptFieldData.get(fieldName) != null || fieldTypeLookup.apply(fieldName) != null; } @Override diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index 989667965dd0f..f7defe1359487 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -9,7 +9,6 @@ package org.elasticsearch.search.lookup; import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; @@ -43,7 +42,7 @@ public class SearchLookup { private final SourceLookup sourceLookup; private final Function fieldTypeLookup; private final BiFunction, IndexFieldData> fieldDataLookup; - private final BiFunction, Tuple>> scriptFieldDataLookup; + private final BiFunction, IndexFieldData> scriptFieldDataLookup; private final Function> sourcePathsLookup; /** @@ -53,7 +52,7 @@ public class SearchLookup { public SearchLookup( Function fieldTypeLookup, BiFunction, IndexFieldData> fieldDataLookup, - BiFunction, Tuple>> scriptFieldDataLookup, + BiFunction, IndexFieldData> scriptFieldDataLookup, Function> sourcePathsLookup ) { this.fieldTypeLookup = fieldTypeLookup; @@ -118,7 +117,7 @@ public IndexFieldData getForField(MappedFieldType fieldType) { return fieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name())); } - public Tuple> getForScriptField(MappedFieldType fieldType) { + public IndexFieldData getForScriptField(MappedFieldType fieldType) { return scriptFieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name())); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index 71a4f0261881a..b27f02d48af78 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.script.BooleanFieldScript; import org.elasticsearch.script.DateFieldScript; @@ -204,7 +203,7 @@ protected static SearchExecutionContext mockContext(boolean allowExpensiveQuerie SearchLookup lookup = new SearchLookup( context::getFieldType, (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null), - (mft, lookupSupplier) -> new Tuple<>(true, mft.fielddataBuilder("test", lookupSupplier).build(null, null)), + (mft, lookupSupplier) -> mft.scriptFielddataBuilder("test", lookupSupplier).build(null, null), context::sourcePath ); when(context.lookup()).thenReturn(lookup); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java index 40ab45c3ea3ce..fca89107d7e53 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.mapper; import org.elasticsearch.common.Strings; -import org.elasticsearch.core.Tuple; import org.elasticsearch.script.CompositeFieldScript; import org.elasticsearch.script.LongFieldScript; import org.elasticsearch.script.Script; @@ -335,7 +334,7 @@ public void testParseDocumentSubFieldAccess() throws IOException { SearchLookup searchLookup = new SearchLookup( mapperService::fieldType, (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null), - (mft, lookupSupplier) -> new Tuple<>(true, mft.fielddataBuilder("test", lookupSupplier).build(null, null)), + (mft, lookupSupplier) -> mft.scriptFielddataBuilder("test", lookupSupplier).build(null, null), mapperService.mappingLookup()::sourcePaths ); diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 82ffea0faefd6..5b63a6705847f 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -429,10 +428,7 @@ private static SearchExecutionContext createSearchExecutionContext( indexSettings, null, (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), - (mappedFieldType, idxName, searchLookup) -> { - Tuple sfd = mappedFieldType.scriptFielddataBuilder(idxName, searchLookup); - return new Tuple<>(sfd.v1(), sfd.v2().build(null, null)); - }, + (mappedFieldType, idxName, searchLookup) -> mappedFieldType.scriptFielddataBuilder(idxName, searchLookup).build(null, null), mapperService, mappingLookup, null, diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java index 48dcfb9e51b76..7992c6dc4e600 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java @@ -7,7 +7,6 @@ */ package org.elasticsearch.search.lookup; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; @@ -52,9 +51,7 @@ public void setUp() throws Exception { docLookup = new LeafDocLookup( field -> field.equals("field") ? fieldType1 : field.equals("alias") ? fieldType2 : null, fieldType -> fieldType == fieldType1 ? fieldData1 : fieldType == fieldType2 ? fieldData2 : null, - fieldType -> fieldType == fieldType1 ? new Tuple<>(true, fieldData1) - : fieldType == fieldType2 ? new Tuple<>(true, fieldData2) - : null, + fieldType -> fieldType == fieldType1 ? fieldData1 : fieldType == fieldType2 ? fieldData2 : null, null ); } @@ -86,9 +83,9 @@ public void testFlattenedField() throws IOException { return keyedFieldType.key().equals("key1") ? fieldData1 : fieldData2; }; - Function>> scriptFieldDataSupplier = ft -> { + Function> scriptFieldDataSupplier = ft -> { FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) ft; - return keyedFieldType.key().equals("key1") ? new Tuple<>(true, fieldData1) : new Tuple<>(true, fieldData2); + return keyedFieldType.key().equals("key1") ? fieldData1 : fieldData2; }; LeafDocLookup docLookup = new LeafDocLookup(field -> { diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 0d030d94fd51e..8a5ad6c5500d2 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -15,7 +15,6 @@ import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.Tuple; import org.elasticsearch.env.Environment; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; @@ -197,12 +196,12 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null); }; - TriFunction, Tuple>> scriptIndexFieldDataLookup = ( + TriFunction, IndexFieldData> scriptIndexFieldDataLookup = ( fieldType, fieldIndexName, searchLookup) -> { - Tuple sfd = fieldType.scriptFielddataBuilder(fieldIndexName, searchLookup); - return new Tuple<>(sfd.v1(), sfd.v2().build(new IndexFieldDataCache.None(), null)); + IndexFieldData.Builder builder = fieldType.scriptFielddataBuilder(fieldIndexName, searchLookup); + return builder.build(new IndexFieldDataCache.None(), null); }; NestedLookup nestedLookup = NestedLookup.build( List.of(new NestedObjectMapper.Builder("path", Version.CURRENT).build(MapperBuilderContext.ROOT)) diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index b4cf9725e6c28..5adf9a425360a 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -31,7 +31,6 @@ import org.elasticsearch.common.util.MockPageCacheRecycler; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; @@ -627,10 +626,8 @@ public void onRemoval(ShardId shardId, Accountable accountable) { }), (ft, idxName, lookup) -> ft.fielddataBuilder(idxName, lookup) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), - (ft, idxName, lookup) -> { - Tuple sfd = ft.scriptFielddataBuilder(idxName, lookup); - return new Tuple<>(sfd.v1(), sfd.v2().build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService())); - }, + (ft, idxName, lookup) -> ft.scriptFielddataBuilder(idxName, lookup) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), mapperService, mapperService.mappingLookup(), similarityService, @@ -653,11 +650,9 @@ protected BiFunction, IndexFieldData> .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } - protected BiFunction, Tuple>> scriptFieldDataLookup() { - return (mft, lookupSource) -> new Tuple<>( - true, - mft.fielddataBuilder("test", lookupSource).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) - ); + protected BiFunction, IndexFieldData> scriptFieldDataLookup() { + return (mft, lookupSource) -> mft.scriptFielddataBuilder("test", lookupSource) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } protected final String syntheticSource(DocumentMapper mapper, CheckedConsumer build) throws IOException { diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index c2b50d90e1862..c03a0958130c0 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -59,7 +59,6 @@ import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; @@ -298,13 +297,11 @@ protected AggregationContext createAggregationContext( s, searchLookup) -> fieldType.fielddataBuilder(indexSettings.getIndex().getName(), searchLookup) .build(new IndexFieldDataCache.None(), breakerService); - TriFunction, Tuple>> scriptFieldDataBuilder = ( + TriFunction, IndexFieldData> scriptFieldDataBuilder = ( fieldType, s, - searchLookup) -> { - Tuple sfd = fieldType.scriptFielddataBuilder(indexSettings.getIndex().getName(), searchLookup); - return new Tuple<>(sfd.v1(), sfd.v2().build(new IndexFieldDataCache.None(), breakerService)); - }; + searchLookup) -> fieldType.fielddataBuilder(indexSettings.getIndex().getName(), searchLookup) + .build(new IndexFieldDataCache.None(), breakerService); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() { @Override public void onRemoval(ShardId shardId, Accountable accountable) {} diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java index 6461423a6f6c5..2469deaf910af 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java @@ -17,7 +17,6 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.elasticsearch.Version; import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.MappedFieldType; @@ -123,7 +122,7 @@ public void testUsedInScript() throws IOException { SearchLookup lookup = new SearchLookup( searchExecutionContext::getFieldType, (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null), - (mft, lookupSupplier) -> new Tuple<>(true, mft.fielddataBuilder("test", lookupSupplier).build(null, null)), + (mft, lookupSupplier) -> mft.scriptFielddataBuilder("test", lookupSupplier).build(null, null), searchExecutionContext::sourcePath ); when(searchExecutionContext.lookup()).thenReturn(lookup); diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index b0411f9c754dd..26ab0ffeeea60 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -48,7 +48,6 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; -import org.elasticsearch.core.Tuple; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; @@ -1083,12 +1082,12 @@ protected final SearchExecutionContext createMockContext() { IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null); }; - TriFunction, Tuple>> scriptIndexFieldDataLookup = ( + TriFunction, IndexFieldData> scriptIndexFieldDataLookup = ( fieldType, fieldIndexName, searchLookup) -> { - Tuple sfd = fieldType.scriptFielddataBuilder(fieldIndexName, searchLookup); - return new Tuple<>(sfd.v1(), sfd.v2().build(new IndexFieldDataCache.None(), null)); + IndexFieldData.Builder builder = fieldType.scriptFielddataBuilder(fieldIndexName, searchLookup); + return builder.build(new IndexFieldDataCache.None(), null); }; return new SearchExecutionContext( 0, From 170b9af406c81a504ae9a6137556aa058b82f125 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Thu, 14 Jul 2022 13:52:07 -0700 Subject: [PATCH 11/23] Split off value fetcher index field data into base classes. --- .../KeywordValueFetcherIndexFieldData.java | 214 ------------------ .../SourceValueFetcherIndexFieldData.java | 147 ++++++++++++ ...alueFetcherSortedBinaryIndexFieldData.java | 129 +++++++++++ .../index/mapper/KeywordFieldMapper.java | 7 +- .../search/lookup/LeafDocLookup.java | 4 +- 5 files changed, 283 insertions(+), 218 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/KeywordValueFetcherIndexFieldData.java create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/KeywordValueFetcherIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/KeywordValueFetcherIndexFieldData.java deleted file mode 100644 index 8613d777abb7f..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/fielddata/KeywordValueFetcherIndexFieldData.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.index.fielddata; - -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.SortField; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.common.util.BigArrays; -import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; -import org.elasticsearch.index.mapper.ValueFetcher; -import org.elasticsearch.indices.breaker.CircuitBreakerService; -import org.elasticsearch.script.field.DocValuesScriptFieldFactory; -import org.elasticsearch.script.field.ToScriptFieldFactory; -import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.MultiValueMode; -import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import org.elasticsearch.search.lookup.SourceLookup; -import org.elasticsearch.search.sort.BucketedSort; -import org.elasticsearch.search.sort.SortOrder; - -import java.io.IOException; -import java.util.Collections; -import java.util.Iterator; -import java.util.SortedSet; -import java.util.TreeSet; - -public class KeywordValueFetcherIndexFieldData - implements - IndexFieldData { - - public static class Builder implements IndexFieldData.Builder { - private final String name; - private final ValueFetcher valueFetcher; - private final SourceLookup sourceLookup; - private final ToScriptFieldFactory toScriptFieldFactory; - - public Builder( - String name, - ValueFetcher valueFetcher, - SourceLookup sourceLookup, - ToScriptFieldFactory toScriptFieldFactory - ) { - this.name = name; - this.valueFetcher = valueFetcher; - this.sourceLookup = sourceLookup; - this.toScriptFieldFactory = toScriptFieldFactory; - } - - @Override - public KeywordValueFetcherIndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { - return new KeywordValueFetcherIndexFieldData(name, valueFetcher, sourceLookup, toScriptFieldFactory); - } - } - - private final String fieldName; - private final ValueFetcher valueFetcher; - private final SourceLookup sourceLookup; - private final ToScriptFieldFactory toScriptFieldFactory; - - protected KeywordValueFetcherIndexFieldData( - String fieldName, - ValueFetcher valueFetcher, - SourceLookup sourceLookup, - ToScriptFieldFactory toScriptFieldFactory - ) { - this.fieldName = fieldName; - this.valueFetcher = valueFetcher; - this.sourceLookup = sourceLookup; - this.toScriptFieldFactory = toScriptFieldFactory; - } - - @Override - public String getFieldName() { - return fieldName; - } - - @Override - public ValuesSourceType getValuesSourceType() { - return CoreValuesSourceType.KEYWORD; - } - - @Override - public KeywordValueFetcherLeafFieldData load(LeafReaderContext context) { - try { - return loadDirect(context); - } catch (Exception e) { - throw ExceptionsHelper.convertToElastic(e); - } - } - - @Override - public KeywordValueFetcherLeafFieldData loadDirect(LeafReaderContext context) throws Exception { - return new KeywordValueFetcherLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup); - - } - - @Override - public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { - final XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); - return new SortField(getFieldName(), source, reverse); - } - - @Override - public BucketedSort newBucketedSort( - BigArrays bigArrays, - Object missingValue, - MultiValueMode sortMode, - XFieldComparatorSource.Nested nested, - SortOrder sortOrder, - DocValueFormat format, - int bucketSize, - BucketedSort.ExtraData extra - ) { - throw new IllegalArgumentException("only supported on numeric fields"); - } - - public static class KeywordValueFetcherLeafFieldData implements LeafFieldData { - - private final ToScriptFieldFactory toScriptFieldFactory; - private final LeafReaderContext leafReaderContext; - - private final ValueFetcher valueFetcher; - private final SourceLookup sourceLookup; - - public KeywordValueFetcherLeafFieldData( - ToScriptFieldFactory toScriptFieldFactory, - LeafReaderContext leafReaderContext, - ValueFetcher valueFetcher, - SourceLookup sourceLookup - ) { - this.toScriptFieldFactory = toScriptFieldFactory; - this.leafReaderContext = leafReaderContext; - this.valueFetcher = valueFetcher; - this.sourceLookup = sourceLookup; - } - - @Override - public long ramBytesUsed() { - return 0; - } - - @Override - public void close() { - - } - - @Override - public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { - return toScriptFieldFactory.getScriptFieldFactory(getBytesValues(), name); - } - - @Override - public SortedBinaryDocValues getBytesValues() { - return new KeywordValueFetcherBinaryDocValues(leafReaderContext, valueFetcher, sourceLookup); - } - } - - public static class KeywordValueFetcherBinaryDocValues extends SortedBinaryDocValues implements ValueFetcherDocValues { - - private final LeafReaderContext leafReaderContext; - - private final ValueFetcher valueFetcher; - private final SourceLookup sourceLookup; - - private SortedSet values; - private Iterator iterator; - - public KeywordValueFetcherBinaryDocValues( - LeafReaderContext leafReaderContext, - ValueFetcher valueFetcher, - SourceLookup sourceLookup - ) { - this.leafReaderContext = leafReaderContext; - this.valueFetcher = valueFetcher; - this.sourceLookup = sourceLookup; - } - - @Override - public boolean advanceExact(int doc) throws IOException { - sourceLookup.setSegmentAndDocument(leafReaderContext, doc); - values = new TreeSet<>(valueFetcher.fetchValues(sourceLookup, Collections.emptyList())); - iterator = values.iterator(); - - return true; - } - - @Override - public int docValueCount() { - return values.size(); - } - - @Override - public BytesRef nextValue() throws IOException { - assert iterator.hasNext(); - return new BytesRef(iterator.next().toString()); - } - } - - /** - * Marker interface to indicate these doc values are generated - * on-the-fly from a {@code ValueFetcher}. - */ - public interface ValueFetcherDocValues { - // marker interface - } -} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java new file mode 100644 index 0000000000000..dc05af9d65387 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java @@ -0,0 +1,147 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.SortField; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.script.field.ToScriptFieldFactory; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.search.sort.BucketedSort; +import org.elasticsearch.search.sort.SortOrder; + +public abstract class SourceValueFetcherIndexFieldData + implements + IndexFieldData> { + + public static abstract class Builder implements IndexFieldData.Builder { + + protected final String fieldName; + protected final ValuesSourceType valuesSourceType; + protected final ValueFetcher valueFetcher; + protected final SourceLookup sourceLookup; + protected final ToScriptFieldFactory toScriptFieldFactory; + + public Builder( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + this.fieldName = fieldName; + this.valuesSourceType = valuesSourceType; + this.valueFetcher = valueFetcher; + this.sourceLookup = sourceLookup; + this.toScriptFieldFactory = toScriptFieldFactory; + } + } + + protected final String fieldName; + protected final ValuesSourceType valuesSourceType; + protected final ValueFetcher valueFetcher; + protected final SourceLookup sourceLookup; + protected final ToScriptFieldFactory toScriptFieldFactory; + + protected SourceValueFetcherIndexFieldData( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + this.fieldName = fieldName; + this.valuesSourceType = valuesSourceType; + this.valueFetcher = valueFetcher; + this.sourceLookup = sourceLookup; + this.toScriptFieldFactory = toScriptFieldFactory; + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return valuesSourceType; + } + + @Override + public SourceValueFetcherLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { + throw new IllegalArgumentException("not supported for source fallback"); + } + + @Override + public BucketedSort newBucketedSort( + BigArrays bigArrays, + Object missingValue, + MultiValueMode sortMode, + XFieldComparatorSource.Nested nested, + SortOrder sortOrder, + DocValueFormat format, + int bucketSize, + BucketedSort.ExtraData extra + ) { + throw new IllegalArgumentException("not supported for source fallback"); + } + + public abstract static class SourceValueFetcherLeafFieldData implements LeafFieldData { + + protected final ToScriptFieldFactory toScriptFieldFactory; + protected final LeafReaderContext leafReaderContext; + + protected final ValueFetcher valueFetcher; + protected final SourceLookup sourceLookup; + + public SourceValueFetcherLeafFieldData( + ToScriptFieldFactory toScriptFieldFactory, + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + this.toScriptFieldFactory = toScriptFieldFactory; + this.leafReaderContext = leafReaderContext; + this.valueFetcher = valueFetcher; + this.sourceLookup = sourceLookup; + } + + @Override + public long ramBytesUsed() { + return 0; + } + + @Override + public void close() { + + } + } + + /** + * Marker interface to indicate these doc values are generated + * on-the-fly from a {@code ValueFetcher}. + */ + public interface ValueFetcherDocValues { + // marker interface + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java new file mode 100644 index 0000000000000..2e0780c45b895 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.ToScriptFieldFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.SortedSet; +import java.util.TreeSet; + +public class SourceValueFetcherSortedBinaryIndexFieldData extends SourceValueFetcherIndexFieldData { + + public static class Builder extends SourceValueFetcherIndexFieldData.Builder { + + public Builder( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory); + } + + @Override + public SourceValueFetcherSortedBinaryIndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { + return new SourceValueFetcherSortedBinaryIndexFieldData( + fieldName, + valuesSourceType, + valueFetcher, + sourceLookup, + toScriptFieldFactory + ); + } + } + + protected SourceValueFetcherSortedBinaryIndexFieldData( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory); + } + + @Override + public SourceValueFetcherSortedBinaryLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + return new SourceValueFetcherSortedBinaryLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup); + } + + public static class SourceValueFetcherSortedBinaryLeafFieldData extends SourceValueFetcherLeafFieldData { + + public SourceValueFetcherSortedBinaryLeafFieldData( + ToScriptFieldFactory toScriptFieldFactory, + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceLookup); + } + + @Override + public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { + return toScriptFieldFactory.getScriptFieldFactory(getBytesValues(), name); + } + + @Override + public SortedBinaryDocValues getBytesValues() { + return new ValueFetcherSortedBinaryDocValues(leafReaderContext, valueFetcher, sourceLookup); + } + } + + public static class ValueFetcherSortedBinaryDocValues extends SortedBinaryDocValues implements ValueFetcherDocValues { + + private final LeafReaderContext leafReaderContext; + + private final ValueFetcher valueFetcher; + private final SourceLookup sourceLookup; + + private SortedSet values; + private Iterator iterator; + + public ValueFetcherSortedBinaryDocValues( + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + this.leafReaderContext = leafReaderContext; + this.valueFetcher = valueFetcher; + this.sourceLookup = sourceLookup; + } + + @Override + public boolean advanceExact(int doc) throws IOException { + sourceLookup.setSegmentAndDocument(leafReaderContext, doc); + values = new TreeSet<>(valueFetcher.fetchValues(sourceLookup, Collections.emptyList())); + iterator = values.iterator(); + + return true; + } + + @Override + public int docValueCount() { + return values.size(); + } + + @Override + public BytesRef nextValue() throws IOException { + assert iterator.hasNext(); + return new BytesRef(iterator.next().toString()); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index aaf81ba2c85a5..0ee2e015061b6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -47,7 +47,8 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.KeywordValueFetcherIndexFieldData; +import org.elasticsearch.index.fielddata.SourceValueFetcherIndexFieldData; +import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.similarity.SimilarityProvider; @@ -57,6 +58,7 @@ import org.elasticsearch.script.StringFieldScript; import org.elasticsearch.script.field.KeywordDocValuesField; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.runtime.StringScriptFieldFuzzyQuery; @@ -697,8 +699,9 @@ public IndexFieldData.Builder scriptFielddataBuilder(String fullyQualifiedIndexN return fielddataBuilder(fullyQualifiedIndexName, searchLookup); } - return new KeywordValueFetcherIndexFieldData.Builder( + return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), + CoreValuesSourceType.KEYWORD, new SourceValueFetcher(searchLookup.get().sourcePaths(name()), nullValue) { @Override protected String parseSourceValue(Object value) { diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java index 8aa6af9927d24..f7d1f8fab59af 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java @@ -11,8 +11,8 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.util.Maps; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.KeywordValueFetcherIndexFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SourceValueFetcherIndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.script.field.DocValuesScriptFieldFactory; import org.elasticsearch.script.field.Field; @@ -56,7 +56,7 @@ protected DocValuesScriptFieldFactory getScriptFieldFactory(String fieldName, Fu DocValuesScriptFieldFactory factory = localCacheScriptFieldData.get(fieldName); // do not use cached source fallback fields for old style doc access - if (lookup == fieldDataLookup && factory instanceof KeywordValueFetcherIndexFieldData.ValueFetcherDocValues) { + if (lookup == fieldDataLookup && factory instanceof SourceValueFetcherIndexFieldData.ValueFetcherDocValues) { factory = null; } From 50beaea9d00ecb499fd32adf43c8b547f0d922ea Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 18 Jul 2022 09:25:36 -0700 Subject: [PATCH 12/23] Add integer field to source fallback w/ value fetcher strategy. --- .../test/painless/50_script_doc_values.yml | 81 ++++++++++ .../SourceValueFetcherIndexFieldData.java | 5 + ...alueFetcherSortedBinaryIndexFieldData.java | 14 +- ...lueFetcherSortedNumericIndexFieldData.java | 151 ++++++++++++++++++ .../index/mapper/KeywordFieldMapper.java | 2 - .../index/mapper/NumberFieldMapper.java | 42 +++++ 6 files changed, 285 insertions(+), 10 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index 8573b7d3e85ae..b4f6fb9fd5c8a 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -26,6 +26,9 @@ setup: type: long integer: type: integer + integer_no_doc_values: + type: integer + doc_values: false short: type: short byte: @@ -63,6 +66,7 @@ setup: keyword_no_doc_values: no doc values long: 12348732141234 integer: 134134566 + integer_no_doc_values: 134134566 short: 1324 byte: 12 double: 3.14159265358979 @@ -93,6 +97,7 @@ setup: keyword_no_doc_values: ["no doc values 1", "no doc values 0", "no doc values 2"] long: [1152921504606846976, 576460752303423488] integer: [5, 17, 29] + integer_no_doc_values: [17, 29, 5] short: [6, 18, 30, 45] byte: [16, 32, 64, 8, 4] double: [3.141592653588, 2.141592653587] @@ -1047,6 +1052,82 @@ setup: - match: { hits.hits.1.fields.field.0: 0 } - match: { hits.hits.2.fields.field.0: 54 } +--- +"integer_no_doc_values": + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "doc['integer_no_doc_values'].get(0)" + - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "doc['integer_no_doc_values'].value" + - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "field('integer_no_doc_values').get(-1)" + - match: { hits.hits.0.fields.field.0: 134134566 } + - match: { hits.hits.1.fields.field.0: -1 } + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('integer_no_doc_values', -1)" + - match: { hits.hits.0.fields.field.0: 134134566 } + - match: { hits.hits.1.fields.field.0: -1 } + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "field('integer_no_doc_values').get(1, -3)" + - match: { hits.hits.0.fields.field.0: -3 } + - match: { hits.hits.1.fields.field.0: -3 } + - match: { hits.hits.2.fields.field.0: 17 } + + - do: + search: + rest_total_hits_as_int: true + body: + sort: [ { rank: asc } ] + script_fields: + field: + script: + source: "int total = 0; for (int i : field('integer_no_doc_values')) { total += i; } total + field('integer_no_doc_values').size();" + - match: { hits.hits.0.fields.field.0: 134134567 } + - match: { hits.hits.1.fields.field.0: 0 } + - match: { hits.hits.2.fields.field.0: 54 } + --- "short": - do: diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java index dc05af9d65387..8da5b3a06d7cc 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java @@ -135,6 +135,11 @@ public long ramBytesUsed() { public void close() { } + + @Override + public SortedBinaryDocValues getBytesValues() { + throw new IllegalArgumentException("not supported for source fallback"); + } } /** diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java index 2e0780c45b895..501430149a0ce 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedBinaryIndexFieldData.java @@ -77,16 +77,14 @@ public SourceValueFetcherSortedBinaryLeafFieldData( @Override public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { - return toScriptFieldFactory.getScriptFieldFactory(getBytesValues(), name); - } - - @Override - public SortedBinaryDocValues getBytesValues() { - return new ValueFetcherSortedBinaryDocValues(leafReaderContext, valueFetcher, sourceLookup); + return toScriptFieldFactory.getScriptFieldFactory( + new SourceValueFetcherSortedBinaryDocValues(leafReaderContext, valueFetcher, sourceLookup), + name + ); } } - public static class ValueFetcherSortedBinaryDocValues extends SortedBinaryDocValues implements ValueFetcherDocValues { + public static class SourceValueFetcherSortedBinaryDocValues extends SortedBinaryDocValues implements ValueFetcherDocValues { private final LeafReaderContext leafReaderContext; @@ -96,7 +94,7 @@ public static class ValueFetcherSortedBinaryDocValues extends SortedBinaryDocVal private SortedSet values; private Iterator iterator; - public ValueFetcherSortedBinaryDocValues( + public SourceValueFetcherSortedBinaryDocValues( LeafReaderContext leafReaderContext, ValueFetcher valueFetcher, SourceLookup sourceLookup diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java new file mode 100644 index 0000000000000..fbe0fb4da859f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java @@ -0,0 +1,151 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.ToScriptFieldFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.Collections; +import java.util.Iterator; +import java.util.TreeSet; + +public class SourceValueFetcherSortedNumericIndexFieldData extends SourceValueFetcherIndexFieldData { + + public static class Builder extends SourceValueFetcherIndexFieldData.Builder { + + public Builder( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory); + } + + @Override + public SourceValueFetcherSortedNumericIndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { + return new SourceValueFetcherSortedNumericIndexFieldData( + fieldName, + valuesSourceType, + valueFetcher, + sourceLookup, + toScriptFieldFactory + ); + } + } + + protected SourceValueFetcherSortedNumericIndexFieldData( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory); + } + + @Override + public SourceValueFetcherSortedNumericLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + return new SourceValueFetcherSortedNumericLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup); + } + + public static class SourceValueFetcherSortedNumericLeafFieldData extends SourceValueFetcherLeafFieldData { + + public SourceValueFetcherSortedNumericLeafFieldData( + ToScriptFieldFactory toScriptFieldFactory, + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceLookup); + } + + @Override + public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { + return toScriptFieldFactory.getScriptFieldFactory( + new SourceValueFetcherSortedNumericDocValues(leafReaderContext, valueFetcher, sourceLookup), + name + ); + } + } + + public static class SourceValueFetcherSortedNumericDocValues extends SortedNumericDocValues implements ValueFetcherDocValues { + + private final LeafReaderContext leafReaderContext; + + private final ValueFetcher valueFetcher; + private final SourceLookup sourceLookup; + + private TreeSet values; + private Iterator iterator; + + public SourceValueFetcherSortedNumericDocValues( + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + this.leafReaderContext = leafReaderContext; + this.valueFetcher = valueFetcher; + this.sourceLookup = sourceLookup; + } + + @Override + public boolean advanceExact(int doc) throws IOException { + sourceLookup.setSegmentAndDocument(leafReaderContext, doc); + values = new TreeSet<>(); + + for (Object value : valueFetcher.fetchValues(sourceLookup, Collections.emptyList())) { + values.add(((Number) value).longValue()); + } + + iterator = values.iterator(); + + return true; + } + + @Override + public int docValueCount() { + return values.size(); + } + + @Override + public long nextValue() throws IOException { + assert iterator.hasNext(); + return iterator.next(); + } + + @Override + public int docID() { + throw new UnsupportedOperationException("not supported for source fallback"); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException("not supported for source fallback"); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException("not supported for source fallback"); + } + + @Override + public long cost() { + throw new UnsupportedOperationException("not supported for source fallback"); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 0ee2e015061b6..5a9ee744c439b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -47,7 +47,6 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.SourceValueFetcherIndexFieldData; import org.elasticsearch.index.fielddata.SourceValueFetcherSortedBinaryIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; @@ -58,7 +57,6 @@ import org.elasticsearch.script.StringFieldScript; import org.elasticsearch.script.field.KeywordDocValuesField; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.runtime.StringScriptFieldFuzzyQuery; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 5e538ec258b28..d1090b3f69077 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -35,6 +35,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexNumericFieldData.NumericType; +import org.elasticsearch.index.fielddata.SourceValueFetcherSortedNumericIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedDoublesIndexFieldData; import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.mapper.TimeSeriesParams.MetricType; @@ -53,6 +54,7 @@ import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentParser.Token; @@ -918,6 +920,21 @@ public IndexFieldData.Builder getFieldDataBuilder(String name) { return new SortedNumericIndexFieldData.Builder(name, numericType(), IntegerDocValuesField::new); } + @Override + public IndexFieldData.Builder getValueFetcherFieldDataBuilder( + String name, + SourceLookup sourceLookup, + ValueFetcher valueFetcher + ) { + return new SourceValueFetcherSortedNumericIndexFieldData.Builder( + name, + numericType().getValuesSourceType(), + valueFetcher, + sourceLookup, + IntegerDocValuesField::new + ); + } + @Override SourceLoader.SyntheticFieldLoader syntheticFieldLoader(String fieldName, String fieldSimpleName) { return NumberType.syntheticLongFieldLoader(fieldName, fieldSimpleName); @@ -1253,6 +1270,10 @@ public static Query longRangeQuery( public abstract IndexFieldData.Builder getFieldDataBuilder(String name); + public IndexFieldData.Builder getValueFetcherFieldDataBuilder(String name, SourceLookup sourceLookup, ValueFetcher valueFetcher) { + throw new UnsupportedOperationException("not supported for source fallback"); + } + /** * Adjusts a value to the value it would have been had it been parsed by that mapper * and then cast up to a double. This is meant to be an entry point to manipulate values @@ -1407,6 +1428,27 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S return type.getFieldDataBuilder(name()); } + @Override + public IndexFieldData.Builder scriptFielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + if (hasDocValues()) { + return type.getFieldDataBuilder(name()); + } + + return type.getValueFetcherFieldDataBuilder( + name(), + searchLookup.get().source(), + new SourceValueFetcher(searchLookup.get().sourcePaths(name()), nullValue) { + @Override + protected Object parseSourceValue(Object value) { + if (value.equals("")) { + return nullValue; + } + return type.parse(value, coerce); + } + } + ); + } + @Override public Object valueForDisplay(Object value) { if (value == null) { From c91f730eb473c717198b648af3ab9bbb57e05157 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 18 Jul 2022 10:59:13 -0700 Subject: [PATCH 13/23] Add script source fallback for geopoint field. --- .../test/painless/50_script_doc_values.yml | 103 ++++++++++++++++ .../SourceValueFetcherIndexFieldData.java | 2 +- ...lueFetcherMultiGeoPointIndexFieldData.java | 115 ++++++++++++++++++ ...lueFetcherSortedNumericIndexFieldData.java | 10 +- .../mapper/AbstractGeometryFieldMapper.java | 13 ++ .../index/mapper/ArraySourceValueFetcher.java | 9 ++ .../index/mapper/GeoPointFieldMapper.java | 16 +++ 7 files changed, 262 insertions(+), 6 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml index b4f6fb9fd5c8a..4ba1b270cd995 100644 --- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml +++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/50_script_doc_values.yml @@ -15,6 +15,9 @@ setup: type: date_nanos geo_point: type: geo_point + geo_point_no_doc_values: + type: geo_point + doc_values: false ip: type: ip keyword: @@ -61,6 +64,7 @@ setup: date: 2017-01-01T12:11:12 nanos: 2015-01-01T12:10:30.123456789Z geo_point: 41.12,-71.34 + geo_point_no_doc_values: 41.12,-71.34 ip: 192.168.0.19 keyword: not split at all keyword_no_doc_values: no doc values @@ -93,6 +97,7 @@ setup: date: [2017-01-01T12:11:12, 2018-01-01T12:11:12] nanos: [2015-01-01T12:10:30.123456789Z, 2015-01-01T12:10:30.987654321Z] geo_point: [[-71.34,41.12],[60.32,21.25]] + geo_point_no_doc_values: [[60.32,21.25],[-71.34,41.12]] keyword: ["one string", "another string"] keyword_no_doc_values: ["no doc values 1", "no doc values 0", "no doc values 2"] long: [1152921504606846976, 576460752303423488] @@ -702,6 +707,104 @@ setup: - match: { hits.hits.0.fields.width.0: 0.0 } - match: { hits.hits.0.fields.height.0: 0.0 } +--- +"geo_point_no_doc_values": + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "doc['geo_point_no_doc_values'].get(0)" + - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + + - do: + catch: bad_request + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "doc['geo_point_no_doc_values'].value" + - match: { error.failed_shards.0.reason.caused_by.type: "illegal_argument_exception" } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "field('geo_point_no_doc_values').get(new GeoPoint())" + - match: { hits.hits.0.fields.field.0.lat: 41.1199999647215 } + - match: { hits.hits.0.fields.field.0.lon: -71.34000004269183 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "1" } } + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('geo_point_no_doc_values', new GeoPoint())" + - match: { hits.hits.0.fields.field.0.lat: 41.1199999647215 } + - match: { hits.hits.0.fields.field.0.lon: -71.34000004269183 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "3" } } + script_fields: + field: + script: + source: "field('geo_point_no_doc_values').get(new GeoPoint())" + - match: { hits.hits.0.fields.field.0.lat: 21.249999990686774 } + - match: { hits.hits.0.fields.field.0.lon: 60.319999968633056 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "3" } } + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('geo_point_no_doc_values', new GeoPoint())" + - match: { hits.hits.0.fields.field.0.lat: 21.249999990686774 } + - match: { hits.hits.0.fields.field.0.lon: 60.319999968633056 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "3" } } + script_fields: + field: + script: + source: "field('geo_point_no_doc_values').get(1, new GeoPoint())" + - match: { hits.hits.0.fields.field.0.lat: 41.1199999647215 } + - match: { hits.hits.0.fields.field.0.lon: -71.34000004269183 } + + - do: + search: + rest_total_hits_as_int: true + body: + query: { term: { _id: "2" } } + script_fields: + field: + script: + source: "/* avoid yaml stash */ $('geo_point_no_doc_values', new GeoPoint(1.0, 2.0))" + - match: { hits.hits.0.fields.field.0.lat: 1.0 } + - match: { hits.hits.0.fields.field.0.lon: 2.0 } + --- "ip": - do: diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java index 8da5b3a06d7cc..8d24977a9c22f 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherIndexFieldData.java @@ -25,7 +25,7 @@ public abstract class SourceValueFetcherIndexFieldData implements IndexFieldData> { - public static abstract class Builder implements IndexFieldData.Builder { + public abstract static class Builder implements IndexFieldData.Builder { protected final String fieldName; protected final ValuesSourceType valuesSourceType; diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java new file mode 100644 index 0000000000000..9c54be22a4f27 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherMultiGeoPointIndexFieldData.java @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.script.field.DocValuesScriptFieldFactory; +import org.elasticsearch.script.field.ToScriptFieldFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeSet; + +public class SourceValueFetcherMultiGeoPointIndexFieldData extends SourceValueFetcherIndexFieldData { + + public static class Builder extends SourceValueFetcherIndexFieldData.Builder { + + public Builder( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory); + } + + @Override + public SourceValueFetcherMultiGeoPointIndexFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService) { + return new SourceValueFetcherMultiGeoPointIndexFieldData( + fieldName, + valuesSourceType, + valueFetcher, + sourceLookup, + toScriptFieldFactory + ); + } + } + + protected SourceValueFetcherMultiGeoPointIndexFieldData( + String fieldName, + ValuesSourceType valuesSourceType, + ValueFetcher valueFetcher, + SourceLookup sourceLookup, + ToScriptFieldFactory toScriptFieldFactory + ) { + super(fieldName, valuesSourceType, valueFetcher, sourceLookup, toScriptFieldFactory); + } + + @Override + public SourceValueFetcherMultiGeoPointLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + return new SourceValueFetcherMultiGeoPointLeafFieldData(toScriptFieldFactory, context, valueFetcher, sourceLookup); + } + + public static class SourceValueFetcherMultiGeoPointLeafFieldData extends + SourceValueFetcherIndexFieldData.SourceValueFetcherLeafFieldData { + + public SourceValueFetcherMultiGeoPointLeafFieldData( + ToScriptFieldFactory toScriptFieldFactory, + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + super(toScriptFieldFactory, leafReaderContext, valueFetcher, sourceLookup); + } + + @Override + public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { + return toScriptFieldFactory.getScriptFieldFactory( + new MultiGeoPointValues(new SourceValueFetcherMultiGeoPointDocValues(leafReaderContext, valueFetcher, sourceLookup)), + name + ); + } + } + + public static class SourceValueFetcherMultiGeoPointDocValues extends + SourceValueFetcherSortedNumericIndexFieldData.SourceValueFetcherSortedNumericDocValues { + + public SourceValueFetcherMultiGeoPointDocValues( + LeafReaderContext leafReaderContext, + ValueFetcher valueFetcher, + SourceLookup sourceLookup + ) { + super(leafReaderContext, valueFetcher, sourceLookup); + } + + @Override + @SuppressWarnings("unchecked") + public boolean advanceExact(int doc) throws IOException { + sourceLookup.setSegmentAndDocument(leafReaderContext, doc); + values = new TreeSet<>(); + + for (Object value : valueFetcher.fetchValues(sourceLookup, Collections.emptyList())) { + List coordinates = ((Map>) value).get("coordinates"); + values.add(new GeoPoint((double) coordinates.get(1), (double) coordinates.get(0)).getEncoded()); + } + + iterator = values.iterator(); + + return true; + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java index fbe0fb4da859f..671c547155cdd 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SourceValueFetcherSortedNumericIndexFieldData.java @@ -85,13 +85,13 @@ public DocValuesScriptFieldFactory getScriptFieldFactory(String name) { public static class SourceValueFetcherSortedNumericDocValues extends SortedNumericDocValues implements ValueFetcherDocValues { - private final LeafReaderContext leafReaderContext; + protected final LeafReaderContext leafReaderContext; - private final ValueFetcher valueFetcher; - private final SourceLookup sourceLookup; + protected final ValueFetcher valueFetcher; + protected final SourceLookup sourceLookup; - private TreeSet values; - private Iterator iterator; + protected TreeSet values; + protected Iterator iterator; public SourceValueFetcherSortedNumericDocValues( LeafReaderContext leafReaderContext, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java index 04a6d62576073..87b6abb217bc9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractGeometryFieldMapper.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; @@ -123,6 +124,18 @@ protected Object parseSourceValue(Object value) { } }; } + + public ValueFetcher valueFetcher(Set sourcePaths, Object nullValue, String format) { + Function, List> formatter = getFormatter(format != null ? format : GeometryFormatterFactory.GEOJSON); + return new ArraySourceValueFetcher(sourcePaths, nullValue) { + @Override + protected Object parseSourceValue(Object value) { + final List values = new ArrayList<>(); + geometryParser.fetchFromSource(value, values::add); + return formatter.apply(values); + } + }; + } } private final Explicit ignoreMalformed; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java index 809dc8d3e3e04..6d138250ca3b6 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java @@ -43,6 +43,15 @@ public ArraySourceValueFetcher(String fieldName, SearchExecutionContext context, this.nullValue = nullValue; } + /** + * @param sourcePaths The paths to pull source values from + * @param nullValue An optional substitute value if the _source value is `null` + */ + public ArraySourceValueFetcher(Set sourcePaths, Object nullValue) { + this.sourcePaths = sourcePaths; + this.nullValue = nullValue; + } + @Override public List fetchValues(SourceLookup lookup, List ignoredValues) { List values = new ArrayList<>(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 4e639b6db9bae..f2ebe98ff2f01 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -29,6 +29,7 @@ import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.geometry.Point; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.SourceValueFetcherMultiGeoPointIndexFieldData; import org.elasticsearch.index.fielddata.plain.AbstractLatLonPointIndexFieldData; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.script.GeoPointFieldScript; @@ -373,6 +374,21 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S return new AbstractLatLonPointIndexFieldData.Builder(name(), CoreValuesSourceType.GEOPOINT, GeoPointDocValuesField::new); } + @Override + public IndexFieldData.Builder scriptFielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + if (hasDocValues()) { + return fielddataBuilder(fullyQualifiedIndexName, searchLookup); + } + + return new SourceValueFetcherMultiGeoPointIndexFieldData.Builder( + name(), + CoreValuesSourceType.GEOPOINT, + valueFetcher(searchLookup.get().sourcePaths(name()), null, null), + searchLookup.get().source(), + GeoPointDocValuesField::new + ); + } + @Override public Query distanceFeatureQuery(Object origin, String pivot, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); From 32d8acef465bfa5516ebb50b4dea2ec7417d94b3 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Thu, 21 Jul 2022 13:24:37 -0700 Subject: [PATCH 14/23] add fielddatatype --- .../script/ScriptScoreBenchmark.java | 5 +- .../search/QueryParserHelperBenchmark.java | 4 +- .../AggConstructionContentionBenchmark.java | 3 +- .../expression/ExpressionScriptEngine.java | 2 +- .../ExpressionFieldScriptTests.java | 3 +- .../ExpressionNumberSortScriptTests.java | 3 +- .../ExpressionTermsSetQueryTests.java | 3 +- .../extras/MatchOnlyTextFieldMapper.java | 7 +- .../mapper/extras/RankFeatureFieldMapper.java | 6 +- .../extras/RankFeaturesFieldMapper.java | 6 +- .../mapper/extras/ScaledFloatFieldMapper.java | 7 +- .../mapper/extras/TokenCountFieldMapper.java | 2 +- .../extras/ScaledFloatFieldTypeTests.java | 10 ++- .../join/mapper/ParentIdFieldMapper.java | 7 +- .../join/mapper/ParentJoinFieldMapper.java | 6 +- .../join/query/HasChildQueryBuilder.java | 2 +- .../join/query/HasParentQueryBuilder.java | 2 +- .../percolator/PercolateQueryBuilder.java | 8 ++- .../percolator/QueryBuilderStoreTests.java | 4 +- .../icu/ICUCollationKeywordFieldMapper.java | 6 +- .../mapper/murmur3/Murmur3FieldMapper.java | 6 +- .../index/mapper/size/SizeFieldMapper.java | 2 +- .../java/org/elasticsearch/QuadFunction.java | 34 ++++++++++ .../org/elasticsearch/index/IndexService.java | 8 ++- .../elasticsearch/index/IndexSortConfig.java | 7 +- .../org/elasticsearch/index/IndexWarmer.java | 3 +- .../fielddata/IndexFieldDataService.java | 52 +++++---------- .../index/mapper/AbstractScriptFieldType.java | 2 +- .../index/mapper/BinaryFieldMapper.java | 6 +- .../index/mapper/BooleanFieldMapper.java | 6 +- .../index/mapper/BooleanScriptFieldType.java | 6 +- .../index/mapper/DateFieldMapper.java | 6 +- .../index/mapper/DateScriptFieldType.java | 2 +- .../index/mapper/DocumentParser.java | 4 +- .../index/mapper/DoubleScriptFieldType.java | 6 +- .../index/mapper/GeoPointFieldMapper.java | 36 +++++----- .../index/mapper/GeoPointScriptFieldType.java | 6 +- .../index/mapper/IndexFieldMapper.java | 6 +- .../index/mapper/IpFieldMapper.java | 6 +- .../index/mapper/IpScriptFieldType.java | 6 +- .../index/mapper/KeywordFieldMapper.java | 60 +++++++++-------- .../index/mapper/KeywordScriptFieldType.java | 6 +- .../index/mapper/LongScriptFieldType.java | 6 +- .../index/mapper/MappedFieldType.java | 31 ++++----- .../index/mapper/NumberFieldMapper.java | 44 +++++++------ .../index/mapper/PlaceHolderFieldMapper.java | 6 +- .../index/mapper/ProvidedIdFieldMapper.java | 6 +- .../index/mapper/RangeFieldMapper.java | 6 +- .../index/mapper/SeqNoFieldMapper.java | 6 +- .../index/mapper/TextFieldMapper.java | 6 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 8 ++- .../mapper/TsidExtractingIdFieldMapper.java | 6 +- .../index/mapper/VersionFieldMapper.java | 8 ++- .../flattened/FlattenedFieldMapper.java | 12 +++- .../vectors/DenseVectorFieldMapper.java | 6 +- .../index/query/SearchExecutionContext.java | 28 ++++---- .../index/query/TermsSetQueryBuilder.java | 2 +- .../functionscore/DecayFunctionBuilder.java | 6 +- .../FieldValueFactorFunctionBuilder.java | 3 +- .../RandomScoreFunctionBuilder.java | 7 +- .../support/AggregationContext.java | 2 +- .../fetch/subphase/FetchDocValuesPhase.java | 2 +- .../search/lookup/LeafDocLookup.java | 18 +++-- .../search/lookup/SearchLookup.java | 20 ++---- .../search/slice/SliceBuilder.java | 2 +- .../search/sort/FieldSortBuilder.java | 4 +- .../search/sort/GeoDistanceSortBuilder.java | 2 +- .../MetadataCreateIndexServiceTests.java | 1 - .../index/IndexSortSettingsTests.java | 8 ++- .../fielddata/AbstractFieldDataTestCase.java | 2 +- .../index/fielddata/FilterFieldDataTests.java | 8 +-- .../fielddata/IndexFieldDataServiceTests.java | 66 +++++++++++++++---- .../AbstractScriptFieldTypeTestCase.java | 3 +- .../index/mapper/BooleanFieldScriptTests.java | 2 +- .../mapper/BooleanScriptFieldTypeTests.java | 9 ++- .../mapper/CompositeRuntimeFieldTests.java | 3 +- .../index/mapper/DateFieldScriptTests.java | 4 +- .../index/mapper/DateFieldTypeTests.java | 3 - .../mapper/DateScriptFieldTypeTests.java | 9 ++- .../index/mapper/DoubleFieldScriptTests.java | 4 +- .../mapper/DoubleScriptFieldTypeTests.java | 9 ++- .../mapper/FieldNamesFieldTypeTests.java | 1 - .../mapper/GeoPointFieldScriptTests.java | 2 +- .../mapper/GeoPointScriptFieldTypeTests.java | 9 ++- .../index/mapper/IgnoredFieldMapperTests.java | 1 - .../index/mapper/IndexFieldMapperTests.java | 7 +- .../index/mapper/IndexFieldTypeTests.java | 1 - .../index/mapper/IpFieldScriptTests.java | 4 +- .../index/mapper/IpScriptFieldTypeTests.java | 9 ++- .../mapper/KeywordScriptFieldTypeTests.java | 9 ++- .../index/mapper/LongFieldScriptTests.java | 4 +- .../mapper/LongScriptFieldTypeTests.java | 15 ++++- .../index/mapper/NumberFieldTypeTests.java | 4 +- .../mapper/PlaceHolderFieldMapperTests.java | 7 +- .../mapper/ProvidedIdFieldMapperTests.java | 7 +- .../index/mapper/RangeFieldTypeTests.java | 1 - .../index/mapper/RoutingFieldMapperTests.java | 1 - .../index/mapper/StringFieldScriptTests.java | 8 +-- .../index/mapper/TextFieldMapperTests.java | 10 +-- .../index/mapper/VersionFieldMapperTests.java | 7 +- .../FlattenedIndexFieldDataTests.java | 6 +- .../vectors/DenseVectorFieldTypeTests.java | 5 +- .../vectors/SparseVectorFieldTypeTests.java | 2 +- .../index/query/RangeQueryRewriteTests.java | 3 - .../query/SearchExecutionContextTests.java | 20 ++++-- .../index/shard/IndexShardTests.java | 3 +- .../script/CompositeFieldScriptTests.java | 6 +- ...dNumericDocValuesLongFieldScriptTests.java | 2 +- ...tedSetDocValuesStringFieldScriptTests.java | 2 +- .../bucket/range/RangeAggregatorTests.java | 2 +- .../bucket/terms/TermsAggregatorTests.java | 2 +- .../support/ScriptValuesTests.java | 2 +- .../fetch/subphase/FieldFetcherTests.java | 11 ++-- .../highlight/HighlightBuilderTests.java | 1 - .../search/lookup/LeafDocLookupTests.java | 14 ++-- .../search/query/ScriptScoreQueryTests.java | 2 +- .../rescore/QueryRescorerBuilderTests.java | 2 - ...tScriptFieldDistanceFeatureQueryTests.java | 2 +- ...gScriptFieldDistanceFeatureQueryTests.java | 2 +- .../search/slice/SliceBuilderTests.java | 2 +- .../search/sort/AbstractSortTestCase.java | 26 +++----- .../AbstractSuggestionBuilderTestCase.java | 2 - .../index/mapper/MapperServiceTestCase.java | 18 ++--- .../index/mapper/MapperTestCase.java | 23 ++++--- .../aggregations/AggregatorTestCase.java | 17 ++--- .../test/AbstractBuilderTestCase.java | 1 - .../mapper/HistogramFieldMapper.java | 6 +- .../mapper/DataTierFieldTypeTests.java | 2 - .../DocumentSubsetBitsetCacheTests.java | 1 - ...ityIndexReaderWrapperIntegrationTests.java | 2 - .../AggregateDoubleMetricFieldMapper.java | 6 +- .../AggregateDoubleMetricFieldTypeTests.java | 3 +- .../mapper/ConstantKeywordFieldMapper.java | 6 +- .../unsignedlong/UnsignedLongFieldMapper.java | 6 +- .../VersionStringFieldMapper.java | 6 +- .../xpack/rollup/v2/FieldValueFetcher.java | 2 +- .../job/RollupIndexerIndexingTests.java | 1 - .../rollup/v2/MetricFieldProducerTests.java | 1 - .../GeoShapeWithDocValuesFieldMapper.java | 6 +- .../wildcard/mapper/WildcardFieldMapper.java | 6 +- .../mapper/WildcardFieldMapperTests.java | 26 +++----- 141 files changed, 671 insertions(+), 444 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/QuadFunction.java diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index 86e1fa292131d..9587587992cdd 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -87,8 +87,7 @@ public class ScriptScoreBenchmark { private final CircuitBreakerService breakerService = new NoneCircuitBreakerService(); private SearchLookup lookup = new SearchLookup( fieldTypes::get, - (mft, lookup) -> mft.fielddataBuilder("test", lookup).build(fieldDataCache, breakerService), - (mft, lookup) -> mft.scriptFielddataBuilder("test", lookup).build(fieldDataCache, breakerService), + (mft, lookup, fdt) -> mft.fielddataBuilder("test", lookup, fdt).build(fieldDataCache, breakerService), sourcePaths::get ); @@ -155,7 +154,7 @@ private Query scriptScoreQuery(ScoreScript.Factory factory) { private ScoreScript.Factory bareMetalScript() { return (params, lookup) -> { MappedFieldType type = fieldTypes.get("n"); - IndexNumericFieldData ifd = (IndexNumericFieldData) lookup.getForField(type); + IndexNumericFieldData ifd = (IndexNumericFieldData) lookup.getForField(type, MappedFieldType.FielddataType.SEARCH); return new ScoreScript.LeafFactory() { @Override public ScoreScript newInstance(DocReader docReader) throws IOException { diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java index 28fab7639a99f..160d4265ae4ed 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java @@ -141,9 +141,7 @@ protected SearchExecutionContext buildSearchExecutionContext() { 0, mapperService.getIndexSettings(), null, - (ft, idxName, lookup) -> ft.fielddataBuilder(idxName, lookup) - .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), - (ft, idxName, lookup) -> ft.scriptFielddataBuilder(idxName, lookup) + (ft, idxName, lookup, fdt) -> ft.fielddataBuilder(idxName, lookup, fdt) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), mapperService, mapperService.mappingLookup(), diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java index d573015bc8f4c..f118aea01b582 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java @@ -210,7 +210,8 @@ public Analyzer buildCustomAnalyzer( protected IndexFieldData buildFieldData(MappedFieldType ft) { IndexFieldDataCache indexFieldDataCache = indicesFieldDataCache.buildIndexFieldDataCache(new IndexFieldDataCache.Listener() { }, index, ft.name()); - return ft.fielddataBuilder("test", this::lookup).build(indexFieldDataCache, breakerService); + return ft.fielddataBuilder("test", this::lookup, MappedFieldType.FielddataType.SEARCH) + .build(indexFieldDataCache, breakerService); } @Override diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java index 0d2f71ade6e51..5fe8294d8d7e0 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java @@ -460,7 +460,7 @@ private static DoubleValuesSource getDocValueSource(String variable, SearchLooku throw new ParseException("Field [" + fieldname + "] does not exist in mappings", 5); } - IndexFieldData fieldData = lookup.getForField(fieldType); + IndexFieldData fieldData = lookup.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); final DoubleValuesSource valueSource; if (fieldType instanceof GeoPointFieldType) { // geo diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java index ff0ec76bfb6a1..80d99604a657e 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionFieldScriptTests.java @@ -52,8 +52,7 @@ public void setUp() throws Exception { service = new ExpressionScriptEngine(); lookup = new SearchLookup( field -> field.equals("field") ? fieldType : null, - (ignored, _lookup) -> fieldData, - (ignored, _lookup) -> fieldData, + (ignored, _lookup, fdt) -> fieldData, field -> field.equals("field") ? Set.of("field") : null ); } diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java index 4418d86947a44..ee2799eb21177 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionNumberSortScriptTests.java @@ -53,8 +53,7 @@ public void setUp() throws Exception { service = new ExpressionScriptEngine(); lookup = new SearchLookup( field -> field.equals("field") ? fieldType : null, - (ignored, _lookup) -> fieldData, - (ignored, _lookup) -> fieldData, + (ignored, _lookup, fdt) -> fieldData, field -> field.equals("field") ? Set.of("field") : null ); } diff --git a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java index 210d1197499f8..878de6f753b42 100644 --- a/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java +++ b/modules/lang-expression/src/test/java/org/elasticsearch/script/expression/ExpressionTermsSetQueryTests.java @@ -52,8 +52,7 @@ public void setUp() throws Exception { service = new ExpressionScriptEngine(); lookup = new SearchLookup( field -> field.equals("field") ? fieldType : null, - (ignored, _lookup) -> fieldData, - (ignored, _lookup) -> fieldData, + (ignored, _lookup, fdt) -> fieldData, field -> field.equals("field") ? Set.of("field") : null ); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 39456154fa6b5..1a522f35fc1fa 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -34,6 +34,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; @@ -276,7 +277,11 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { throw new IllegalArgumentException(CONTENT_TYPE + " fields do not support sorting and aggregations"); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java index f4db702a09e47..98adfa1cb8d2d 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java @@ -111,7 +111,11 @@ public Query existsQuery(SearchExecutionContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { throw new IllegalArgumentException("[rank_feature] fields do not support sorting, scripting or aggregating"); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java index 1c2f31d1d68bb..c83db91040b1d 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java @@ -97,7 +97,11 @@ public Query existsQuery(SearchExecutionContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { throw new IllegalArgumentException("[rank_features] fields do not support sorting, scripting or aggregating"); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 09ac5a21ab725..16354a74c2e58 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.SimpleMappedFieldType; @@ -276,7 +277,11 @@ public Query rangeQuery( } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { failIfNoDocValues(); return (cache, breakerService) -> { final IndexNumericFieldData scaledValues = new SortedNumericIndexFieldData.Builder( diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java index f5c69329e3599..ab746121045c4 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java @@ -122,7 +122,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (hasDocValues() == false) { return (lookup, ignoredValues) -> List.of(); } - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataType.SEARCH)); } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java index 7f1094280bbda..4593807dbe3e6 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java @@ -191,7 +191,8 @@ public void testFieldData() throws IOException { ); IndexNumericFieldData fielddata = (IndexNumericFieldData) f1.fielddataBuilder( "index", - () -> { throw new UnsupportedOperationException(); } + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataType.SEARCH ).build(null, null); assertEquals(fielddata.getNumericType(), IndexNumericFieldData.NumericType.DOUBLE); LeafNumericFieldData leafFieldData = fielddata.load(reader.leaves().get(0)); @@ -205,8 +206,11 @@ public void testFieldData() throws IOException { "scaled_float2", scalingFactor ); - fielddata = (IndexNumericFieldData) f2.fielddataBuilder("index", () -> { throw new UnsupportedOperationException(); }) - .build(null, null); + fielddata = (IndexNumericFieldData) f2.fielddataBuilder( + "index", + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataType.SEARCH + ).build(null, null); leafFieldData = fielddata.load(reader.leaves().get(0)); values = leafFieldData.getDoubleValues(); assertTrue(values.advanceExact(0)); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index be7bc800cc3e2..95a7df794bf58 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -21,6 +21,7 @@ import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; @@ -72,7 +73,11 @@ public boolean eagerGlobalOrdinals() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder( name(), diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 9ac5ee4938f83..50a345cc6e4d5 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -167,7 +167,11 @@ public String typeName() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { return new SortedSetOrdinalsIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java index d877db6a7ff3c..e58748363a31c 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java @@ -344,7 +344,7 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { Query childFilter = joiner.filter(type); Query filteredQuery = Queries.filtered(query.toQuery(context), childFilter); MappedFieldType ft = context.getFieldType(parentJoinField); - final SortedSetOrdinalsIndexFieldData fieldData = context.getForField(ft); + final SortedSetOrdinalsIndexFieldData fieldData = context.getForField(ft, MappedFieldType.FielddataType.SEARCH); return new LateParsingQuery( parentFilter, filteredQuery, diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java index 64c3e627df75f..36fb6f6caae66 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java @@ -181,7 +181,7 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { Query innerQuery = Queries.filtered(query.toQuery(context), parentFilter); Query childFilter = joiner.childrenFilter(parentType); MappedFieldType fieldType = context.getFieldType(joiner.childJoinField(parentType)); - final SortedSetOrdinalsIndexFieldData fieldData = context.getForField(fieldType); + final SortedSetOrdinalsIndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); return new HasChildQueryBuilder.LateParsingQuery( childFilter, innerQuery, diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index ee8e3370ef6cd..0af9dad54dd67 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -640,8 +640,12 @@ public BitSetProducer bitsetFilter(Query query) { @Override @SuppressWarnings("unchecked") - public > IFD getForField(MappedFieldType fieldType) { - IndexFieldData.Builder builder = fieldType.fielddataBuilder(delegate.getFullyQualifiedIndex().getName(), delegate::lookup); + public > IFD getForField(MappedFieldType fieldType, MappedFieldType.FielddataType fielddataType) { + IndexFieldData.Builder builder = fieldType.fielddataBuilder( + delegate.getFullyQualifiedIndex().getName(), + delegate::lookup, + fielddataType + ); IndexFieldDataCache cache = new IndexFieldDataCache.None(); CircuitBreakerService circuitBreaker = new NoneCircuitBreakerService(); return (IFD) builder.build(cache, circuitBreaker); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java index e2d3ffe259e47..5accca81584a1 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.index.mapper.BinaryFieldMapper; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.TestDocumentParserContext; import org.elasticsearch.index.query.SearchExecutionContext; @@ -61,6 +62,7 @@ public void testStoringQueryBuilders() throws IOException { IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer()); config.setMergePolicy(NoMergePolicy.INSTANCE); BinaryFieldMapper fieldMapper = PercolatorFieldMapper.Builder.createQueryBuilderFieldBuilder(MapperBuilderContext.ROOT); + MappedFieldType.FielddataType fielddataType = MappedFieldType.FielddataType.SEARCH; Version version = Version.CURRENT; try (IndexWriter indexWriter = new IndexWriter(directory, config)) { @@ -76,7 +78,7 @@ public void testStoringQueryBuilders() throws IOException { when(searchExecutionContext.indexVersionCreated()).thenReturn(version); when(searchExecutionContext.getWriteableRegistry()).thenReturn(writableRegistry()); when(searchExecutionContext.getParserConfig()).thenReturn(parserConfig()); - when(searchExecutionContext.getForField(fieldMapper.fieldType())).thenReturn( + when(searchExecutionContext.getForField(fieldMapper.fieldType(), fielddataType)).thenReturn( new BytesBinaryIndexFieldData(fieldMapper.name(), CoreValuesSourceType.KEYWORD) ); when(searchExecutionContext.getFieldType(Mockito.anyString())).thenAnswer(invocation -> { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index 94b6e32995688..bca89333b0a14 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -108,7 +108,11 @@ protected String parseSourceValue(Object value) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder( name(), diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index c33aad0c3d42c..e89ef33fc8fb6 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -89,7 +89,11 @@ public String typeName() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, Murmur3DocValueField::new); } diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index 385efcb0ebaee..6cd3946a01308 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -57,7 +57,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (hasDocValues() == false) { return (lookup, ignoredValues) -> List.of(); } - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataType.SEARCH)); } } diff --git a/server/src/main/java/org/elasticsearch/QuadFunction.java b/server/src/main/java/org/elasticsearch/QuadFunction.java new file mode 100644 index 0000000000000..985a121f9ccf5 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/QuadFunction.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + * + */ + +package org.elasticsearch.common; + +/** + * Represents a function that accepts three arguments and produces a result. + * + * @param the type of the first argument + * @param the type of the second argument + * @param the type of the third argument + * @param the type of the fourth argument + * + * @param the return type + */ +@FunctionalInterface +public interface QuadFunction { + /** + * Applies this function to the given arguments. + * + * @param s the first function argument + * @param t the second function argument + * @param u the third function argument + * @param v the fourth function argument + * @return the result + */ + R apply(S s, T t, U u, V v); +} diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 7ff9d55dd582d..804833eddb047 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -205,7 +205,12 @@ public IndexService( this.indexSortSupplier = () -> indexSettings.getIndexSortConfig() .buildIndexSort( mapperService::fieldType, - (fieldType, searchLookup) -> indexFieldData.getForField(fieldType, indexFieldData.index().getName(), searchLookup) + (fieldType, searchLookup, fielddataType) -> indexFieldData.getForField( + fieldType, + indexFieldData.index().getName(), + searchLookup, + fielddataType + ) ); } else { this.indexSortSupplier = () -> null; @@ -621,7 +626,6 @@ public SearchExecutionContext newSearchExecutionContext( indexSettings, indexCache.bitsetFilterCache(), indexFieldData::getForField, - indexFieldData::getForScriptField, mapperService(), mapperService().mappingLookup(), similarityService(), diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index 0cf09d0d49f77..578fdfb67c92d 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSortField; import org.elasticsearch.Version; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Setting; @@ -29,7 +30,6 @@ import java.util.EnumSet; import java.util.List; import java.util.Objects; -import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -205,7 +205,7 @@ public boolean hasPrimarySortOnField(String field) { */ public Sort buildIndexSort( Function fieldTypeLookup, - BiFunction, IndexFieldData> fieldDataLookup + TriFunction, MappedFieldType.FielddataType, IndexFieldData> fieldDataLookup ) { if (hasIndexSort() == false) { return null; @@ -249,7 +249,8 @@ public Sort buildIndexSort( try { fieldData = fieldDataLookup.apply( ft, - () -> { throw new UnsupportedOperationException("index sorting not supported on runtime field [" + ft.name() + "]"); } + () -> { throw new UnsupportedOperationException("index sorting not supported on runtime field [" + ft.name() + "]"); }, + MappedFieldType.FielddataType.SEARCH ); } catch (Exception e) { throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]", e); diff --git a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java index 48d1037df9e6b..098779aa81f8d 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java +++ b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java @@ -119,7 +119,8 @@ public TerminationHandle warmReader(final IndexShard indexShard, final Elasticse IndexFieldData.Global ifd = indexFieldDataService.getForField( fieldType, indexFieldDataService.index().getName(), - () -> { throw new UnsupportedOperationException("search lookup not available when warming an index"); } + () -> { throw new UnsupportedOperationException("search lookup not available when warming an index"); }, + MappedFieldType.FielddataType.SEARCH ); IndexFieldData global = ifd.loadGlobal(reader); if (reader.leaves().isEmpty() == false) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index 72490bfd50765..30be51a0069b6 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -101,45 +101,29 @@ public synchronized void clearField(final String fieldName) { public > IFD getForField( MappedFieldType fieldType, String fullyQualifiedIndexName, - Supplier searchLookup + Supplier searchLookup, + MappedFieldType.FielddataType fielddataType ) { final String fieldName = fieldType.name(); - IndexFieldData.Builder builder = fieldType.fielddataBuilder(fullyQualifiedIndexName, searchLookup); - IndexFieldDataCache cache = getFieldDataCache(fieldName); - return (IFD) builder.build(cache, circuitBreakerService); - } - - /** - * Returns fielddata for the provided script field type, given the provided fully qualified index name, - * while also making a {@link SearchLookup} supplier available that is required for runtime fields. - */ - @SuppressWarnings("unchecked") - public > IFD getForScriptField( - MappedFieldType fieldType, - String fullyQualifiedIndexName, - Supplier searchLookup - ) { - final String fieldName = fieldType.name(); - IndexFieldData.Builder builder = fieldType.scriptFielddataBuilder(fullyQualifiedIndexName, searchLookup); - IndexFieldDataCache cache = getFieldDataCache(fieldName); - return (IFD) builder.build(cache, circuitBreakerService); - } - - private synchronized IndexFieldDataCache getFieldDataCache(String fieldName) { - IndexFieldDataCache cache = fieldDataCaches.get(fieldName); - if (cache == null) { - String cacheType = indexSettings.getValue(INDEX_FIELDDATA_CACHE_KEY); - if (FIELDDATA_CACHE_VALUE_NODE.equals(cacheType)) { - cache = indicesFieldDataCache.buildIndexFieldDataCache(listener, index(), fieldName); - } else if ("none".equals(cacheType)) { - cache = new IndexFieldDataCache.None(); - } else { - throw new IllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldName + "]"); + IndexFieldData.Builder builder = fieldType.fielddataBuilder(fullyQualifiedIndexName, searchLookup, fielddataType); + + IndexFieldDataCache cache; + synchronized (this) { + cache = fieldDataCaches.get(fieldName); + if (cache == null) { + String cacheType = indexSettings.getValue(INDEX_FIELDDATA_CACHE_KEY); + if (FIELDDATA_CACHE_VALUE_NODE.equals(cacheType)) { + cache = indicesFieldDataCache.buildIndexFieldDataCache(listener, index(), fieldName); + } else if ("none".equals(cacheType)) { + cache = new IndexFieldDataCache.None(); + } else { + throw new IllegalArgumentException("cache type not supported [" + cacheType + "] for field [" + fieldName + "]"); + } + fieldDataCaches.put(fieldName, cache); } - fieldDataCaches.put(fieldName, cache); } - return cache; + return (IFD) builder.build(cache, circuitBreakerService); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java index 66d1390ea59ff..da383ddbd3902 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java @@ -173,7 +173,7 @@ protected final void applyScriptContext(SearchExecutionContext context) { @Override public final ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataType.SEARCH)); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index eb6a1d611b11d..ac189e2f00d6e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -121,7 +121,11 @@ public BytesReference valueForDisplay(Object value) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { failIfNoDocValues(); return new BytesBinaryIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 9736898a22d09..9808204c2f247 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -254,7 +254,11 @@ public Boolean valueForDisplay(Object value) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.BOOLEAN, BooleanDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java index 2e837d0f5d54f..7dd93a01c4b32 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java @@ -102,7 +102,11 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { } @Override - public BooleanScriptFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public BooleanScriptFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { return new BooleanScriptFieldData.Builder(name(), leafFactory(searchLookup.get()), BooleanDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 666a6032aa048..47e7b8f1b23a5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -749,7 +749,11 @@ public Function pointReaderIfPossible() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), resolution.numericType(), resolution.getDefaultToScriptFieldFactory()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java index 7bd9ce529fe59..6afea7fca5b22 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java @@ -159,7 +159,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { } @Override - public DateScriptFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier lookup) { + public DateScriptFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier lookup, FielddataType type) { return new DateScriptFieldData.Builder(name(), leafFactory(lookup.get()), Resolution.MILLISECONDS.getDefaultToScriptFieldFactory()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 5649a638decf9..0e8656f3a880f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -141,9 +141,7 @@ private static void executeIndexTimeScripts(DocumentParserContext context) { } SearchLookup searchLookup = new SearchLookup( context.mappingLookup().indexTimeLookup()::get, - (ft, lookup) -> ft.fielddataBuilder(context.indexSettings().getIndex().getName(), lookup) - .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), - (ft, lookup) -> ft.scriptFielddataBuilder(context.indexSettings().getIndex().getName(), lookup) + (ft, lookup, ftd) -> ft.fielddataBuilder(context.indexSettings().getIndex().getName(), lookup, ftd) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), context.mappingLookup()::sourcePaths ); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java index a542a299df73e..24ccd51f90d1e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java @@ -97,7 +97,11 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { } @Override - public DoubleScriptFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public DoubleScriptFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { return new DoubleScriptFieldData.Builder(name(), leafFactory(searchLookup.get()), DoubleDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index f2ebe98ff2f01..6379053d6aab8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -369,24 +369,30 @@ private boolean isPointGeometry(LatLonGeometry[] geometries) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - failIfNoDocValues(); - return new AbstractLatLonPointIndexFieldData.Builder(name(), CoreValuesSourceType.GEOPOINT, GeoPointDocValuesField::new); - } + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { + if (type == FielddataType.SEARCH) { + failIfNoDocValues(); + } - @Override - public IndexFieldData.Builder scriptFielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - if (hasDocValues()) { - return fielddataBuilder(fullyQualifiedIndexName, searchLookup); + if ((type == FielddataType.SEARCH || type == FielddataType.SCRIPT) && hasDocValues()) { + return new AbstractLatLonPointIndexFieldData.Builder(name(), CoreValuesSourceType.GEOPOINT, GeoPointDocValuesField::new); } - return new SourceValueFetcherMultiGeoPointIndexFieldData.Builder( - name(), - CoreValuesSourceType.GEOPOINT, - valueFetcher(searchLookup.get().sourcePaths(name()), null, null), - searchLookup.get().source(), - GeoPointDocValuesField::new - ); + if (type == FielddataType.SCRIPT) { + return new SourceValueFetcherMultiGeoPointIndexFieldData.Builder( + name(), + CoreValuesSourceType.GEOPOINT, + valueFetcher(searchLookup.get().sourcePaths(name()), null, null), + searchLookup.get().source(), + GeoPointDocValuesField::new + ); + } + + throw new IllegalStateException("unknown field data type [" + type.name() + "]"); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java index e6a5bcc82d86c..0d73b3ba77d2f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java @@ -94,7 +94,11 @@ public Query termQuery(Object value, SearchExecutionContext context) { } @Override - public GeoPointScriptFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public GeoPointScriptFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { return new GeoPointScriptFieldData.Builder(name(), leafFactory(searchLookup.get()), GeoPointDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index cac0594899573..dfaeeb5f0249d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -64,7 +64,11 @@ public Query existsQuery(SearchExecutionContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { return new ConstantIndexFieldData.Builder( fullyQualifiedIndexName, name(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index c45840e8d0830..037599669d9cf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -391,7 +391,11 @@ public static Query rangeQuery( } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.IP, IpDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java index b8acd5b0cc953..efd7dd3cc87b4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java @@ -91,7 +91,11 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { } @Override - public IpScriptFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IpScriptFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { return new IpScriptFieldData.Builder(name(), leafFactory(searchLookup.get()), IpDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 5a9ee744c439b..6099c666b4ca5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -682,38 +682,44 @@ NamedAnalyzer normalizer() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - failIfNoDocValues(); - return new SortedSetOrdinalsIndexFieldData.Builder( - name(), - CoreValuesSourceType.KEYWORD, - (dv, n) -> new KeywordDocValuesField(FieldData.toString(dv), n) - ); - } + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { + if (type == FielddataType.SEARCH) { + failIfNoDocValues(); + } - @Override - public IndexFieldData.Builder scriptFielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - if (hasDocValues()) { - return fielddataBuilder(fullyQualifiedIndexName, searchLookup); + if ((type == FielddataType.SEARCH || type == FielddataType.SCRIPT) && hasDocValues()) { + return new SortedSetOrdinalsIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + (dv, n) -> new KeywordDocValuesField(FieldData.toString(dv), n) + ); } - return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( - name(), - CoreValuesSourceType.KEYWORD, - new SourceValueFetcher(searchLookup.get().sourcePaths(name()), nullValue) { - @Override - protected String parseSourceValue(Object value) { - String keywordValue = value.toString(); - if (keywordValue.length() > ignoreAbove) { - return null; + if (type == FielddataType.SCRIPT) { + return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( + name(), + CoreValuesSourceType.KEYWORD, + new SourceValueFetcher(searchLookup.get().sourcePaths(name()), nullValue) { + @Override + protected String parseSourceValue(Object value) { + String keywordValue = value.toString(); + if (keywordValue.length() > ignoreAbove) { + return null; + } + + return normalizeValue(normalizer(), name(), keywordValue); } + }, + searchLookup.get().source(), + KeywordDocValuesField::new + ); + } - return normalizeValue(normalizer(), name(), keywordValue); - } - }, - searchLookup.get().source(), - KeywordDocValuesField::new - ); + throw new IllegalStateException("unknown field data type [" + type.name() + "]"); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java index 8586dc4771c07..691d7a10d2925 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java @@ -100,7 +100,11 @@ public Object valueForDisplay(Object value) { } @Override - public StringScriptFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public StringScriptFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { return new StringScriptFieldData.Builder(name(), leafFactory(searchLookup.get()), KeywordDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java index 3b9ea4a89a850..668c9561328d0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java @@ -92,7 +92,11 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { } @Override - public LongScriptFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public LongScriptFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { return new LongScriptFieldData.Builder(name(), leafFactory(searchLookup.get()), LongDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index 26ad58a96cb80..d70c2e459e158 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -81,32 +81,33 @@ public MappedFieldType( this.meta = Objects.requireNonNull(meta); } + /** + * Type to specify what data structures are used to retrieve + * field data from and generate a representation of doc values. + */ + public enum FielddataType { + SEARCH, + SCRIPT + } + /** * Return a fielddata builder for this field * * @param fullyQualifiedIndexName the name of the index this field-data is build for * @param searchLookup a {@link SearchLookup} supplier to allow for accessing other fields values in the context of runtime fields + * @param options * @throws IllegalArgumentException if the fielddata is not supported on this type. * An IllegalArgumentException is needed in order to return an http error 400 * when this error occurs in a request. see: {@link org.elasticsearch.ExceptionsHelper#status} */ - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType options + ) { throw new IllegalArgumentException("Fielddata is not supported on field [" + name() + "] of type [" + typeName() + "]"); } - /** - * A specialized version of {@link MappedFieldType#fielddataBuilder(String, Supplier)} for scripting. Allows specific - * fields types to customize which data structure(s) values are fetched from for a field such as using - * source when doc values are not available or defaulting to source for unparsed value access when - * it makes sense to do so. - * - * @param fullyQualifiedIndexName the name of the index this field-data is build for - * @param searchLookup a {@link SearchLookup} supplier to allow for accessing other fields values in the context of runtime fields - */ - public IndexFieldData.Builder scriptFielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - return fielddataBuilder(fullyQualifiedIndexName, searchLookup); - } - /** * Create a helper class to fetch field values during the {@link FetchFieldsPhase}. * @@ -185,7 +186,7 @@ public Function pointReaderIfPossible() { */ public boolean isAggregatable() { try { - fielddataBuilder("", () -> { throw new UnsupportedOperationException("SearchLookup not available"); }); + fielddataBuilder("", () -> { throw new UnsupportedOperationException("SearchLookup not available"); }, FielddataType.SEARCH); return true; } catch (IllegalArgumentException e) { return false; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index d1090b3f69077..6d41a01b04b4b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1423,30 +1423,36 @@ public Function pointReaderIfPossible() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - failIfNoDocValues(); - return type.getFieldDataBuilder(name()); - } + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { + if (type == FielddataType.SEARCH) { + failIfNoDocValues(); + } - @Override - public IndexFieldData.Builder scriptFielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - if (hasDocValues()) { - return type.getFieldDataBuilder(name()); + if ((type == FielddataType.SEARCH || type == FielddataType.SCRIPT) && hasDocValues()) { + return this.type.getFieldDataBuilder(name()); } - return type.getValueFetcherFieldDataBuilder( - name(), - searchLookup.get().source(), - new SourceValueFetcher(searchLookup.get().sourcePaths(name()), nullValue) { - @Override - protected Object parseSourceValue(Object value) { - if (value.equals("")) { - return nullValue; + if (type == FielddataType.SCRIPT) { + return this.type.getValueFetcherFieldDataBuilder( + name(), + searchLookup.get().source(), + new SourceValueFetcher(searchLookup.get().sourcePaths(name()), nullValue) { + @Override + protected Object parseSourceValue(Object value) { + if (value.equals("")) { + return nullValue; + } + return NumberFieldType.this.type.parse(value, coerce); } - return type.parse(value, coerce); } - } - ); + ); + } + + throw new IllegalStateException("unknown field data type [" + type.name() + "]"); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java index fab325c7947c0..d77271ceaadaf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java @@ -254,7 +254,11 @@ public IntervalsSource wildcardIntervals(BytesRef pattern, SearchExecutionContex } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { throw new IllegalArgumentException(fail("aggregation or sorts")); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java index cd6f19f24502e..82ed12212113b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java @@ -137,7 +137,11 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { if (fieldDataEnabled.getAsBoolean() == false) { throw new IllegalArgumentException( "Fielddata access on the _id field is disallowed, " diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index 56ce812d1ad2d..f3c57644f1e32 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -220,7 +220,11 @@ public RangeType rangeType() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { failIfNoDocValues(); return new BinaryIndexFieldData.Builder(name(), CoreValuesSourceType.RANGE); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index 45ffa6b58b99c..c1851864657b2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -201,7 +201,11 @@ public Query rangeQuery( } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, SeqNoDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 8a987988861c7..a9b99b0cbacfe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -894,7 +894,11 @@ public static boolean hasGaps(TokenStream stream) throws IOException { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { if (fielddata == false) { throw new IllegalArgumentException( "Text fields are not optimised for operations that require per-document " diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index ec4c04ea81e4a..44871c14083eb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -104,7 +104,7 @@ public String typeName() { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataType.SEARCH)); } @Override @@ -116,7 +116,11 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { failIfNoDocValues(); // TODO don't leak the TSID's binary format into the script return new SortedOrdinalsIndexFieldData.Builder( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index 89201239e46cc..b3f02bf81fd2a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -98,7 +98,11 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { throw new IllegalArgumentException("Fielddata is not supported on [_id] field in [time_series] indices"); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java index 706a862b989b5..8f960bb0c1345 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java @@ -52,11 +52,15 @@ public Query termQuery(Object value, SearchExecutionContext context) { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataType.SEARCH)); } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, VersionDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 4dd6a4e9e4c00..ba528e3b08a73 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -352,7 +352,11 @@ public BytesRef indexedValueForSearch(Object value) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { failIfNoDocValues(); return new KeyedFlattenedFieldData.Builder(name(), key, (dv, n) -> new FlattenedDocValuesField(FieldData.toString(dv), n)); } @@ -633,7 +637,11 @@ public Object valueForDisplay(Object value) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder( name(), diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index b66ae08386c68..7c57f4683a855 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -279,7 +279,11 @@ public boolean isAggregatable() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + FielddataType type + ) { return new VectorIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, indexVersionCreated, dims, indexed); } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 606470be0c083..822b3ffe85d6d 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -23,7 +23,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.TriFunction; +import org.elasticsearch.common.QuadFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.regex.Regex; @@ -37,6 +37,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MappedFieldType.FielddataType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperParsingException; @@ -91,8 +92,7 @@ public class SearchExecutionContext extends QueryRewriteContext { private final MappingLookup mappingLookup; private final SimilarityService similarityService; private final BitsetFilterCache bitsetFilterCache; - private final TriFunction, IndexFieldData> indexFieldDataLookup; - private final TriFunction, IndexFieldData> scriptIndexFieldDataLookup; + private final QuadFunction, FielddataType, IndexFieldData> indexFieldDataLookup; private SearchLookup lookup = null; private final int shardId; @@ -122,8 +122,7 @@ public SearchExecutionContext( int shardRequestIndex, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, - TriFunction, IndexFieldData> indexFieldDataLookup, - TriFunction, IndexFieldData> scriptIndexFieldDataLookup, + QuadFunction, FielddataType, IndexFieldData> indexFieldDataLookup, MapperService mapperService, MappingLookup mappingLookup, SimilarityService similarityService, @@ -145,7 +144,6 @@ public SearchExecutionContext( indexSettings, bitsetFilterCache, indexFieldDataLookup, - scriptIndexFieldDataLookup, mapperService, mappingLookup, similarityService, @@ -174,7 +172,6 @@ public SearchExecutionContext(SearchExecutionContext source) { source.indexSettings, source.bitsetFilterCache, source.indexFieldDataLookup, - source.scriptIndexFieldDataLookup, source.mapperService, source.mappingLookup, source.similarityService, @@ -198,8 +195,7 @@ private SearchExecutionContext( int shardRequestIndex, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, - TriFunction, IndexFieldData> indexFieldDataLookup, - TriFunction, IndexFieldData> scriptIndexFieldDataLookup, + QuadFunction, FielddataType, IndexFieldData> indexFieldDataLookup, MapperService mapperService, MappingLookup mappingLookup, SimilarityService similarityService, @@ -224,7 +220,6 @@ private SearchExecutionContext( this.mappingLookup = mappingLookup; this.bitsetFilterCache = bitsetFilterCache; this.indexFieldDataLookup = indexFieldDataLookup; - this.scriptIndexFieldDataLookup = scriptIndexFieldDataLookup; this.allowUnmappedFields = indexSettings.isDefaultAllowUnmappedFields(); this.nestedScope = new NestedScope(); this.scriptService = scriptService; @@ -284,11 +279,12 @@ public boolean allowExpensiveQueries() { } @SuppressWarnings("unchecked") - public > IFD getForField(MappedFieldType fieldType) { + public > IFD getForField(MappedFieldType fieldType, FielddataType fielddataType) { return (IFD) indexFieldDataLookup.apply( fieldType, fullyQualifiedIndex.getName(), - () -> this.lookup().forkAndTrackFieldReferences(fieldType.name()) + () -> this.lookup().forkAndTrackFieldReferences(fieldType.name()), + fielddataType ); } @@ -484,8 +480,12 @@ public SearchLookup lookup() { if (this.lookup == null) { this.lookup = new SearchLookup( this::getFieldType, - (fieldType, searchLookup) -> indexFieldDataLookup.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup), - (fieldType, searchLookup) -> scriptIndexFieldDataLookup.apply(fieldType, fullyQualifiedIndex.getName(), searchLookup), + (fieldType, searchLookup, fielddataType) -> indexFieldDataLookup.apply( + fieldType, + fullyQualifiedIndex.getName(), + searchLookup, + fielddataType + ), this::sourcePath ); } diff --git a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java index 37bceefb9a7fd..14993d908f167 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java @@ -266,7 +266,7 @@ private LongValuesSource createValuesSource(SearchExecutionContext context) { throw new QueryShardException(context, "failed to find minimum_should_match field [" + minimumShouldMatchField + "]"); } - IndexNumericFieldData fieldData = context.getForField(msmFieldType); + IndexNumericFieldData fieldData = context.getForField(msmFieldType, MappedFieldType.FielddataType.SEARCH); longValuesSource = new FieldValuesSource(fieldData); } else if (minimumShouldMatchScript != null) { TermsSetQueryScript.Factory factory = context.compile(minimumShouldMatchScript, TermsSetQueryScript.CONTEXT); diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java index 568886f3804f1..d2874784d0b0d 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java @@ -260,7 +260,7 @@ private AbstractDistanceScoreFunction parseNumberVariable( DecayFunctionBuilder.ORIGIN ); } - IndexNumericFieldData numericFieldData = context.getForField(fieldType); + IndexNumericFieldData numericFieldData = context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); return new NumericFieldDataScoreFunction(origin, scale, decay, offset, getDecayFunction(), numericFieldData, mode); } @@ -300,7 +300,7 @@ private AbstractDistanceScoreFunction parseGeoVariable( } double scale = DistanceUnit.DEFAULT.parse(scaleString, DistanceUnit.DEFAULT); double offset = DistanceUnit.DEFAULT.parse(offsetString, DistanceUnit.DEFAULT); - IndexGeoPointFieldData indexFieldData = context.getForField(fieldType); + IndexGeoPointFieldData indexFieldData = context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); return new GeoFieldDataScoreFunction(origin, scale, decay, offset, getDecayFunction(), indexFieldData, mode); } @@ -350,7 +350,7 @@ private AbstractDistanceScoreFunction parseDateVariable( double scale = val.getMillis(); val = TimeValue.parseTimeValue(offsetString, TimeValue.timeValueHours(24), DecayFunctionParser.class.getSimpleName() + ".offset"); double offset = val.getMillis(); - IndexNumericFieldData numericFieldData = context.getForField(dateFieldType); + IndexNumericFieldData numericFieldData = context.getForField(dateFieldType, MappedFieldType.FielddataType.SEARCH); return new NumericFieldDataScoreFunction(origin, scale, decay, offset, getDecayFunction(), numericFieldData, mode); } diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java index 3431349332873..7c20b866543be 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.lucene.search.function.FieldValueFactorFunction; import org.elasticsearch.common.lucene.search.function.ScoreFunction; import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -140,7 +141,7 @@ public Version getMinimalSupportedVersion() { protected ScoreFunction doToFunction(SearchExecutionContext context) { IndexNumericFieldData fieldData = null; if (context.isFieldMapped(field)) { - fieldData = context.getForField(context.getFieldType(field)); + fieldData = context.getForField(context.getFieldType(field), MappedFieldType.FielddataType.SEARCH); } else { if (missing == null) { throw new ElasticsearchException("Unable to find a field mapper for field [" + field + "]. No 'missing' value defined."); diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java index 53e09336abfe9..11662be3c8880 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java @@ -16,6 +16,7 @@ import org.elasticsearch.common.lucene.search.function.RandomScoreFunction; import org.elasticsearch.common.lucene.search.function.ScoreFunction; import org.elasticsearch.index.mapper.IdFieldMapper; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; @@ -166,7 +167,11 @@ protected ScoreFunction doToFunction(SearchExecutionContext context) { ); } int seed = this.seed == null ? hash(context.nowInMillis()) : this.seed; - return new RandomScoreFunction(seed, salt, context.getForField(context.getFieldType(fieldName))); + return new RandomScoreFunction( + seed, + salt, + context.getForField(context.getFieldType(fieldName), MappedFieldType.FielddataType.SEARCH) + ); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java index 9170afc1af03a..671022567ca12 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java @@ -435,7 +435,7 @@ public Analyzer buildCustomAnalyzer( @Override protected IndexFieldData buildFieldData(MappedFieldType ft) { - return context.getForField(ft); + return context.getForField(ft, MappedFieldType.FielddataType.SEARCH); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index 8515bff1049d9..d43ee2d8b0df7 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -46,7 +46,7 @@ public FetchSubPhaseProcessor getProcessor(FetchContext context) { } ValueFetcher fetcher = new DocValueFetcher( ft.docValueFormat(fieldAndFormat.format, null), - context.searchLookup().getForField(ft) + context.searchLookup().getForField(ft, MappedFieldType.FielddataType.SEARCH) ); fields.add(new DocValueField(fieldAndFormat.field, fetcher)); } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java index f7d1f8fab59af..241a21da7c391 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java @@ -23,13 +23,13 @@ import java.util.Collection; import java.util.Map; import java.util.Set; +import java.util.function.BiFunction; import java.util.function.Function; public class LeafDocLookup implements Map> { private final Function fieldTypeLookup; - private final Function> fieldDataLookup; - private final Function> scriptFieldDataLookup; + private final BiFunction> fieldDataLookup; private final LeafReaderContext reader; private int docId = -1; @@ -38,13 +38,11 @@ public class LeafDocLookup implements Map> { LeafDocLookup( Function fieldTypeLookup, - Function> fieldDataLookup, - Function> scriptFieldDataLookup, + BiFunction> fieldDataLookup, LeafReaderContext reader ) { this.fieldTypeLookup = fieldTypeLookup; this.fieldDataLookup = fieldDataLookup; - this.scriptFieldDataLookup = scriptFieldDataLookup; this.reader = reader; } @@ -52,11 +50,11 @@ public void setDocument(int docId) { this.docId = docId; } - protected DocValuesScriptFieldFactory getScriptFieldFactory(String fieldName, Function> lookup) { + protected DocValuesScriptFieldFactory getScriptFieldFactory(String fieldName, MappedFieldType.FielddataType options) { DocValuesScriptFieldFactory factory = localCacheScriptFieldData.get(fieldName); // do not use cached source fallback fields for old style doc access - if (lookup == fieldDataLookup && factory instanceof SourceValueFetcherIndexFieldData.ValueFetcherDocValues) { + if (options == MappedFieldType.FielddataType.SEARCH && factory instanceof SourceValueFetcherIndexFieldData.ValueFetcherDocValues) { factory = null; } @@ -72,7 +70,7 @@ protected DocValuesScriptFieldFactory getScriptFieldFactory(String fieldName, Fu factory = AccessController.doPrivileged(new PrivilegedAction() { @Override public DocValuesScriptFieldFactory run() { - return lookup.apply(fieldType).load(reader).getScriptFieldFactory(fieldName); + return fieldDataLookup.apply(fieldType, options).load(reader).getScriptFieldFactory(fieldName); } }); @@ -89,12 +87,12 @@ public DocValuesScriptFieldFactory run() { } public Field getScriptField(String fieldName) { - return getScriptFieldFactory(fieldName, scriptFieldDataLookup).toScriptField(); + return getScriptFieldFactory(fieldName, MappedFieldType.FielddataType.SCRIPT).toScriptField(); } @Override public ScriptDocValues get(Object key) { - return getScriptFieldFactory(key.toString(), fieldDataLookup).toScriptDocValues(); + return getScriptFieldFactory(key.toString(), MappedFieldType.FielddataType.SEARCH).toScriptDocValues(); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index f7defe1359487..ac65c8785aef1 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -9,6 +9,7 @@ package org.elasticsearch.search.lookup; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; @@ -16,7 +17,6 @@ import java.util.LinkedHashSet; import java.util.Objects; import java.util.Set; -import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -41,8 +41,7 @@ public class SearchLookup { private final Set fieldChain; private final SourceLookup sourceLookup; private final Function fieldTypeLookup; - private final BiFunction, IndexFieldData> fieldDataLookup; - private final BiFunction, IndexFieldData> scriptFieldDataLookup; + private final TriFunction, MappedFieldType.FielddataType, IndexFieldData> fieldDataLookup; private final Function> sourcePathsLookup; /** @@ -51,15 +50,13 @@ public class SearchLookup { */ public SearchLookup( Function fieldTypeLookup, - BiFunction, IndexFieldData> fieldDataLookup, - BiFunction, IndexFieldData> scriptFieldDataLookup, + TriFunction, MappedFieldType.FielddataType, IndexFieldData> fieldDataLookup, Function> sourcePathsLookup ) { this.fieldTypeLookup = fieldTypeLookup; this.fieldChain = Collections.emptySet(); this.sourceLookup = new SourceLookup(); this.fieldDataLookup = fieldDataLookup; - this.scriptFieldDataLookup = scriptFieldDataLookup; this.sourcePathsLookup = sourcePathsLookup; } @@ -75,7 +72,6 @@ private SearchLookup(SearchLookup searchLookup, Set fieldChain) { this.sourceLookup = searchLookup.sourceLookup; this.fieldTypeLookup = searchLookup.fieldTypeLookup; this.fieldDataLookup = searchLookup.fieldDataLookup; - this.scriptFieldDataLookup = searchLookup.scriptFieldDataLookup; this.sourcePathsLookup = searchLookup.sourcePathsLookup; } @@ -103,7 +99,7 @@ public final SearchLookup forkAndTrackFieldReferences(String field) { public LeafSearchLookup getLeafSearchLookup(LeafReaderContext context) { return new LeafSearchLookup( context, - new LeafDocLookup(fieldTypeLookup, this::getForField, this::getForScriptField, context), + new LeafDocLookup(fieldTypeLookup, this::getForField, context), sourceLookup, new LeafStoredFieldsLookup(fieldTypeLookup, (doc, visitor) -> context.reader().document(doc, visitor)) ); @@ -113,12 +109,8 @@ public MappedFieldType fieldType(String fieldName) { return fieldTypeLookup.apply(fieldName); } - public IndexFieldData getForField(MappedFieldType fieldType) { - return fieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name())); - } - - public IndexFieldData getForScriptField(MappedFieldType fieldType) { - return scriptFieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name())); + public IndexFieldData getForField(MappedFieldType fieldType, MappedFieldType.FielddataType options) { + return fieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name()), options); } public Set sourcePaths(String fieldName) { diff --git a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java index 30a9e5b6d5ad6..89b096e070dae 100644 --- a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java @@ -260,7 +260,7 @@ private Query createSliceQuery(int id, int max, SearchExecutionContext context, if (type.hasDocValues() == false) { throw new IllegalArgumentException("cannot load numeric doc values on " + field); } else { - IndexFieldData ifm = context.getForField(type); + IndexFieldData ifm = context.getForField(type, MappedFieldType.FielddataType.SEARCH); if (ifm instanceof IndexNumericFieldData == false) { throw new IllegalArgumentException("cannot load numeric doc values on " + field); } diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index 46c88b8915782..05dac56754cfc 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -362,7 +362,7 @@ public SortFieldAndFormat build(SearchExecutionContext context) throws IOExcepti fieldType = resolveUnmappedType(context); } - IndexFieldData fieldData = context.getForField(fieldType); + IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); if (fieldData instanceof IndexNumericFieldData == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) { throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields"); @@ -462,7 +462,7 @@ public BucketedSort buildBucketedSort(SearchExecutionContext context, BigArrays fieldType = resolveUnmappedType(context); } - IndexFieldData fieldData = context.getForField(fieldType); + IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); if (fieldData instanceof IndexNumericFieldData == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) { throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields"); diff --git a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java index cd9478e7a8a68..0d2625ec7ee9f 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java @@ -616,7 +616,7 @@ private IndexGeoPointFieldData fieldData(SearchExecutionContext context) { throw new IllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort"); } } - return context.getForField(fieldType); + return context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); } private Nested nested(SearchExecutionContext context) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 46b27a548908f..7fb08c86192ad 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -127,7 +127,6 @@ public void setupCreateIndexRequestAndAliasValidator() { null, null, null, - null, parserConfig(), writableRegistry(), null, diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index 87c89f9e5fa7f..bff019a76f7a8 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -132,7 +132,11 @@ public String typeName() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { searchLookup.get(); return null; } @@ -216,6 +220,6 @@ private Sort buildIndexSort(IndexSettings indexSettings, Map indexFieldDataService.getForField(ft, "index", s)); + return config.buildIndexSort(lookup::get, (ft, s, fdt) -> indexFieldDataService.getForField(ft, "index", s, fdt)); } } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 0007525f4b357..3c2c1277f71ab 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -157,7 +157,7 @@ public > IFD getForField(String type, String field } else { throw new UnsupportedOperationException(type); } - return searchExecutionContext.getForField(fieldType); + return searchExecutionContext.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); } @Before diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java index 0bee4ffdf8313..c5b06adf18495 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java @@ -56,7 +56,7 @@ public void testFilterByFrequency() throws Exception { .fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0) .build(builderContext) .fieldType(); - IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft); + IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataType.SEARCH); for (LeafReaderContext context : contexts) { LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); @@ -71,7 +71,7 @@ public void testFilterByFrequency() throws Exception { .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, 201, 100) .build(builderContext) .fieldType(); - IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft); + IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataType.SEARCH); for (LeafReaderContext context : contexts) { LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); @@ -86,7 +86,7 @@ public void testFilterByFrequency() throws Exception { .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101) .build(builderContext) .fieldType(); - IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft); + IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataType.SEARCH); for (LeafReaderContext context : contexts) { LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); @@ -102,7 +102,7 @@ public void testFilterByFrequency() throws Exception { .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101) .build(builderContext) .fieldType(); - IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft); + IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataType.SEARCH); for (LeafReaderContext context : contexts) { LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 084246a8bd0dc..15f18eaf1a74f 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -80,7 +80,12 @@ public void testGetForFieldDefaults() { MapperBuilderContext context = MapperBuilderContext.ROOT; final MappedFieldType stringMapper = new KeywordFieldMapper.Builder("string", Version.CURRENT).build(context).fieldType(); ifdService.clear(); - IndexFieldData fd = ifdService.getForField(stringMapper, "test", () -> { throw new UnsupportedOperationException(); }); + IndexFieldData fd = ifdService.getForField( + stringMapper, + "test", + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataType.SEARCH + ); assertTrue(fd instanceof SortedSetOrdinalsIndexFieldData); for (MappedFieldType mapper : Arrays.asList( @@ -90,7 +95,12 @@ public void testGetForFieldDefaults() { new NumberFieldMapper.Builder("long", LONG, ScriptCompiler.NONE, false, true, Version.CURRENT).build(context).fieldType() )) { ifdService.clear(); - fd = ifdService.getForField(mapper, "test", () -> { throw new UnsupportedOperationException(); }); + fd = ifdService.getForField( + mapper, + "test", + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataType.SEARCH + ); assertTrue(fd instanceof SortedNumericIndexFieldData); } @@ -103,7 +113,12 @@ public void testGetForFieldDefaults() { Version.CURRENT ).build(context).fieldType(); ifdService.clear(); - fd = ifdService.getForField(floatMapper, "test", () -> { throw new UnsupportedOperationException(); }); + fd = ifdService.getForField( + floatMapper, + "test", + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataType.SEARCH + ); assertTrue(fd instanceof SortedDoublesIndexFieldData); final MappedFieldType doubleMapper = new NumberFieldMapper.Builder( @@ -115,7 +130,12 @@ public void testGetForFieldDefaults() { Version.CURRENT ).build(context).fieldType(); ifdService.clear(); - fd = ifdService.getForField(doubleMapper, "test", () -> { throw new UnsupportedOperationException(); }); + fd = ifdService.getForField( + doubleMapper, + "test", + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataType.SEARCH + ); assertTrue(fd instanceof SortedDoublesIndexFieldData); } @@ -129,14 +149,15 @@ public void testGetForFieldRuntimeField() { ); final SetOnce> searchLookupSetOnce = new SetOnce<>(); MappedFieldType ft = mock(MappedFieldType.class); - when(ft.fielddataBuilder(ArgumentMatchers.any(), ArgumentMatchers.any())).thenAnswer(invocationOnMock -> { + MappedFieldType.FielddataType fdt = MappedFieldType.FielddataType.SEARCH; + when(ft.fielddataBuilder(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenAnswer(invocationOnMock -> { @SuppressWarnings("unchecked") Supplier searchLookup = (Supplier) invocationOnMock.getArguments()[1]; searchLookupSetOnce.set(searchLookup); return (IndexFieldData.Builder) (cache, breakerService) -> null; }); - SearchLookup searchLookup = new SearchLookup(null, null, null, null); - ifdService.getForField(ft, "qualified", () -> searchLookup); + SearchLookup searchLookup = new SearchLookup(null, null, null); + ifdService.getForField(ft, "qualified", () -> searchLookup, fdt); assertSame(searchLookup, searchLookupSetOnce.get().get()); } @@ -176,8 +197,18 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon onRemovalCalled.incrementAndGet(); } }); - IndexFieldData ifd1 = ifdService.getForField(mapper1, "test", () -> { throw new UnsupportedOperationException(); }); - IndexFieldData ifd2 = ifdService.getForField(mapper2, "test", () -> { throw new UnsupportedOperationException(); }); + IndexFieldData ifd1 = ifdService.getForField( + mapper1, + "test", + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataType.SEARCH + ); + IndexFieldData ifd2 = ifdService.getForField( + mapper2, + "test", + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataType.SEARCH + ); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); LeafFieldData loadField1 = ifd1.load(leafReaderContext); LeafFieldData loadField2 = ifd2.load(leafReaderContext); @@ -251,7 +282,12 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon onRemovalCalled.incrementAndGet(); } }); - IndexFieldData ifd = ifdService.getForField(mapper1, "test", () -> { throw new UnsupportedOperationException(); }); + IndexFieldData ifd = ifdService.getForField( + mapper1, + "test", + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataType.SEARCH + ); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); LeafFieldData load = ifd.load(leafReaderContext); assertEquals(1, onCacheCalled.get()); @@ -313,11 +349,17 @@ private void doTestRequireDocValues(MappedFieldType ft) { null ); if (ft.hasDocValues()) { - ifds.getForField(ft, "test", () -> { throw new UnsupportedOperationException(); }); // no exception + // no exception + ifds.getForField(ft, "test", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataType.SEARCH); } else { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> ifds.getForField(ft, "test", () -> { throw new UnsupportedOperationException(); }) + () -> ifds.getForField( + ft, + "test", + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataType.SEARCH + ) ); assertThat(e.getMessage(), containsString("doc values")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index b27f02d48af78..ab1eefa661293 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -202,8 +202,7 @@ protected static SearchExecutionContext mockContext(boolean allowExpensiveQuerie when(context.allowExpensiveQueries()).thenReturn(allowExpensiveQueries); SearchLookup lookup = new SearchLookup( context::getFieldType, - (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null), - (mft, lookupSupplier) -> mft.scriptFielddataBuilder("test", lookupSupplier).build(null, null), + (mft, lookupSupplier, fdt) -> mft.fielddataBuilder("test", lookupSupplier, fdt).build(null, null), context::sourcePath ); when(context.lookup()).thenReturn(lookup); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java index 974918e86a4ed..15bf1eda3b709 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanFieldScriptTests.java @@ -57,7 +57,7 @@ public void testTooManyValues() throws IOException { BooleanFieldScript script = new BooleanFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java index 759162ff30710..1be8ba71a0d30 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java @@ -68,7 +68,8 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); BooleanScriptFieldType ft = simpleMappedFieldType(); - BooleanScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup).build(null, null); + BooleanScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataType.SEARCH) + .build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override public ScoreMode scoreMode() { @@ -105,7 +106,11 @@ public void testSort() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [false]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - BooleanScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder("test", mockContext()::lookup).build(null, null); + BooleanScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( + "test", + mockContext()::lookup, + MappedFieldType.FielddataType.SEARCH + ).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [false]}")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java index fca89107d7e53..11527acb53678 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/CompositeRuntimeFieldTests.java @@ -333,8 +333,7 @@ public void testParseDocumentSubFieldAccess() throws IOException { withLuceneIndex(mapperService, iw -> iw.addDocuments(Arrays.asList(doc1.rootDoc(), doc2.rootDoc())), reader -> { SearchLookup searchLookup = new SearchLookup( mapperService::fieldType, - (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null), - (mft, lookupSupplier) -> mft.scriptFielddataBuilder("test", lookupSupplier).build(null, null), + (mft, lookupSupplier, fdt) -> mft.fielddataBuilder("test", lookupSupplier, fdt).build(null, null), mapperService.mappingLookup()::sourcePaths ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java index 088fca9ee76f0..425e75a112330 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldScriptTests.java @@ -66,7 +66,7 @@ public void testTooManyValues() throws IOException { DateFieldScript script = new DateFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, field -> null), DateFormatter.forPattern(randomDateFormatterPattern()).withLocale(randomLocale(random())), reader.leaves().get(0) ) { @@ -102,7 +102,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { DateFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, field -> null), DateFormatter.forPattern("epoch_millis") ); DateFieldScript dateFieldScript = leafFactory.newInstance(reader.leaves().get(0)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 6c5c1ab4e7634..28606d9e99840 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -183,7 +183,6 @@ public void testTermQuery() { null, null, null, - null, parserConfig(), writableRegistry(), null, @@ -239,7 +238,6 @@ public void testRangeQuery() throws IOException { null, null, null, - null, parserConfig(), writableRegistry(), null, @@ -318,7 +316,6 @@ public void testRangeQueryWithIndexSort() { null, null, null, - null, parserConfig(), writableRegistry(), null, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java index d3b121d5550c5..d79fcee75273e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java @@ -150,7 +150,8 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); DateScriptFieldType ft = build("add_days", Map.of("days", 1)); - DateScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup).build(null, null); + DateScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataType.SEARCH) + .build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override public ScoreMode scoreMode() { @@ -188,7 +189,11 @@ public void testSort() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181356]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - DateScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder("test", mockContext()::lookup).build(null, null); + DateScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( + "test", + mockContext()::lookup, + MappedFieldType.FielddataType.SEARCH + ).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); assertThat(readSource(reader, docs.scoreDocs[0].doc), equalTo("{\"timestamp\": [1595432181351]}")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java index d37a1c635d5b9..7e9812ae2b61f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleFieldScriptTests.java @@ -64,7 +64,7 @@ public void testTooManyValues() throws IOException { DoubleFieldScript script = new DoubleFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, field -> null), reader.leaves().get(0) ) { @Override @@ -99,7 +99,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { DoubleFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, field -> null) ); DoubleFieldScript doubleFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java index c8d8a975afd2e..43ab5030ab517 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java @@ -65,7 +65,8 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); DoubleScriptFieldType ft = build("add_param", Map.of("param", 1)); - DoubleScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup).build(null, null); + DoubleScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataType.SEARCH) + .build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override public ScoreMode scoreMode() { @@ -103,7 +104,11 @@ public void testSort() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2.1]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - DoubleScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder("test", mockContext()::lookup).build(null, null); + DoubleScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( + "test", + mockContext()::lookup, + MappedFieldType.FielddataType.SEARCH + ).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [1.1]}")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java index e096851b3be44..df8e09684f4fc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java @@ -43,7 +43,6 @@ public void testTermQuery() { null, null, null, - null, mappingLookup, null, null, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java index e53c28b4744f8..d6f710ab1825f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldScriptTests.java @@ -59,7 +59,7 @@ public void testTooManyValues() throws IOException { GeoPointFieldScript script = new GeoPointFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java index 93ad371364c7c..043caf1aeae35 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java @@ -65,7 +65,8 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); GeoPointScriptFieldType ft = build("fromLatLon", Map.of()); - GeoPointScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup).build(null, null); + GeoPointScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataType.SEARCH) + .build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override public ScoreMode scoreMode() { @@ -98,7 +99,11 @@ public void collect(int doc) throws IOException { @Override public void testSort() throws IOException { - GeoPointScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder("test", mockContext()::lookup).build(null, null); + GeoPointScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( + "test", + mockContext()::lookup, + MappedFieldType.FielddataType.SEARCH + ).build(null, null); Exception e = expectThrows(IllegalArgumentException.class, () -> ifd.sortField(null, MultiValueMode.MIN, null, false)); assertThat(e.getMessage(), equalTo("can't sort on geo_point field without using specific sorting feature, like geo_distance")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java index f783b78f8f35f..49c663db35d2e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IgnoredFieldMapperTests.java @@ -53,7 +53,6 @@ public void testFetchIgnoredFieldValue() throws IOException { SearchLookup lookup = new SearchLookup( mapperService::fieldType, fieldDataLookup(), - scriptFieldDataLookup(), mapperService.mappingLookup()::sourcePaths ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java index 153072edef7cd..4efdf9e57aca9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldMapperTests.java @@ -47,12 +47,7 @@ public void testFetchFieldValue() throws IOException { iw.addDocument(mapperService.documentMapper().parse(source).rootDoc()); }, iw -> { IndexFieldMapper.IndexFieldType ft = (IndexFieldMapper.IndexFieldType) mapperService.fieldType("_index"); - SearchLookup lookup = new SearchLookup( - mapperService::fieldType, - fieldDataLookup(), - scriptFieldDataLookup(), - mapperService.mappingLookup()::sourcePaths - ); + SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(), mapperService.mappingLookup()::sourcePaths); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); IndexSearcher searcher = newSearcher(iw); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java index 0dcd652d2cc0b..dd90380e02af8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java @@ -70,7 +70,6 @@ private SearchExecutionContext createContext() { null, null, null, - null, parserConfig(), writableRegistry(), null, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java index c8d1d9daadbcb..e80564ee8b47b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpFieldScriptTests.java @@ -65,7 +65,7 @@ public void testTooManyValues() throws IOException { IpFieldScript script = new IpFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, field -> null), reader.leaves().get(0) ) { @Override @@ -100,7 +100,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { IpFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, field -> null) ); IpFieldScript ipFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java index c8c013b2e7408..8ee150123cdf4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java @@ -67,7 +67,8 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); IpScriptFieldType ft = build("append_param", Map.of("param", ".1")); - BinaryScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup).build(null, null); + BinaryScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataType.SEARCH) + .build(null, null); DocValueFormat format = ft.docValueFormat(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override @@ -106,7 +107,11 @@ public void testSort() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.0.2\"]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - BinaryScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder("test", mockContext()::lookup).build(null, null); + BinaryScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( + "test", + mockContext()::lookup, + MappedFieldType.FielddataType.SEARCH + ).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); assertThat( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java index 0a06aca4add6e..1428464e81c9f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java @@ -60,7 +60,8 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); KeywordScriptFieldType ft = build("append_param", Map.of("param", "-suffix")); - StringScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup).build(null, null); + StringScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataType.SEARCH) + .build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override public ScoreMode scoreMode() { @@ -98,7 +99,11 @@ public void testSort() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"b\"]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - BinaryScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder("test", mockContext()::lookup).build(null, null); + BinaryScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( + "test", + mockContext()::lookup, + MappedFieldType.FielddataType.SEARCH + ).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [\"a\"]}")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java index c1062fff2ace7..3ffe0d9f4966c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongFieldScriptTests.java @@ -64,7 +64,7 @@ public void testTooManyValues() throws IOException { LongFieldScript script = new LongFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, field -> null), reader.leaves().get(0) ) { @Override @@ -99,7 +99,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { LongFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, field -> null) ); LongFieldScript longFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java index 539fed677a4e5..1b10a7a9ac932 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java @@ -77,7 +77,8 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); LongScriptFieldType ft = build("add_param", Map.of("param", 1)); - LongScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup).build(null, null); + LongScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataType.SEARCH) + .build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override public ScoreMode scoreMode() { @@ -115,7 +116,11 @@ public void testSort() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - LongScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder("test", mockContext()::lookup).build(null, null); + LongScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( + "test", + mockContext()::lookup, + MappedFieldType.FielddataType.SEARCH + ).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [1]}")); @@ -132,7 +137,11 @@ public void testNow() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181356]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - LongScriptFieldData ifd = build("millis_ago", Map.of()).fielddataBuilder("test", mockContext()::lookup).build(null, null); + LongScriptFieldData ifd = build("millis_ago", Map.of()).fielddataBuilder( + "test", + mockContext()::lookup, + MappedFieldType.FielddataType.SEARCH + ).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); assertThat(readSource(reader, docs.scoreDocs[0].doc), equalTo("{\"timestamp\": [1595432181356]}")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index c138da8a62201..5131422b9e827 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -584,7 +584,8 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS NumberFieldType fieldType = new NumberFieldType("field", type); IndexNumericFieldData fielddata = (IndexNumericFieldData) fieldType.fielddataBuilder( "index", - () -> { throw new UnsupportedOperationException(); } + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataType.SEARCH ).build(null, null); SortField sortField = fielddata.sortField(null, MultiValueMode.MIN, null, randomBoolean()); @@ -614,7 +615,6 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS null, null, null, - null, parserConfig(), writableRegistry(), null, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java index a1d025637f284..955c48b342bb2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapperTests.java @@ -61,12 +61,7 @@ public void testFetchValue() throws Exception { .rootDoc() ); }, iw -> { - SearchLookup lookup = new SearchLookup( - mapperService::fieldType, - fieldDataLookup(), - scriptFieldDataLookup(), - mapperService.mappingLookup()::sourcePaths - ); + SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(), mapperService.mappingLookup()::sourcePaths); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); FieldFetcher fieldFetcher = FieldFetcher.create( searchExecutionContext, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java index ccb6f8eb69e07..d22dc1b29d43e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java @@ -56,13 +56,15 @@ public void testEnableFieldData() throws IOException { IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, - () -> ft.fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }).build(null, null) + () -> ft.fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataType.SEARCH) + .build(null, null) ); assertThat(exc.getMessage(), containsString(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())); assertFalse(ft.isAggregatable()); enabled[0] = true; - ft.fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }).build(null, null); + ft.fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataType.SEARCH) + .build(null, null); assertWarnings(ProvidedIdFieldMapper.ID_FIELD_DATA_DEPRECATION_MESSAGE); assertTrue(ft.isAggregatable()); } @@ -77,7 +79,6 @@ public void testFetchIdFieldValue() throws IOException { SearchLookup lookup = new SearchLookup( mapperService::fieldType, fieldDataLookup(), - scriptFieldDataLookup(), mapperService.mappingLookup()::sourcePaths ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index 9636cc8d5a944..0f79f2e01c1b6 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -212,7 +212,6 @@ private SearchExecutionContext createContext() { null, null, null, - null, parserConfig(), writableRegistry(), null, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java index 228ccddde051f..bd644731baf9e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RoutingFieldMapperTests.java @@ -71,7 +71,6 @@ public void testFetchRoutingFieldValue() throws IOException { SearchLookup lookup = new SearchLookup( mapperService::fieldType, fieldDataLookup(), - scriptFieldDataLookup(), mapperService.mappingLookup()::sourcePaths ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java index d0a989de5649b..75666d18c726b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/StringFieldScriptTests.java @@ -63,7 +63,7 @@ public void testTooManyValues() throws IOException { StringFieldScript script = new StringFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, field -> null), reader.leaves().get(0) ) { @Override @@ -89,7 +89,7 @@ public void testTooManyChars() throws IOException { StringFieldScript script = new StringFieldScript( "test", Map.of(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, field -> null), reader.leaves().get(0) ) { @Override @@ -129,7 +129,7 @@ public final void testFromSourceDoesNotEnforceValuesLimit() throws IOException { StringFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, field -> null) ); StringFieldScript stringFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = stringFieldScript.resultsForDoc(0); @@ -158,7 +158,7 @@ public final void testFromSourceDoesNotEnforceCharsLimit() throws IOException { StringFieldScript.LeafFactory leafFactory = fromSource().newFactory( "field", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null) + new SearchLookup(field -> null, (ft, lookup, fdt) -> null, field -> null) ); StringFieldScript stringFieldScript = leafFactory.newInstance(reader.leaves().get(0)); List results = stringFieldScript.resultsForDoc(0); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 96de3d4ce1a9c..1185d894c4b79 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -529,13 +529,15 @@ public void testFielddata() throws IOException { MapperService disabledMapper = createMapperService(fieldMapping(this::minimalMapping)); Exception e = expectThrows( IllegalArgumentException.class, - () -> disabledMapper.fieldType("field").fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }) + () -> disabledMapper.fieldType("field") + .fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataType.SEARCH) ); assertThat(e.getMessage(), containsString("Text fields are not optimised for operations that require per-document field data")); MapperService enabledMapper = createMapperService(fieldMapping(b -> b.field("type", "text").field("fielddata", true))); - enabledMapper.fieldType("field").fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }); // no exception - // this time + enabledMapper.fieldType("field") + // no exception this time + .fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataType.SEARCH); e = expectThrows( MapperParsingException.class, @@ -1198,7 +1200,7 @@ public void testIgnoreFieldDataOnLegacyIndex() throws IOException { expectThrows( IllegalArgumentException.class, () -> ((TextFieldMapper) finalMapperService.documentMapper().mappers().getMapper("field")).fieldType() - .fielddataBuilder("test", null) + .fielddataBuilder("test", null, MappedFieldType.FielddataType.SEARCH) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java index 15295b0ac4f57..ab6895f3f26fd 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/VersionFieldMapperTests.java @@ -50,12 +50,7 @@ public void testFetchFieldValue() throws IOException { iw.addDocument(parsedDoc.rootDoc()); }, iw -> { VersionFieldMapper.VersionFieldType ft = (VersionFieldMapper.VersionFieldType) mapperService.fieldType("_version"); - SearchLookup lookup = new SearchLookup( - mapperService::fieldType, - fieldDataLookup(), - scriptFieldDataLookup(), - mapperService.mappingLookup()::sourcePaths - ); + SearchLookup lookup = new SearchLookup(mapperService::fieldType, fieldDataLookup(), mapperService.mappingLookup()::sourcePaths); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); ValueFetcher valueFetcher = ft.valueFetcher(searchExecutionContext, null); IndexSearcher searcher = newSearcher(iw); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java index 209d9fab3eb37..e4dad1e9ae1e9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java @@ -73,7 +73,8 @@ public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) { IndexFieldData ifd1 = ifdService.getForField( fieldType1, "test", - () -> { throw new UnsupportedOperationException("search lookup not available"); } + () -> { throw new UnsupportedOperationException("search lookup not available"); }, + MappedFieldType.FielddataType.SEARCH ); assertTrue(ifd1 instanceof KeyedFlattenedFieldData); @@ -87,7 +88,8 @@ public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) { IndexFieldData ifd2 = ifdService.getForField( fieldType2, "test", - () -> { throw new UnsupportedOperationException("search lookup not available"); } + () -> { throw new UnsupportedOperationException("search lookup not available"); }, + MappedFieldType.FielddataType.SEARCH ); assertTrue(ifd2 instanceof KeyedFlattenedFieldData); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index b155f0b51bfc7..6540effdf2bc0 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -10,6 +10,7 @@ import org.elasticsearch.Version; import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.DenseVectorFieldType; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.VectorSimilarity; @@ -52,7 +53,9 @@ public void testIsAggregatable() { public void testFielddataBuilder() { DenseVectorFieldType ft = createFieldType(); - assertNotNull(ft.fielddataBuilder("index", () -> { throw new UnsupportedOperationException(); })); + assertNotNull( + ft.fielddataBuilder("index", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataType.SEARCH) + ); } public void testDocValueFormat() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java index 05187bc3d3b0d..7a11a694dfc06 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java @@ -18,7 +18,7 @@ public class SparseVectorFieldTypeTests extends FieldTypeTestCase { public void testDocValuesDisabled() { MappedFieldType fieldType = new SparseVectorFieldMapper.SparseVectorFieldType("field", Collections.emptyMap()); assertFalse(fieldType.hasDocValues()); - expectThrows(IllegalArgumentException.class, () -> fieldType.fielddataBuilder("index", null)); + expectThrows(IllegalArgumentException.class, () -> fieldType.fielddataBuilder("index", null, MappedFieldType.FielddataType.SEARCH)); } public void testIsNotAggregatable() { diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java index 721568c62ceae..6e40b0c3316ee 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java @@ -35,7 +35,6 @@ public void testRewriteMissingField() throws Exception { indexService.getIndexSettings(), null, null, - null, indexService.mapperService(), indexService.mapperService().mappingLookup(), null, @@ -76,7 +75,6 @@ public void testRewriteMissingReader() throws Exception { indexService.getIndexSettings(), null, null, - null, indexService.mapperService(), indexService.mapperService().mappingLookup(), null, @@ -119,7 +117,6 @@ public void testRewriteEmptyReader() throws Exception { indexService.getIndexSettings(), null, null, - null, indexService.mapperService(), indexService.mapperService().mappingLookup(), null, diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index 5b63a6705847f..fe4045bb563a5 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -149,7 +149,7 @@ public void testClusterAlias() throws IOException { IndexFieldMapper mapper = new IndexFieldMapper(); - IndexFieldData forField = context.getForField(mapper.fieldType()); + IndexFieldData forField = context.getForField(mapper.fieldType(), MappedFieldType.FielddataType.SEARCH); String expected = clusterAlias == null ? context.getIndexSettings().getIndexMetadata().getIndex().getName() : clusterAlias + ":" + context.getIndexSettings().getIndex().getName(); @@ -184,7 +184,6 @@ public void testIndexSortedOnField() { null, null, null, - null, XContentParserConfiguration.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), null, @@ -427,8 +426,11 @@ private static SearchExecutionContext createSearchExecutionContext( 0, indexSettings, null, - (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), - (mappedFieldType, idxName, searchLookup) -> mappedFieldType.scriptFielddataBuilder(idxName, searchLookup).build(null, null), + (mappedFieldType, idxName, searchLookup, fielddataType) -> mappedFieldType.fielddataBuilder( + idxName, + searchLookup, + fielddataType + ).build(null, null), mapperService, mappingLookup, null, @@ -484,7 +486,11 @@ private static RuntimeField runtimeField(String name, Function runtimeDocValues) { TestRuntimeField.TestRuntimeFieldType fieldType = new TestRuntimeField.TestRuntimeFieldType(name, null) { @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { return (cache, breakerService) -> new IndexFieldData<>() { @Override public String getFieldName() { @@ -600,9 +606,9 @@ private static List collect(String field, SearchExecutionContext searchE MappedFieldType fieldType = searchExecutionContext.getFieldType(field); IndexFieldData indexFieldData; if (randomBoolean()) { - indexFieldData = searchExecutionContext.getForField(fieldType); + indexFieldData = searchExecutionContext.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); } else { - indexFieldData = searchExecutionContext.lookup().getForField(fieldType); + indexFieldData = searchExecutionContext.lookup().getForField(fieldType, MappedFieldType.FielddataType.SEARCH); } searcher.search(query, new Collector() { @Override diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index b4748d99fa1b4..c93e68c44be53 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2668,7 +2668,8 @@ public void testReaderWrapperWorksWithGlobalOrdinals() throws IOException { IndexFieldData.Global ifd = indexFieldDataService.getForField( foo, "test", - () -> { throw new UnsupportedOperationException("search lookup not available"); } + () -> { throw new UnsupportedOperationException("search lookup not available"); }, + MappedFieldType.FielddataType.SEARCH ); FieldDataStats before = shard.fieldData().stats("foo"); assertThat(before.getMemorySizeInBytes(), equalTo(0L)); diff --git a/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java index 696dcca5855fb..4ea0b2f3f8fd4 100644 --- a/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/CompositeFieldScriptTests.java @@ -31,7 +31,7 @@ public void testTooManyValues() throws IOException { CompositeFieldScript script = new CompositeFieldScript( "composite", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null, field -> null), reader.leaves().get(0) ) { @Override @@ -62,7 +62,7 @@ public void testTooManyChars() throws IOException { CompositeFieldScript script = new CompositeFieldScript( "composite", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null, field -> null), reader.leaves().get(0) ) { @Override @@ -75,7 +75,7 @@ public void execute() { StringFieldScript stringFieldScript = new StringFieldScript( "composite.leaf", Collections.emptyMap(), - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null, field -> null), reader.leaves().get(0) ) { @Override diff --git a/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java index 50411f507f1e9..be7893b0478b2 100644 --- a/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/field/SortedNumericDocValuesLongFieldScriptTests.java @@ -39,7 +39,7 @@ public void testValuesLimitIsNotEnforced() throws IOException { try (DirectoryReader reader = iw.getReader()) { SortedNumericDocValuesLongFieldScript docValues = new SortedNumericDocValuesLongFieldScript( "test", - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null, field -> null), reader.leaves().get(0) ); List values = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java b/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java index ddc2c96c094fa..ea9c57899b7d3 100644 --- a/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java +++ b/server/src/test/java/org/elasticsearch/script/field/SortedSetDocValuesStringFieldScriptTests.java @@ -40,7 +40,7 @@ public void testValuesLimitIsNotEnforced() throws IOException { try (DirectoryReader reader = iw.getReader()) { SortedSetDocValuesStringFieldScript docValues = new SortedSetDocValuesStringFieldScript( "test", - new SearchLookup(field -> null, (ft, lookup) -> null, (ft, lookup) -> null, field -> null), + new SearchLookup(field -> null, (ft, lookup, ftd) -> null, field -> null), reader.leaves().get(0) ); List values = new ArrayList<>(); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java index fa9a8633b2394..09875f615ae86 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/range/RangeAggregatorTests.java @@ -635,7 +635,7 @@ public void testOverlappingRanges() throws IOException { */ public void testRuntimeFieldTopLevelQueryNotOptimized() throws IOException { long totalDocs = (long) RangeAggregator.DOCS_PER_RANGE_TO_USE_FILTERS * 4; - SearchLookup lookup = new SearchLookup(s -> null, (ft, l) -> null, (ft, l) -> null, s -> null); + SearchLookup lookup = new SearchLookup(s -> null, (ft, l, ftd) -> null, s -> null); StringFieldScript.LeafFactory scriptFactory = ctx -> new StringFieldScript("dummy", Map.of(), lookup, ctx) { @Override public void execute() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index 3d45485f251d8..77d71eba2e78e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -2044,7 +2044,7 @@ public void testWithFilterAndPreciseSize() throws IOException { */ public void testRuntimeFieldTopLevelNotOptimized() throws IOException { long totalDocs = 500; - SearchLookup lookup = new SearchLookup(s -> null, (ft, l) -> null, (ft, l) -> null, s -> null); + SearchLookup lookup = new SearchLookup(s -> null, (ft, l, ftd) -> null, s -> null); StringFieldScript.LeafFactory scriptFactory = ctx -> new StringFieldScript("dummy", Map.of(), lookup, ctx) { @Override public void execute() { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java index cd3dc6816fa10..6e90fbdd54790 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/support/ScriptValuesTests.java @@ -36,7 +36,7 @@ private static class FakeAggregationScript extends AggregationScript { int index; FakeAggregationScript(Object[][] values) { - super(Collections.emptyMap(), new SearchLookup(null, null, null, null) { + super(Collections.emptyMap(), new SearchLookup(null, null, null) { @Override public LeafSearchLookup getLeafSearchLookup(LeafReaderContext context) { diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 3c30a905d3f45..b8ce869a8bfc2 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -13,8 +13,8 @@ import org.apache.lucene.util.automaton.TooComplexToDeterminizeException; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.QuadFunction; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.Settings; @@ -235,7 +235,7 @@ public void testMetadataFields() throws IOException { new FieldAndFormat("_ignored", null) ); FieldFetcher fieldFetcher = FieldFetcher.create( - newSearchExecutionContext(mapperService, (ft, index, sl) -> fieldDataLookup().apply(ft, sl)), + newSearchExecutionContext(mapperService, (ft, index, sl, ftd) -> fieldDataLookup().apply(ft, sl, ftd)), fieldList ); IndexSearcher searcher = newSearcher(iw); @@ -1123,7 +1123,7 @@ public void testFetchRuntimeFieldWithSourceDisabled() throws IOException { MapperService mapperService = createMapperService(mapping); SearchExecutionContext searchExecutionContext = newSearchExecutionContext( mapperService, - (ft, index, sl) -> fieldDataLookup().apply(ft, sl) + (ft, index, sl, ftd) -> fieldDataLookup().apply(ft, sl, ftd) ); withLuceneIndex(mapperService, iw -> iw.addDocument(new LuceneDocument()), iw -> { FieldFetcher fieldFetcher = FieldFetcher.create(searchExecutionContext, fieldAndFormatList("runtime_field", null, false)); @@ -1152,7 +1152,7 @@ public void testFetchMetadataFieldWithSourceDisabled() throws IOException { MapperService mapperService = createMapperService(mapping); SearchExecutionContext searchExecutionContext = newSearchExecutionContext( mapperService, - (ft, index, sl) -> fieldDataLookup().apply(ft, sl) + (ft, index, sl, ftd) -> fieldDataLookup().apply(ft, sl, ftd) ); withLuceneIndex(mapperService, iw -> { ParsedDocument parsedDocument = mapperService.documentMapper().parse(source("{}")); @@ -1239,7 +1239,7 @@ private static SearchExecutionContext newSearchExecutionContext(MapperService ma private static SearchExecutionContext newSearchExecutionContext( MapperService mapperService, - TriFunction, IndexFieldData> indexFieldDataLookup + QuadFunction, MappedFieldType.FielddataType, IndexFieldData> indexFieldDataLookup ) { Settings settings = Settings.builder() .put("index.version.created", Version.CURRENT) @@ -1255,7 +1255,6 @@ private static SearchExecutionContext newSearchExecutionContext( indexSettings, null, indexFieldDataLookup, - null, mapperService, mapperService.mappingLookup(), null, diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index 370272cb687cf..42f95718c522a 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -303,7 +303,6 @@ public void testBuildSearchContextHighlight() throws IOException { null, null, null, - null, parserConfig(), namedWriteableRegistry, null, diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java index 7992c6dc4e600..ddb2ffbcb42cd 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java @@ -19,7 +19,7 @@ import org.junit.Before; import java.io.IOException; -import java.util.function.Function; +import java.util.function.BiFunction; import static org.mockito.AdditionalAnswers.returnsFirstArg; import static org.mockito.ArgumentMatchers.any; @@ -50,8 +50,7 @@ public void setUp() throws Exception { docLookup = new LeafDocLookup( field -> field.equals("field") ? fieldType1 : field.equals("alias") ? fieldType2 : null, - fieldType -> fieldType == fieldType1 ? fieldData1 : fieldType == fieldType2 ? fieldData2 : null, - fieldType -> fieldType == fieldType1 ? fieldData1 : fieldType == fieldType2 ? fieldData2 : null, + (fieldType, fielddataType) -> fieldType == fieldType1 ? fieldData1 : fieldType == fieldType2 ? fieldData2 : null, null ); } @@ -78,12 +77,7 @@ public void testFlattenedField() throws IOException { MappedFieldType fieldType1 = fieldType.getChildFieldType("key1"); MappedFieldType fieldType2 = fieldType.getChildFieldType("key2"); - Function> fieldDataSupplier = ft -> { - FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) ft; - return keyedFieldType.key().equals("key1") ? fieldData1 : fieldData2; - }; - - Function> scriptFieldDataSupplier = ft -> { + BiFunction> fieldDataSupplier = (ft, fdt) -> { FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) ft; return keyedFieldType.key().equals("key1") ? fieldData1 : fieldData2; }; @@ -96,7 +90,7 @@ public void testFlattenedField() throws IOException { return fieldType2; } return null; - }, fieldDataSupplier, scriptFieldDataSupplier, null); + }, fieldDataSupplier, null); assertEquals(docValues1, docLookup.get("flattened.key1")); assertEquals(docValues2, docLookup.get("flattened.key2")); diff --git a/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java b/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java index c99a770053b73..74b2000024e8b 100644 --- a/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/ScriptScoreQueryTests.java @@ -49,7 +49,7 @@ public class ScriptScoreQueryTests extends ESTestCase { private DirectoryReader reader; private IndexSearcher searcher; private LeafReaderContext leafReaderContext; - private final SearchLookup lookup = new SearchLookup(null, null, null, null); + private final SearchLookup lookup = new SearchLookup(null, null, null); @Before public void initSearcher() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java index d046d6b60d6c1..7685119e7b660 100644 --- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java @@ -141,7 +141,6 @@ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, null, null, null, - null, parserConfig(), namedWriteableRegistry, null, @@ -204,7 +203,6 @@ public void testRewritingKeepsSettings() throws IOException { null, null, null, - null, parserConfig(), namedWriteableRegistry, null, diff --git a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java index f39f9e5eb1a74..b1e8433c3b144 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/GeoPointScriptFieldDistanceFeatureQueryTests.java @@ -82,7 +82,7 @@ public void testMatches() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"location\": [-3.56, -45.98]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - SearchLookup searchLookup = new SearchLookup(null, null, null, f -> "location".equals(f) ? Set.of("location") : null); + SearchLookup searchLookup = new SearchLookup(null, null, f -> "location".equals(f) ? Set.of("location") : null); Function leafFactory = ctx -> new GeoPointFieldScript( "test", Map.of(), diff --git a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java index e48e9d9e212a1..c06aab3816261 100644 --- a/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java +++ b/server/src/test/java/org/elasticsearch/search/runtime/LongScriptFieldDistanceFeatureQueryTests.java @@ -69,7 +69,7 @@ public void testMatches() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181351]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - SearchLookup searchLookup = new SearchLookup(null, null, null, f -> "timestamp".equals(f) ? Set.of("timestamp") : null); + SearchLookup searchLookup = new SearchLookup(null, null, f -> "timestamp".equals(f) ? Set.of("timestamp") : null); Function leafFactory = ctx -> new DateFieldScript( "test", Map.of(), diff --git a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java index ee16d22a90f25..6005d4ac84bf2 100644 --- a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java @@ -171,7 +171,7 @@ public Query existsQuery(SearchExecutionContext context) { when(context.getIndexSettings()).thenReturn(indexSettings); if (dvType != null) { IndexNumericFieldData fd = mock(IndexNumericFieldData.class); - when(context.getForField(fieldType)).thenReturn(fd); + when(context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH)).thenReturn(fd); } return context; diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 8a5ad6c5500d2..13c8f875b6c3f 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -12,7 +12,7 @@ import org.apache.lucene.search.SortField; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.TriFunction; +import org.elasticsearch.common.QuadFunction; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.env.Environment; @@ -189,20 +189,15 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build() ); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, Mockito.mock(BitsetFilterCache.Listener.class)); - TriFunction, IndexFieldData> indexFieldDataLookup = ( - fieldType, - fieldIndexName, - searchLookup) -> { - IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); - return builder.build(new IndexFieldDataCache.None(), null); - }; - TriFunction, IndexFieldData> scriptIndexFieldDataLookup = ( - fieldType, - fieldIndexName, - searchLookup) -> { - IndexFieldData.Builder builder = fieldType.scriptFielddataBuilder(fieldIndexName, searchLookup); - return builder.build(new IndexFieldDataCache.None(), null); - }; + QuadFunction< + MappedFieldType, + String, + Supplier, + MappedFieldType.FielddataType, + IndexFieldData> indexFieldDataLookup = (fieldType, fieldIndexName, searchLookup, fielddataType) -> { + IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup, fielddataType); + return builder.build(new IndexFieldDataCache.None(), null); + }; NestedLookup nestedLookup = NestedLookup.build( List.of(new NestedObjectMapper.Builder("path", Version.CURRENT).build(MapperBuilderContext.ROOT)) ); @@ -212,7 +207,6 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea idxSettings, bitsetFilterCache, indexFieldDataLookup, - scriptIndexFieldDataLookup, null, null, null, diff --git a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java index 24d8a4cec82be..35b1fa8001d7b 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -179,7 +179,6 @@ public NamedAnalyzer get(Object key) { idxSettings, null, null, - null, mapperService, lookup, null, @@ -236,7 +235,6 @@ public void testBuildWithUnmappedField() { idxSettings, null, null, - null, mock(MapperService.class), MappingLookup.EMPTY, null, diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index c3f93f6aa35e8..48adae3dc4880 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -21,6 +21,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -76,7 +77,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.BiFunction; import java.util.function.BooleanSupplier; import java.util.function.Function; import java.util.function.Supplier; @@ -450,7 +450,8 @@ public Query filterQuery(Query query) { @Override protected IndexFieldData buildFieldData(MappedFieldType ft) { - return ft.fielddataBuilder("test", null).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); + return ft.fielddataBuilder("test", null, MappedFieldType.FielddataType.SEARCH) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } @Override @@ -624,9 +625,7 @@ public void onRemoval(ShardId shardId, Accountable accountable) { } }), - (ft, idxName, lookup) -> ft.fielddataBuilder(idxName, lookup) - .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), - (ft, idxName, lookup) -> ft.scriptFielddataBuilder(idxName, lookup) + (ft, idxName, lookup, fdt) -> ft.fielddataBuilder(idxName, lookup, fdt) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()), mapperService, mapperService.mappingLookup(), @@ -645,13 +644,8 @@ public void onRemoval(ShardId shardId, Accountable accountable) { ); } - protected BiFunction, IndexFieldData> fieldDataLookup() { - return (mft, lookupSource) -> mft.fielddataBuilder("test", lookupSource) - .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); - } - - protected BiFunction, IndexFieldData> scriptFieldDataLookup() { - return (mft, lookupSource) -> mft.scriptFielddataBuilder("test", lookupSource) + protected TriFunction, MappedFieldType.FielddataType, IndexFieldData> fieldDataLookup() { + return (mft, lookupSource, fdt) -> mft.fielddataBuilder("test", lookupSource, fdt) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 3a28e0496974f..9ae917f679c59 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -324,10 +324,9 @@ protected final List fetchFromDocValues(MapperService mapperService, MappedFi SearchLookup lookup = new SearchLookup( mapperService::fieldType, fieldDataLookup(), - scriptFieldDataLookup(), mapperService.mappingLookup()::sourcePaths ); - ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft)); + ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft, MappedFieldType.FielddataType.SEARCH)); IndexSearcher searcher = newSearcher(iw); LeafReaderContext context = searcher.getIndexReader().leaves().get(0); lookup.source().setSegmentAndDocument(context, 0); @@ -577,16 +576,18 @@ protected void registerDimensionChecks(ParameterChecker checker) throws IOExcept */ protected void assertFetch(MapperService mapperService, String field, Object value, String format) throws IOException { MappedFieldType ft = mapperService.fieldType(field); + MappedFieldType.FielddataType fdt = MappedFieldType.FielddataType.SEARCH; SourceToParse source = source(b -> b.field(ft.name(), value)); ValueFetcher docValueFetcher = new DocValueFetcher( ft.docValueFormat(format, null), - ft.fielddataBuilder("test", () -> null).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) + ft.fielddataBuilder("test", () -> null, MappedFieldType.FielddataType.SEARCH) + .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); when(searchExecutionContext.isSourceEnabled()).thenReturn(true); when(searchExecutionContext.sourcePath(field)).thenReturn(Set.of(field)); - when(searchExecutionContext.getForField(ft)).thenAnswer( - inv -> fieldDataLookup().apply(ft, () -> { throw new UnsupportedOperationException(); }) + when(searchExecutionContext.getForField(ft, fdt)).thenAnswer( + inv -> fieldDataLookup().apply(ft, () -> { throw new UnsupportedOperationException(); }, fdt) ); ValueFetcher nativeFetcher = ft.valueFetcher(searchExecutionContext, format); ParsedDocument doc = mapperService.documentMapper().parse(source); @@ -660,13 +661,18 @@ public final void testIndexTimeFieldData() throws IOException { DocValuesScriptFieldFactory docValuesFieldSource = fieldType.fielddataBuilder( "test", - () -> { throw new UnsupportedOperationException(); } + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataType.SEARCH ).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()).load(ctx).getScriptFieldFactory("test"); docValuesFieldSource.setNextDocId(0); DocumentLeafReader reader = new DocumentLeafReader(doc.rootDoc(), Collections.emptyMap()); - DocValuesScriptFieldFactory indexData = fieldType.fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }) + DocValuesScriptFieldFactory indexData = fieldType.fielddataBuilder( + "test", + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataType.SEARCH + ) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) .load(reader.getContext()) .getScriptFieldFactory("test"); @@ -703,8 +709,7 @@ public final void testIndexTimeStoredFieldsAccess() throws IOException { SearchLookup lookup = new SearchLookup( f -> fieldType, - (f, s) -> { throw new UnsupportedOperationException(); }, - (f, s) -> { throw new UnsupportedOperationException(); }, + (f, s, t) -> { throw new UnsupportedOperationException(); }, f -> f.equals("field") ? Set.of("field") : null ); diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 225e4b87c56d7..895b05c367aff 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -44,8 +44,8 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedBiConsumer; +import org.elasticsearch.common.QuadFunction; import org.elasticsearch.common.TriConsumer; -import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.breaker.CircuitBreaker; import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -292,15 +292,11 @@ protected AggregationContext createAggregationContext( .map(ft -> new FieldAliasMapper(ft.name() + "-alias", ft.name() + "-alias", ft.name())) .collect(toList()) ); - TriFunction, IndexFieldData> fieldDataBuilder = ( + QuadFunction, MappedFieldType.FielddataType, IndexFieldData> fieldDataBuilder = ( fieldType, s, - searchLookup) -> fieldType.fielddataBuilder(indexSettings.getIndex().getName(), searchLookup) - .build(new IndexFieldDataCache.None(), breakerService); - TriFunction, IndexFieldData> scriptFieldDataBuilder = ( - fieldType, - s, - searchLookup) -> fieldType.fielddataBuilder(indexSettings.getIndex().getName(), searchLookup) + searchLookup, + fdt) -> fieldType.fielddataBuilder(indexSettings.getIndex().getName(), searchLookup, fdt) .build(new IndexFieldDataCache.None(), breakerService); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() { @Override @@ -315,7 +311,6 @@ public void onCache(ShardId shardId, Accountable accountable) {} indexSettings, bitsetFilterCache, fieldDataBuilder, - scriptFieldDataBuilder, null, mappingLookup, null, @@ -1057,7 +1052,9 @@ public void testSupportedFieldTypes() throws IOException { } private ValuesSourceType fieldToVST(MappedFieldType fieldType) { - return fieldType.fielddataBuilder("", () -> { throw new UnsupportedOperationException(); }).build(null, null).getValuesSourceType(); + return fieldType.fielddataBuilder("", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataType.SEARCH) + .build(null, null) + .getValuesSourceType(); } /** diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index be4ad37b7be93..1a49a898a69f3 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -495,7 +495,6 @@ SearchExecutionContext createShardContext(IndexSearcher searcher) { idxSettings, bitsetFilterCache, indexFieldDataService::getForField, - indexFieldDataService::getForScriptField, mapperService, mapperService.mappingLookup(), similarityService, diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index b971a048a06d1..61c4c416f218d 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -169,7 +169,11 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { failIfNoDocValues(); return (cache, breakerService) -> new IndexHistogramFieldData(name(), AnalyticsValuesSourceType.HISTOGRAM) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java index 3f87cdceef1bd..9d712d2381384 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/cluster/routing/allocation/mapper/DataTierFieldTypeTests.java @@ -122,7 +122,6 @@ private SearchExecutionContext createContext() { null, null, null, - null, parserConfig(), writableRegistry(), null, @@ -153,7 +152,6 @@ private SearchExecutionContext createContextWithoutSetting() { null, null, null, - null, parserConfig(), writableRegistry(), null, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java index b6d0168825477..c576c66948373 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java @@ -595,7 +595,6 @@ private TestIndexContext testIndex(MappingLookup mappingLookup, Client client) t null, null, null, - null, mappingLookup, null, null, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java index 8cf3769c2534b..dc5e7475e1c7d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java @@ -92,7 +92,6 @@ public void testDLS() throws Exception { null, null, null, - null, mappingLookup, null, null, @@ -253,7 +252,6 @@ public void testDLSWithLimitedPermissions() throws Exception { null, null, null, - null, mappingLookup, null, null, diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java index 13691e3e8e676..4a6cfc3c5b464 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -391,7 +391,11 @@ public Relation isFieldWithinQuery( } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { return (cache, breakerService) -> new IndexAggregateDoubleMetricFieldData( name(), AggregateMetricsValuesSourceType.AGGREGATE_METRIC diff --git a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java index 2469deaf910af..5046fe77768f1 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/test/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldTypeTests.java @@ -121,8 +121,7 @@ public void testUsedInScript() throws IOException { when(searchExecutionContext.allowExpensiveQueries()).thenReturn(true); SearchLookup lookup = new SearchLookup( searchExecutionContext::getFieldType, - (mft, lookupSupplier) -> mft.fielddataBuilder("test", lookupSupplier).build(null, null), - (mft, lookupSupplier) -> mft.scriptFielddataBuilder("test", lookupSupplier).build(null, null), + (mft, lookupSupplier, fdt) -> mft.fielddataBuilder("test", lookupSupplier, fdt).build(null, null), searchExecutionContext::sourcePath ); when(searchExecutionContext.lookup()).thenReturn(lookup); diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index a93f2417ef5b2..ba1c507619736 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -132,7 +132,11 @@ public String familyTypeName() { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { return new ConstantIndexFieldData.Builder( value, name(), diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index ea1469bac9d40..c0e40c7db5bc7 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -288,7 +288,11 @@ public Query rangeQuery( } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { failIfNoDocValues(); return (cache, breakerService) -> { final IndexNumericFieldData signedLongValues = new SortedNumericIndexFieldData.Builder( diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index ca1d722c6fc01..7248f537c68c5 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -279,7 +279,11 @@ protected BytesRef indexedValueForSearch(Object value) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, VersionStringDocValuesField::new); } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java index d299e4a1d01ad..bad322592acee 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java @@ -109,7 +109,7 @@ static List build(SearchExecutionContext context, String[] fi if (fieldType == null) { throw new IllegalArgumentException("Unknown field: [" + field + "]"); } - IndexFieldData fieldData = context.getForField(fieldType); + IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); fetchers.add(new FieldValueFetcher(field, fieldType, fieldData, getValidator(field))); } return Collections.unmodifiableList(fetchers); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index 793cc79709a4d..8e2714f8bb9e1 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -105,7 +105,6 @@ private void setup() { null, null, null, - null, () -> 0L, null, null, diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/MetricFieldProducerTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/MetricFieldProducerTests.java index f0bbd7f3a289f..e0f97315b7082 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/MetricFieldProducerTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/v2/MetricFieldProducerTests.java @@ -198,7 +198,6 @@ public void testBuildMetricProducers() { null, null, null, - null, () -> 0L, null, null, diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 246cb744b75cb..2dac8f0cdd32b 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -181,7 +181,11 @@ public GeoShapeWithDocValuesFieldType( this.geoFormatterFactory = geoFormatterFactory; } - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { failIfNoDocValues(); return new AbstractLatLonShapeIndexFieldData.Builder(name(), GeoShapeValuesSourceType.instance(), GeoShapeDocValuesField::new); } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 425ff07a0418e..f8f0d10371bcc 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -832,7 +832,11 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { } @Override - public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + public IndexFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier searchLookup, + MappedFieldType.FielddataType type + ) { failIfNoDocValues(); return (cache, breakerService) -> new StringBinaryIndexFieldData( name(), diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 26ab0ffeeea60..8e2f3dce0b11b 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -44,7 +44,7 @@ import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; -import org.elasticsearch.common.TriFunction; +import org.elasticsearch.common.QuadFunction; import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; @@ -1075,27 +1075,21 @@ protected final SearchExecutionContext createMockContext() { Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build() ); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, Mockito.mock(BitsetFilterCache.Listener.class)); - TriFunction, IndexFieldData> indexFieldDataLookup = ( - fieldType, - fieldIndexName, - searchLookup) -> { - IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); - return builder.build(new IndexFieldDataCache.None(), null); - }; - TriFunction, IndexFieldData> scriptIndexFieldDataLookup = ( - fieldType, - fieldIndexName, - searchLookup) -> { - IndexFieldData.Builder builder = fieldType.scriptFielddataBuilder(fieldIndexName, searchLookup); - return builder.build(new IndexFieldDataCache.None(), null); - }; + QuadFunction< + MappedFieldType, + String, + Supplier, + MappedFieldType.FielddataType, + IndexFieldData> indexFieldDataLookup = (fieldType, fieldIndexName, searchLookup, fielddataType) -> { + IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup, fielddataType); + return builder.build(new IndexFieldDataCache.None(), null); + }; return new SearchExecutionContext( 0, 0, idxSettings, bitsetFilterCache, indexFieldDataLookup, - scriptIndexFieldDataLookup, null, null, null, From f039525994f74384df72584e326c42504ed8342e Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Fri, 22 Jul 2022 08:06:56 -0700 Subject: [PATCH 15/23] Fix package --- .../main/java/org/elasticsearch/{ => common}/QuadFunction.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename server/src/main/java/org/elasticsearch/{ => common}/QuadFunction.java (100%) diff --git a/server/src/main/java/org/elasticsearch/QuadFunction.java b/server/src/main/java/org/elasticsearch/common/QuadFunction.java similarity index 100% rename from server/src/main/java/org/elasticsearch/QuadFunction.java rename to server/src/main/java/org/elasticsearch/common/QuadFunction.java From e8f813a4c7e2bb8ee668e73f26989f6b06c2f37c Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Fri, 22 Jul 2022 08:43:16 -0700 Subject: [PATCH 16/23] change name from type to operation --- .../script/ScriptScoreBenchmark.java | 2 +- .../AggConstructionContentionBenchmark.java | 2 +- .../expression/ExpressionScriptEngine.java | 2 +- .../extras/MatchOnlyTextFieldMapper.java | 3 +-- .../mapper/extras/RankFeatureFieldMapper.java | 2 +- .../extras/RankFeaturesFieldMapper.java | 2 +- .../mapper/extras/ScaledFloatFieldMapper.java | 3 +-- .../mapper/extras/TokenCountFieldMapper.java | 2 +- .../extras/ScaledFloatFieldTypeTests.java | 4 +-- .../join/mapper/ParentIdFieldMapper.java | 3 +-- .../join/mapper/ParentJoinFieldMapper.java | 2 +- .../join/query/HasChildQueryBuilder.java | 2 +- .../join/query/HasParentQueryBuilder.java | 2 +- .../percolator/PercolateQueryBuilder.java | 7 ++++-- .../percolator/QueryBuilderStoreTests.java | 4 +-- .../icu/ICUCollationKeywordFieldMapper.java | 2 +- .../mapper/murmur3/Murmur3FieldMapper.java | 2 +- .../index/mapper/size/SizeFieldMapper.java | 2 +- .../elasticsearch/index/IndexSortConfig.java | 4 +-- .../org/elasticsearch/index/IndexWarmer.java | 2 +- .../fielddata/IndexFieldDataService.java | 4 +-- .../index/mapper/AbstractScriptFieldType.java | 2 +- .../index/mapper/BinaryFieldMapper.java | 2 +- .../index/mapper/BooleanFieldMapper.java | 2 +- .../index/mapper/BooleanScriptFieldType.java | 2 +- .../index/mapper/DateFieldMapper.java | 2 +- .../index/mapper/DateScriptFieldType.java | 6 ++++- .../index/mapper/DoubleScriptFieldType.java | 2 +- .../index/mapper/GeoPointFieldMapper.java | 10 ++++---- .../index/mapper/GeoPointScriptFieldType.java | 2 +- .../index/mapper/IndexFieldMapper.java | 2 +- .../index/mapper/IpFieldMapper.java | 2 +- .../index/mapper/IpScriptFieldType.java | 2 +- .../index/mapper/KeywordFieldMapper.java | 10 ++++---- .../index/mapper/KeywordScriptFieldType.java | 2 +- .../index/mapper/LongScriptFieldType.java | 2 +- .../index/mapper/MappedFieldType.java | 14 +++++++---- .../index/mapper/NumberFieldMapper.java | 10 ++++---- .../index/mapper/PlaceHolderFieldMapper.java | 2 +- .../index/mapper/ProvidedIdFieldMapper.java | 2 +- .../index/mapper/RangeFieldMapper.java | 2 +- .../index/mapper/SeqNoFieldMapper.java | 2 +- .../index/mapper/TextFieldMapper.java | 2 +- .../index/mapper/TimeSeriesIdFieldMapper.java | 4 +-- .../mapper/TsidExtractingIdFieldMapper.java | 2 +- .../index/mapper/VersionFieldMapper.java | 4 +-- .../flattened/FlattenedFieldMapper.java | 4 +-- .../vectors/DenseVectorFieldMapper.java | 2 +- .../index/query/SearchExecutionContext.java | 12 ++++----- .../index/query/TermsSetQueryBuilder.java | 2 +- .../functionscore/DecayFunctionBuilder.java | 6 ++--- .../FieldValueFactorFunctionBuilder.java | 2 +- .../RandomScoreFunctionBuilder.java | 2 +- .../support/AggregationContext.java | 2 +- .../fetch/subphase/FetchDocValuesPhase.java | 2 +- .../search/lookup/LeafDocLookup.java | 13 +++++----- .../search/lookup/SearchLookup.java | 10 +++++--- .../search/slice/SliceBuilder.java | 2 +- .../search/sort/FieldSortBuilder.java | 4 +-- .../search/sort/GeoDistanceSortBuilder.java | 2 +- .../index/IndexSortSettingsTests.java | 2 +- .../fielddata/AbstractFieldDataTestCase.java | 2 +- .../index/fielddata/FilterFieldDataTests.java | 8 +++--- .../fielddata/IndexFieldDataServiceTests.java | 25 +++++++++++-------- .../mapper/BooleanScriptFieldTypeTests.java | 4 +-- .../mapper/DateScriptFieldTypeTests.java | 4 +-- .../mapper/DoubleScriptFieldTypeTests.java | 4 +-- .../mapper/GeoPointScriptFieldTypeTests.java | 4 +-- .../index/mapper/IpScriptFieldTypeTests.java | 4 +-- .../mapper/KeywordScriptFieldTypeTests.java | 4 +-- .../mapper/LongScriptFieldTypeTests.java | 6 ++--- .../index/mapper/NumberFieldTypeTests.java | 2 +- .../mapper/ProvidedIdFieldMapperTests.java | 9 ++++--- .../index/mapper/TextFieldMapperTests.java | 6 ++--- .../FlattenedIndexFieldDataTests.java | 4 +-- .../vectors/DenseVectorFieldTypeTests.java | 2 +- .../vectors/SparseVectorFieldTypeTests.java | 5 +++- .../query/SearchExecutionContextTests.java | 8 +++--- .../index/shard/IndexShardTests.java | 2 +- .../fetch/subphase/FieldFetcherTests.java | 7 +++++- .../search/lookup/LeafDocLookupTests.java | 2 +- .../search/slice/SliceBuilderTests.java | 2 +- .../search/sort/AbstractSortTestCase.java | 2 +- .../index/mapper/MapperServiceTestCase.java | 6 +++-- .../index/mapper/MapperTestCase.java | 10 ++++---- .../aggregations/AggregatorTestCase.java | 24 +++++++++++------- .../mapper/HistogramFieldMapper.java | 2 +- .../AggregateDoubleMetricFieldMapper.java | 2 +- .../mapper/ConstantKeywordFieldMapper.java | 2 +- .../unsignedlong/UnsignedLongFieldMapper.java | 2 +- .../VersionStringFieldMapper.java | 2 +- .../xpack/rollup/v2/FieldValueFetcher.java | 2 +- .../GeoShapeWithDocValuesFieldMapper.java | 2 +- .../wildcard/mapper/WildcardFieldMapper.java | 2 +- .../mapper/WildcardFieldMapperTests.java | 2 +- 95 files changed, 215 insertions(+), 178 deletions(-) diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java index 9587587992cdd..57eddecff7462 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java @@ -154,7 +154,7 @@ private Query scriptScoreQuery(ScoreScript.Factory factory) { private ScoreScript.Factory bareMetalScript() { return (params, lookup) -> { MappedFieldType type = fieldTypes.get("n"); - IndexNumericFieldData ifd = (IndexNumericFieldData) lookup.getForField(type, MappedFieldType.FielddataType.SEARCH); + IndexNumericFieldData ifd = (IndexNumericFieldData) lookup.getForField(type, MappedFieldType.FielddataOperation.SEARCH); return new ScoreScript.LeafFactory() { @Override public ScoreScript newInstance(DocReader docReader) throws IOException { diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java index f118aea01b582..8891cb6a60180 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/search/aggregations/AggConstructionContentionBenchmark.java @@ -210,7 +210,7 @@ public Analyzer buildCustomAnalyzer( protected IndexFieldData buildFieldData(MappedFieldType ft) { IndexFieldDataCache indexFieldDataCache = indicesFieldDataCache.buildIndexFieldDataCache(new IndexFieldDataCache.Listener() { }, index, ft.name()); - return ft.fielddataBuilder("test", this::lookup, MappedFieldType.FielddataType.SEARCH) + return ft.fielddataBuilder("test", this::lookup, MappedFieldType.FielddataOperation.SEARCH) .build(indexFieldDataCache, breakerService); } diff --git a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java index 5fe8294d8d7e0..a6c62b10b9f81 100644 --- a/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java +++ b/modules/lang-expression/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngine.java @@ -460,7 +460,7 @@ private static DoubleValuesSource getDocValueSource(String variable, SearchLooku throw new ParseException("Field [" + fieldname + "] does not exist in mappings", 5); } - IndexFieldData fieldData = lookup.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); + IndexFieldData fieldData = lookup.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); final DoubleValuesSource valueSource; if (fieldType instanceof GeoPointFieldType) { // geo diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index 1a522f35fc1fa..56e3f6e5d6fa8 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -34,7 +34,6 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; @@ -280,7 +279,7 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { throw new IllegalArgumentException(CONTENT_TYPE + " fields do not support sorting and aggregations"); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java index 98adfa1cb8d2d..ce0a49d90acb0 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapper.java @@ -114,7 +114,7 @@ public Query existsQuery(SearchExecutionContext context) { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { throw new IllegalArgumentException("[rank_feature] fields do not support sorting, scripting or aggregating"); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java index c83db91040b1d..6a4952a0b03a9 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapper.java @@ -100,7 +100,7 @@ public Query existsQuery(SearchExecutionContext context) { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { throw new IllegalArgumentException("[rank_features] fields do not support sorting, scripting or aggregating"); } diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java index 16354a74c2e58..dc4549d1e5d8e 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java @@ -29,7 +29,6 @@ import org.elasticsearch.index.fielddata.plain.SortedNumericIndexFieldData; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.NumberFieldMapper; import org.elasticsearch.index.mapper.SimpleMappedFieldType; @@ -280,7 +279,7 @@ public Query rangeQuery( public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return (cache, breakerService) -> { diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java index ab746121045c4..585b2ad44720e 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java @@ -122,7 +122,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (hasDocValues() == false) { return (lookup, ignoredValues) -> List.of(); } - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataType.SEARCH)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java index 4593807dbe3e6..ea34c6abf56d0 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldTypeTests.java @@ -192,7 +192,7 @@ public void testFieldData() throws IOException { IndexNumericFieldData fielddata = (IndexNumericFieldData) f1.fielddataBuilder( "index", () -> { throw new UnsupportedOperationException(); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ).build(null, null); assertEquals(fielddata.getNumericType(), IndexNumericFieldData.NumericType.DOUBLE); LeafNumericFieldData leafFieldData = fielddata.load(reader.leaves().get(0)); @@ -209,7 +209,7 @@ public void testFieldData() throws IOException { fielddata = (IndexNumericFieldData) f2.fielddataBuilder( "index", () -> { throw new UnsupportedOperationException(); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ).build(null, null); leafFieldData = fielddata.load(reader.leaves().get(0)); values = leafFieldData.getDoubleValues(); diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java index 95a7df794bf58..c1d6e0e9818aa 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java @@ -21,7 +21,6 @@ import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; -import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; @@ -76,7 +75,7 @@ public boolean eagerGlobalOrdinals() { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder( diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java index 50a345cc6e4d5..bba7be9d2cfa5 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentJoinFieldMapper.java @@ -170,7 +170,7 @@ public String typeName() { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { return new SortedSetOrdinalsIndexFieldData.Builder( name(), diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java index e58748363a31c..a8845b2731abb 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasChildQueryBuilder.java @@ -344,7 +344,7 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { Query childFilter = joiner.filter(type); Query filteredQuery = Queries.filtered(query.toQuery(context), childFilter); MappedFieldType ft = context.getFieldType(parentJoinField); - final SortedSetOrdinalsIndexFieldData fieldData = context.getForField(ft, MappedFieldType.FielddataType.SEARCH); + final SortedSetOrdinalsIndexFieldData fieldData = context.getForField(ft, MappedFieldType.FielddataOperation.SEARCH); return new LateParsingQuery( parentFilter, filteredQuery, diff --git a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java index 36fb6f6caae66..9c297236a8e0b 100644 --- a/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java +++ b/modules/parent-join/src/main/java/org/elasticsearch/join/query/HasParentQueryBuilder.java @@ -181,7 +181,7 @@ protected Query doToQuery(SearchExecutionContext context) throws IOException { Query innerQuery = Queries.filtered(query.toQuery(context), parentFilter); Query childFilter = joiner.childrenFilter(parentType); MappedFieldType fieldType = context.getFieldType(joiner.childJoinField(parentType)); - final SortedSetOrdinalsIndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); + final SortedSetOrdinalsIndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); return new HasChildQueryBuilder.LateParsingQuery( childFilter, innerQuery, diff --git a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java index 0af9dad54dd67..5dcfd73e06d79 100644 --- a/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java +++ b/modules/percolator/src/main/java/org/elasticsearch/percolator/PercolateQueryBuilder.java @@ -640,11 +640,14 @@ public BitSetProducer bitsetFilter(Query query) { @Override @SuppressWarnings("unchecked") - public > IFD getForField(MappedFieldType fieldType, MappedFieldType.FielddataType fielddataType) { + public > IFD getForField( + MappedFieldType fieldType, + MappedFieldType.FielddataOperation fielddataOperation + ) { IndexFieldData.Builder builder = fieldType.fielddataBuilder( delegate.getFullyQualifiedIndex().getName(), delegate::lookup, - fielddataType + fielddataOperation ); IndexFieldDataCache cache = new IndexFieldDataCache.None(); CircuitBreakerService circuitBreaker = new NoneCircuitBreakerService(); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java index 5accca81584a1..5a2162fa80f50 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/QueryBuilderStoreTests.java @@ -62,7 +62,7 @@ public void testStoringQueryBuilders() throws IOException { IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer()); config.setMergePolicy(NoMergePolicy.INSTANCE); BinaryFieldMapper fieldMapper = PercolatorFieldMapper.Builder.createQueryBuilderFieldBuilder(MapperBuilderContext.ROOT); - MappedFieldType.FielddataType fielddataType = MappedFieldType.FielddataType.SEARCH; + MappedFieldType.FielddataOperation fielddataOperation = MappedFieldType.FielddataOperation.SEARCH; Version version = Version.CURRENT; try (IndexWriter indexWriter = new IndexWriter(directory, config)) { @@ -78,7 +78,7 @@ public void testStoringQueryBuilders() throws IOException { when(searchExecutionContext.indexVersionCreated()).thenReturn(version); when(searchExecutionContext.getWriteableRegistry()).thenReturn(writableRegistry()); when(searchExecutionContext.getParserConfig()).thenReturn(parserConfig()); - when(searchExecutionContext.getForField(fieldMapper.fieldType(), fielddataType)).thenReturn( + when(searchExecutionContext.getForField(fieldMapper.fieldType(), fielddataOperation)).thenReturn( new BytesBinaryIndexFieldData(fieldMapper.name(), CoreValuesSourceType.KEYWORD) ); when(searchExecutionContext.getFieldType(Mockito.anyString())).thenAnswer(invocation -> { diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index bca89333b0a14..3976821cadb43 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -111,7 +111,7 @@ protected String parseSourceValue(Object value) { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder( diff --git a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java index e89ef33fc8fb6..f8cd6eef47015 100644 --- a/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java +++ b/plugins/mapper-murmur3/src/main/java/org/elasticsearch/index/mapper/murmur3/Murmur3FieldMapper.java @@ -92,7 +92,7 @@ public String typeName() { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, Murmur3DocValueField::new); diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index 6cd3946a01308..34603c2cd3843 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -57,7 +57,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (hasDocValues() == false) { return (lookup, ignoredValues) -> List.of(); } - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataType.SEARCH)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); } } diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index 578fdfb67c92d..89014f9b3006c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -205,7 +205,7 @@ public boolean hasPrimarySortOnField(String field) { */ public Sort buildIndexSort( Function fieldTypeLookup, - TriFunction, MappedFieldType.FielddataType, IndexFieldData> fieldDataLookup + TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup ) { if (hasIndexSort() == false) { return null; @@ -250,7 +250,7 @@ public Sort buildIndexSort( fieldData = fieldDataLookup.apply( ft, () -> { throw new UnsupportedOperationException("index sorting not supported on runtime field [" + ft.name() + "]"); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ); } catch (Exception e) { throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]", e); diff --git a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java index 098779aa81f8d..f81d0de524037 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexWarmer.java +++ b/server/src/main/java/org/elasticsearch/index/IndexWarmer.java @@ -120,7 +120,7 @@ public TerminationHandle warmReader(final IndexShard indexShard, final Elasticse fieldType, indexFieldDataService.index().getName(), () -> { throw new UnsupportedOperationException("search lookup not available when warming an index"); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ); IndexFieldData global = ifd.loadGlobal(reader); if (reader.leaves().isEmpty() == false) { diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index 30be51a0069b6..f6cb8c530b488 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -102,10 +102,10 @@ public > IFD getForField( MappedFieldType fieldType, String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType fielddataType + MappedFieldType.FielddataOperation fielddataOperation ) { final String fieldName = fieldType.name(); - IndexFieldData.Builder builder = fieldType.fielddataBuilder(fullyQualifiedIndexName, searchLookup, fielddataType); + IndexFieldData.Builder builder = fieldType.fielddataBuilder(fullyQualifiedIndexName, searchLookup, fielddataOperation); IndexFieldDataCache cache; synchronized (this) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java index da383ddbd3902..ae2ba6442c7ab 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java @@ -173,7 +173,7 @@ protected final void applyScriptContext(SearchExecutionContext context) { @Override public final ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataType.SEARCH)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java index ac189e2f00d6e..f892a2ccf4370 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BinaryFieldMapper.java @@ -124,7 +124,7 @@ public BytesReference valueForDisplay(Object value) { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return new BytesBinaryIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java index 9808204c2f247..8ab90951aef73 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java @@ -257,7 +257,7 @@ public Boolean valueForDisplay(Object value) { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.BOOLEAN, BooleanDocValuesField::new); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java index 7dd93a01c4b32..7aa49b110e819 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/BooleanScriptFieldType.java @@ -105,7 +105,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { public BooleanScriptFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { return new BooleanScriptFieldData.Builder(name(), leafFactory(searchLookup.get()), BooleanDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 47e7b8f1b23a5..b7bf3aa11cb25 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -752,7 +752,7 @@ public Function pointReaderIfPossible() { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), resolution.numericType(), resolution.getDefaultToScriptFieldFactory()); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java index 6afea7fca5b22..d62180b70ac90 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateScriptFieldType.java @@ -159,7 +159,11 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { } @Override - public DateScriptFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier lookup, FielddataType type) { + public DateScriptFieldData.Builder fielddataBuilder( + String fullyQualifiedIndexName, + Supplier lookup, + FielddataOperation operation + ) { return new DateScriptFieldData.Builder(name(), leafFactory(lookup.get()), Resolution.MILLISECONDS.getDefaultToScriptFieldFactory()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java index 24ccd51f90d1e..e91d9dba3bd70 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DoubleScriptFieldType.java @@ -100,7 +100,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { public DoubleScriptFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { return new DoubleScriptFieldData.Builder(name(), leafFactory(searchLookup.get()), DoubleDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java index 6379053d6aab8..763b1c3420463 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java @@ -372,17 +372,17 @@ private boolean isPointGeometry(LatLonGeometry[] geometries) { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { - if (type == FielddataType.SEARCH) { + if (operation == FielddataOperation.SEARCH) { failIfNoDocValues(); } - if ((type == FielddataType.SEARCH || type == FielddataType.SCRIPT) && hasDocValues()) { + if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) { return new AbstractLatLonPointIndexFieldData.Builder(name(), CoreValuesSourceType.GEOPOINT, GeoPointDocValuesField::new); } - if (type == FielddataType.SCRIPT) { + if (operation == FielddataOperation.SCRIPT) { return new SourceValueFetcherMultiGeoPointIndexFieldData.Builder( name(), CoreValuesSourceType.GEOPOINT, @@ -392,7 +392,7 @@ public IndexFieldData.Builder fielddataBuilder( ); } - throw new IllegalStateException("unknown field data type [" + type.name() + "]"); + throw new IllegalStateException("unknown field data type [" + operation.name() + "]"); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java index 0d73b3ba77d2f..3175da90d8557 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java @@ -97,7 +97,7 @@ public Query termQuery(Object value, SearchExecutionContext context) { public GeoPointScriptFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { return new GeoPointScriptFieldData.Builder(name(), leafFactory(searchLookup.get()), GeoPointDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java index dfaeeb5f0249d..b8079d1c21c3e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java @@ -67,7 +67,7 @@ public Query existsQuery(SearchExecutionContext context) { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { return new ConstantIndexFieldData.Builder( fullyQualifiedIndexName, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 037599669d9cf..0ce13b7d68e08 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -394,7 +394,7 @@ public static Query rangeQuery( public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.IP, IpDocValuesField::new); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java index efd7dd3cc87b4..ccb860bdebe92 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpScriptFieldType.java @@ -94,7 +94,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { public IpScriptFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { return new IpScriptFieldData.Builder(name(), leafFactory(searchLookup.get()), IpDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 6099c666b4ca5..b620691a41887 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -685,13 +685,13 @@ NamedAnalyzer normalizer() { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { - if (type == FielddataType.SEARCH) { + if (operation == FielddataOperation.SEARCH) { failIfNoDocValues(); } - if ((type == FielddataType.SEARCH || type == FielddataType.SCRIPT) && hasDocValues()) { + if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) { return new SortedSetOrdinalsIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, @@ -699,7 +699,7 @@ public IndexFieldData.Builder fielddataBuilder( ); } - if (type == FielddataType.SCRIPT) { + if (operation == FielddataOperation.SCRIPT) { return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, @@ -719,7 +719,7 @@ protected String parseSourceValue(Object value) { ); } - throw new IllegalStateException("unknown field data type [" + type.name() + "]"); + throw new IllegalStateException("unknown field data type [" + operation.name() + "]"); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java index 691d7a10d2925..47c3bed6d230e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordScriptFieldType.java @@ -103,7 +103,7 @@ public Object valueForDisplay(Object value) { public StringScriptFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { return new StringScriptFieldData.Builder(name(), leafFactory(searchLookup.get()), KeywordDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java index 668c9561328d0..22af14927b20c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/LongScriptFieldType.java @@ -95,7 +95,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { public LongScriptFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { return new LongScriptFieldData.Builder(name(), leafFactory(searchLookup.get()), LongDocValuesField::new); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index d70c2e459e158..af7fa1aa348bb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -82,10 +82,10 @@ public MappedFieldType( } /** - * Type to specify what data structures are used to retrieve + * Operation to specify what data structures are used to retrieve * field data from and generate a representation of doc values. */ - public enum FielddataType { + public enum FielddataOperation { SEARCH, SCRIPT } @@ -95,7 +95,7 @@ public enum FielddataType { * * @param fullyQualifiedIndexName the name of the index this field-data is build for * @param searchLookup a {@link SearchLookup} supplier to allow for accessing other fields values in the context of runtime fields - * @param options + * @param operation Specifies the operation such as search or script to determine the appropriate fielddata to retrieve * @throws IllegalArgumentException if the fielddata is not supported on this type. * An IllegalArgumentException is needed in order to return an http error 400 * when this error occurs in a request. see: {@link org.elasticsearch.ExceptionsHelper#status} @@ -103,7 +103,7 @@ public enum FielddataType { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType options + FielddataOperation operation ) { throw new IllegalArgumentException("Fielddata is not supported on field [" + name() + "] of type [" + typeName() + "]"); } @@ -186,7 +186,11 @@ public Function pointReaderIfPossible() { */ public boolean isAggregatable() { try { - fielddataBuilder("", () -> { throw new UnsupportedOperationException("SearchLookup not available"); }, FielddataType.SEARCH); + fielddataBuilder( + "", + () -> { throw new UnsupportedOperationException("SearchLookup not available"); }, + FielddataOperation.SEARCH + ); return true; } catch (IllegalArgumentException e) { return false; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 6d41a01b04b4b..e4a5869811f01 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1426,17 +1426,17 @@ public Function pointReaderIfPossible() { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { - if (type == FielddataType.SEARCH) { + if (operation == FielddataOperation.SEARCH) { failIfNoDocValues(); } - if ((type == FielddataType.SEARCH || type == FielddataType.SCRIPT) && hasDocValues()) { + if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) { return this.type.getFieldDataBuilder(name()); } - if (type == FielddataType.SCRIPT) { + if (operation == FielddataOperation.SCRIPT) { return this.type.getValueFetcherFieldDataBuilder( name(), searchLookup.get().source(), @@ -1452,7 +1452,7 @@ protected Object parseSourceValue(Object value) { ); } - throw new IllegalStateException("unknown field data type [" + type.name() + "]"); + throw new IllegalStateException("unknown field data type [" + operation.name() + "]"); } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java index d77271ceaadaf..1abac7ed6d495 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/PlaceHolderFieldMapper.java @@ -257,7 +257,7 @@ public IntervalsSource wildcardIntervals(BytesRef pattern, SearchExecutionContex public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { throw new IllegalArgumentException(fail("aggregation or sorts")); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java index 82ed12212113b..99244eeac9f7f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapper.java @@ -140,7 +140,7 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { if (fieldDataEnabled.getAsBoolean() == false) { throw new IllegalArgumentException( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java index f3c57644f1e32..b2e4d5a745a2c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RangeFieldMapper.java @@ -223,7 +223,7 @@ public RangeType rangeType() { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return new BinaryIndexFieldData.Builder(name(), CoreValuesSourceType.RANGE); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java index c1851864657b2..501aca98c1f2f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/SeqNoFieldMapper.java @@ -204,7 +204,7 @@ public Query rangeQuery( public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, SeqNoDocValuesField::new); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index a9b99b0cbacfe..c41d295c14e89 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -897,7 +897,7 @@ public static boolean hasGaps(TokenStream stream) throws IOException { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { if (fielddata == false) { throw new IllegalArgumentException( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java index 44871c14083eb..2d3140f672bf8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TimeSeriesIdFieldMapper.java @@ -104,7 +104,7 @@ public String typeName() { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataType.SEARCH)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); } @Override @@ -119,7 +119,7 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); // TODO don't leak the TSID's binary format into the script diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java index b3f02bf81fd2a..f5e57bff79966 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TsidExtractingIdFieldMapper.java @@ -101,7 +101,7 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { throw new IllegalArgumentException("Fielddata is not supported on [_id] field in [time_series] indices"); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java index 8f960bb0c1345..35bf2684e7e6e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/VersionFieldMapper.java @@ -52,14 +52,14 @@ public Query termQuery(Object value, SearchExecutionContext context) { @Override public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { - return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataType.SEARCH)); + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); } @Override public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return new SortedNumericIndexFieldData.Builder(name(), NumericType.LONG, VersionDocValuesField::new); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index ba528e3b08a73..1da8fc2629705 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -355,7 +355,7 @@ public BytesRef indexedValueForSearch(Object value) { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return new KeyedFlattenedFieldData.Builder(name(), key, (dv, n) -> new FlattenedDocValuesField(FieldData.toString(dv), n)); @@ -640,7 +640,7 @@ public Object valueForDisplay(Object value) { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return new SortedSetOrdinalsIndexFieldData.Builder( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java index 70873793a3d87..3738fdd0949d1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldMapper.java @@ -282,7 +282,7 @@ public boolean isAggregatable() { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - FielddataType type + FielddataOperation operation ) { return new VectorIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, indexVersionCreated, dims, indexed); } diff --git a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java index 822b3ffe85d6d..a008447c15d7e 100644 --- a/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/SearchExecutionContext.java @@ -37,7 +37,7 @@ import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.MappedFieldType.FielddataType; +import org.elasticsearch.index.mapper.MappedFieldType.FielddataOperation; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperParsingException; @@ -92,7 +92,7 @@ public class SearchExecutionContext extends QueryRewriteContext { private final MappingLookup mappingLookup; private final SimilarityService similarityService; private final BitsetFilterCache bitsetFilterCache; - private final QuadFunction, FielddataType, IndexFieldData> indexFieldDataLookup; + private final QuadFunction, FielddataOperation, IndexFieldData> indexFieldDataLookup; private SearchLookup lookup = null; private final int shardId; @@ -122,7 +122,7 @@ public SearchExecutionContext( int shardRequestIndex, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, - QuadFunction, FielddataType, IndexFieldData> indexFieldDataLookup, + QuadFunction, FielddataOperation, IndexFieldData> indexFieldDataLookup, MapperService mapperService, MappingLookup mappingLookup, SimilarityService similarityService, @@ -195,7 +195,7 @@ private SearchExecutionContext( int shardRequestIndex, IndexSettings indexSettings, BitsetFilterCache bitsetFilterCache, - QuadFunction, FielddataType, IndexFieldData> indexFieldDataLookup, + QuadFunction, FielddataOperation, IndexFieldData> indexFieldDataLookup, MapperService mapperService, MappingLookup mappingLookup, SimilarityService similarityService, @@ -279,12 +279,12 @@ public boolean allowExpensiveQueries() { } @SuppressWarnings("unchecked") - public > IFD getForField(MappedFieldType fieldType, FielddataType fielddataType) { + public > IFD getForField(MappedFieldType fieldType, FielddataOperation fielddataOperation) { return (IFD) indexFieldDataLookup.apply( fieldType, fullyQualifiedIndex.getName(), () -> this.lookup().forkAndTrackFieldReferences(fieldType.name()), - fielddataType + fielddataOperation ); } diff --git a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java index 14993d908f167..d6d31b03a8f93 100644 --- a/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/TermsSetQueryBuilder.java @@ -266,7 +266,7 @@ private LongValuesSource createValuesSource(SearchExecutionContext context) { throw new QueryShardException(context, "failed to find minimum_should_match field [" + minimumShouldMatchField + "]"); } - IndexNumericFieldData fieldData = context.getForField(msmFieldType, MappedFieldType.FielddataType.SEARCH); + IndexNumericFieldData fieldData = context.getForField(msmFieldType, MappedFieldType.FielddataOperation.SEARCH); longValuesSource = new FieldValuesSource(fieldData); } else if (minimumShouldMatchScript != null) { TermsSetQueryScript.Factory factory = context.compile(minimumShouldMatchScript, TermsSetQueryScript.CONTEXT); diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java index d2874784d0b0d..efc6f6729f822 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java @@ -260,7 +260,7 @@ private AbstractDistanceScoreFunction parseNumberVariable( DecayFunctionBuilder.ORIGIN ); } - IndexNumericFieldData numericFieldData = context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); + IndexNumericFieldData numericFieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); return new NumericFieldDataScoreFunction(origin, scale, decay, offset, getDecayFunction(), numericFieldData, mode); } @@ -300,7 +300,7 @@ private AbstractDistanceScoreFunction parseGeoVariable( } double scale = DistanceUnit.DEFAULT.parse(scaleString, DistanceUnit.DEFAULT); double offset = DistanceUnit.DEFAULT.parse(offsetString, DistanceUnit.DEFAULT); - IndexGeoPointFieldData indexFieldData = context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); + IndexGeoPointFieldData indexFieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); return new GeoFieldDataScoreFunction(origin, scale, decay, offset, getDecayFunction(), indexFieldData, mode); } @@ -350,7 +350,7 @@ private AbstractDistanceScoreFunction parseDateVariable( double scale = val.getMillis(); val = TimeValue.parseTimeValue(offsetString, TimeValue.timeValueHours(24), DecayFunctionParser.class.getSimpleName() + ".offset"); double offset = val.getMillis(); - IndexNumericFieldData numericFieldData = context.getForField(dateFieldType, MappedFieldType.FielddataType.SEARCH); + IndexNumericFieldData numericFieldData = context.getForField(dateFieldType, MappedFieldType.FielddataOperation.SEARCH); return new NumericFieldDataScoreFunction(origin, scale, decay, offset, getDecayFunction(), numericFieldData, mode); } diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java index 7c20b866543be..50c765d4eba97 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/FieldValueFactorFunctionBuilder.java @@ -141,7 +141,7 @@ public Version getMinimalSupportedVersion() { protected ScoreFunction doToFunction(SearchExecutionContext context) { IndexNumericFieldData fieldData = null; if (context.isFieldMapped(field)) { - fieldData = context.getForField(context.getFieldType(field), MappedFieldType.FielddataType.SEARCH); + fieldData = context.getForField(context.getFieldType(field), MappedFieldType.FielddataOperation.SEARCH); } else { if (missing == null) { throw new ElasticsearchException("Unable to find a field mapper for field [" + field + "]. No 'missing' value defined."); diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java index 11662be3c8880..1cbe92ea4c458 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/RandomScoreFunctionBuilder.java @@ -170,7 +170,7 @@ protected ScoreFunction doToFunction(SearchExecutionContext context) { return new RandomScoreFunction( seed, salt, - context.getForField(context.getFieldType(fieldName), MappedFieldType.FielddataType.SEARCH) + context.getForField(context.getFieldType(fieldName), MappedFieldType.FielddataOperation.SEARCH) ); } } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java index 671022567ca12..df5ede180ad30 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/support/AggregationContext.java @@ -435,7 +435,7 @@ public Analyzer buildCustomAnalyzer( @Override protected IndexFieldData buildFieldData(MappedFieldType ft) { - return context.getForField(ft, MappedFieldType.FielddataType.SEARCH); + return context.getForField(ft, MappedFieldType.FielddataOperation.SEARCH); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java index d43ee2d8b0df7..07d8a2a872984 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FetchDocValuesPhase.java @@ -46,7 +46,7 @@ public FetchSubPhaseProcessor getProcessor(FetchContext context) { } ValueFetcher fetcher = new DocValueFetcher( ft.docValueFormat(fieldAndFormat.format, null), - context.searchLookup().getForField(ft, MappedFieldType.FielddataType.SEARCH) + context.searchLookup().getForField(ft, MappedFieldType.FielddataOperation.SEARCH) ); fields.add(new DocValueField(fieldAndFormat.field, fetcher)); } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java index 241a21da7c391..93fbfc1a5b43c 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java @@ -29,7 +29,7 @@ public class LeafDocLookup implements Map> { private final Function fieldTypeLookup; - private final BiFunction> fieldDataLookup; + private final BiFunction> fieldDataLookup; private final LeafReaderContext reader; private int docId = -1; @@ -38,7 +38,7 @@ public class LeafDocLookup implements Map> { LeafDocLookup( Function fieldTypeLookup, - BiFunction> fieldDataLookup, + BiFunction> fieldDataLookup, LeafReaderContext reader ) { this.fieldTypeLookup = fieldTypeLookup; @@ -50,11 +50,12 @@ public void setDocument(int docId) { this.docId = docId; } - protected DocValuesScriptFieldFactory getScriptFieldFactory(String fieldName, MappedFieldType.FielddataType options) { + protected DocValuesScriptFieldFactory getScriptFieldFactory(String fieldName, MappedFieldType.FielddataOperation options) { DocValuesScriptFieldFactory factory = localCacheScriptFieldData.get(fieldName); // do not use cached source fallback fields for old style doc access - if (options == MappedFieldType.FielddataType.SEARCH && factory instanceof SourceValueFetcherIndexFieldData.ValueFetcherDocValues) { + if (options == MappedFieldType.FielddataOperation.SEARCH + && factory instanceof SourceValueFetcherIndexFieldData.ValueFetcherDocValues) { factory = null; } @@ -87,12 +88,12 @@ public DocValuesScriptFieldFactory run() { } public Field getScriptField(String fieldName) { - return getScriptFieldFactory(fieldName, MappedFieldType.FielddataType.SCRIPT).toScriptField(); + return getScriptFieldFactory(fieldName, MappedFieldType.FielddataOperation.SCRIPT).toScriptField(); } @Override public ScriptDocValues get(Object key) { - return getScriptFieldFactory(key.toString(), MappedFieldType.FielddataType.SEARCH).toScriptDocValues(); + return getScriptFieldFactory(key.toString(), MappedFieldType.FielddataOperation.SEARCH).toScriptDocValues(); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index ac65c8785aef1..130cdbca7a9e8 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -41,7 +41,11 @@ public class SearchLookup { private final Set fieldChain; private final SourceLookup sourceLookup; private final Function fieldTypeLookup; - private final TriFunction, MappedFieldType.FielddataType, IndexFieldData> fieldDataLookup; + private final TriFunction< + MappedFieldType, + Supplier, + MappedFieldType.FielddataOperation, + IndexFieldData> fieldDataLookup; private final Function> sourcePathsLookup; /** @@ -50,7 +54,7 @@ public class SearchLookup { */ public SearchLookup( Function fieldTypeLookup, - TriFunction, MappedFieldType.FielddataType, IndexFieldData> fieldDataLookup, + TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup, Function> sourcePathsLookup ) { this.fieldTypeLookup = fieldTypeLookup; @@ -109,7 +113,7 @@ public MappedFieldType fieldType(String fieldName) { return fieldTypeLookup.apply(fieldName); } - public IndexFieldData getForField(MappedFieldType fieldType, MappedFieldType.FielddataType options) { + public IndexFieldData getForField(MappedFieldType fieldType, MappedFieldType.FielddataOperation options) { return fieldDataLookup.apply(fieldType, () -> forkAndTrackFieldReferences(fieldType.name()), options); } diff --git a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java index 89b096e070dae..895aa5bed5bff 100644 --- a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java @@ -260,7 +260,7 @@ private Query createSliceQuery(int id, int max, SearchExecutionContext context, if (type.hasDocValues() == false) { throw new IllegalArgumentException("cannot load numeric doc values on " + field); } else { - IndexFieldData ifm = context.getForField(type, MappedFieldType.FielddataType.SEARCH); + IndexFieldData ifm = context.getForField(type, MappedFieldType.FielddataOperation.SEARCH); if (ifm instanceof IndexNumericFieldData == false) { throw new IllegalArgumentException("cannot load numeric doc values on " + field); } diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index 05dac56754cfc..19326a214a940 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -362,7 +362,7 @@ public SortFieldAndFormat build(SearchExecutionContext context) throws IOExcepti fieldType = resolveUnmappedType(context); } - IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); + IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); if (fieldData instanceof IndexNumericFieldData == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) { throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields"); @@ -462,7 +462,7 @@ public BucketedSort buildBucketedSort(SearchExecutionContext context, BigArrays fieldType = resolveUnmappedType(context); } - IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); + IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); if (fieldData instanceof IndexNumericFieldData == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) { throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields"); diff --git a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java index 0d2625ec7ee9f..4d842da11163d 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java @@ -616,7 +616,7 @@ private IndexGeoPointFieldData fieldData(SearchExecutionContext context) { throw new IllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort"); } } - return context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); + return context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); } private Nested nested(SearchExecutionContext context) throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index bff019a76f7a8..3d85510101802 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -135,7 +135,7 @@ public String typeName() { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { searchLookup.get(); return null; diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 3c2c1277f71ab..b9542dd4be315 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -157,7 +157,7 @@ public > IFD getForField(String type, String field } else { throw new UnsupportedOperationException(type); } - return searchExecutionContext.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); + return searchExecutionContext.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); } @Before diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java index c5b06adf18495..f7fddd9111d0c 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java @@ -56,7 +56,7 @@ public void testFilterByFrequency() throws Exception { .fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0) .build(builderContext) .fieldType(); - IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataType.SEARCH); + IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataOperation.SEARCH); for (LeafReaderContext context : contexts) { LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); @@ -71,7 +71,7 @@ public void testFilterByFrequency() throws Exception { .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, 201, 100) .build(builderContext) .fieldType(); - IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataType.SEARCH); + IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataOperation.SEARCH); for (LeafReaderContext context : contexts) { LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); @@ -86,7 +86,7 @@ public void testFilterByFrequency() throws Exception { .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101) .build(builderContext) .fieldType(); - IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataType.SEARCH); + IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataOperation.SEARCH); for (LeafReaderContext context : contexts) { LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); @@ -102,7 +102,7 @@ public void testFilterByFrequency() throws Exception { .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d / 200.0d, Integer.MAX_VALUE, 101) .build(builderContext) .fieldType(); - IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataType.SEARCH); + IndexOrdinalsFieldData fieldData = searchExecutionContext.getForField(ft, MappedFieldType.FielddataOperation.SEARCH); for (LeafReaderContext context : contexts) { LeafOrdinalsFieldData loadDirect = fieldData.loadDirect(context); SortedSetDocValues bytesValues = loadDirect.getOrdinalsValues(); diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java index 15f18eaf1a74f..882a0e2736db3 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java @@ -84,7 +84,7 @@ public void testGetForFieldDefaults() { stringMapper, "test", () -> { throw new UnsupportedOperationException(); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ); assertTrue(fd instanceof SortedSetOrdinalsIndexFieldData); @@ -99,7 +99,7 @@ public void testGetForFieldDefaults() { mapper, "test", () -> { throw new UnsupportedOperationException(); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ); assertTrue(fd instanceof SortedNumericIndexFieldData); } @@ -117,7 +117,7 @@ public void testGetForFieldDefaults() { floatMapper, "test", () -> { throw new UnsupportedOperationException(); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ); assertTrue(fd instanceof SortedDoublesIndexFieldData); @@ -134,7 +134,7 @@ public void testGetForFieldDefaults() { doubleMapper, "test", () -> { throw new UnsupportedOperationException(); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ); assertTrue(fd instanceof SortedDoublesIndexFieldData); } @@ -149,7 +149,7 @@ public void testGetForFieldRuntimeField() { ); final SetOnce> searchLookupSetOnce = new SetOnce<>(); MappedFieldType ft = mock(MappedFieldType.class); - MappedFieldType.FielddataType fdt = MappedFieldType.FielddataType.SEARCH; + MappedFieldType.FielddataOperation fdt = MappedFieldType.FielddataOperation.SEARCH; when(ft.fielddataBuilder(ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.any())).thenAnswer(invocationOnMock -> { @SuppressWarnings("unchecked") Supplier searchLookup = (Supplier) invocationOnMock.getArguments()[1]; @@ -201,13 +201,13 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon mapper1, "test", () -> { throw new UnsupportedOperationException(); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ); IndexFieldData ifd2 = ifdService.getForField( mapper2, "test", () -> { throw new UnsupportedOperationException(); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); LeafFieldData loadField1 = ifd1.load(leafReaderContext); @@ -286,7 +286,7 @@ public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, lon mapper1, "test", () -> { throw new UnsupportedOperationException(); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ); LeafReaderContext leafReaderContext = reader.getContext().leaves().get(0); LeafFieldData load = ifd.load(leafReaderContext); @@ -350,7 +350,12 @@ private void doTestRequireDocValues(MappedFieldType ft) { ); if (ft.hasDocValues()) { // no exception - ifds.getForField(ft, "test", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataType.SEARCH); + ifds.getForField( + ft, + "test", + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataOperation.SEARCH + ); } else { IllegalArgumentException e = expectThrows( IllegalArgumentException.class, @@ -358,7 +363,7 @@ private void doTestRequireDocValues(MappedFieldType ft) { ft, "test", () -> { throw new UnsupportedOperationException(); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ) ); assertThat(e.getMessage(), containsString("doc values")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java index 1be8ba71a0d30..8595dfca873ca 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java @@ -68,7 +68,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); BooleanScriptFieldType ft = simpleMappedFieldType(); - BooleanScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataType.SEARCH) + BooleanScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataOperation.SEARCH) .build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override @@ -109,7 +109,7 @@ public void testSort() throws IOException { BooleanScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( "test", mockContext()::lookup, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java index d79fcee75273e..40b10f46b0c4b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java @@ -150,7 +150,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); DateScriptFieldType ft = build("add_days", Map.of("days", 1)); - DateScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataType.SEARCH) + DateScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataOperation.SEARCH) .build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override @@ -192,7 +192,7 @@ public void testSort() throws IOException { DateScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( "test", mockContext()::lookup, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java index 43ab5030ab517..195af67e399e8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java @@ -65,7 +65,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); DoubleScriptFieldType ft = build("add_param", Map.of("param", 1)); - DoubleScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataType.SEARCH) + DoubleScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataOperation.SEARCH) .build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override @@ -107,7 +107,7 @@ public void testSort() throws IOException { DoubleScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( "test", mockContext()::lookup, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java index 043caf1aeae35..3488ce7395453 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java @@ -65,7 +65,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); GeoPointScriptFieldType ft = build("fromLatLon", Map.of()); - GeoPointScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataType.SEARCH) + GeoPointScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataOperation.SEARCH) .build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override @@ -102,7 +102,7 @@ public void testSort() throws IOException { GeoPointScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( "test", mockContext()::lookup, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ).build(null, null); Exception e = expectThrows(IllegalArgumentException.class, () -> ifd.sortField(null, MultiValueMode.MIN, null, false)); assertThat(e.getMessage(), equalTo("can't sort on geo_point field without using specific sorting feature, like geo_distance")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java index 8ee150123cdf4..a98672d9c102f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java @@ -67,7 +67,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); IpScriptFieldType ft = build("append_param", Map.of("param", ".1")); - BinaryScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataType.SEARCH) + BinaryScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataOperation.SEARCH) .build(null, null); DocValueFormat format = ft.docValueFormat(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @@ -110,7 +110,7 @@ public void testSort() throws IOException { BinaryScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( "test", mockContext()::lookup, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java index 1428464e81c9f..3de624fab073a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java @@ -60,7 +60,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); KeywordScriptFieldType ft = build("append_param", Map.of("param", "-suffix")); - StringScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataType.SEARCH) + StringScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataOperation.SEARCH) .build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override @@ -102,7 +102,7 @@ public void testSort() throws IOException { BinaryScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( "test", mockContext()::lookup, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java index 1b10a7a9ac932..245d9efd92268 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java @@ -77,7 +77,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); LongScriptFieldType ft = build("add_param", Map.of("param", 1)); - LongScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataType.SEARCH) + LongScriptFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup, MappedFieldType.FielddataOperation.SEARCH) .build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override @@ -119,7 +119,7 @@ public void testSort() throws IOException { LongScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( "test", mockContext()::lookup, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); @@ -140,7 +140,7 @@ public void testNow() throws IOException { LongScriptFieldData ifd = build("millis_ago", Map.of()).fielddataBuilder( "test", mockContext()::lookup, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 5131422b9e827..08d9fb917b587 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -585,7 +585,7 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS IndexNumericFieldData fielddata = (IndexNumericFieldData) fieldType.fielddataBuilder( "index", () -> { throw new UnsupportedOperationException(); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ).build(null, null); SortField sortField = fielddata.sortField(null, MultiValueMode.MIN, null, randomBoolean()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java index d22dc1b29d43e..c5eb68881a435 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ProvidedIdFieldMapperTests.java @@ -56,14 +56,17 @@ public void testEnableFieldData() throws IOException { IllegalArgumentException exc = expectThrows( IllegalArgumentException.class, - () -> ft.fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataType.SEARCH) - .build(null, null) + () -> ft.fielddataBuilder( + "test", + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataOperation.SEARCH + ).build(null, null) ); assertThat(exc.getMessage(), containsString(IndicesService.INDICES_ID_FIELD_DATA_ENABLED_SETTING.getKey())); assertFalse(ft.isAggregatable()); enabled[0] = true; - ft.fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataType.SEARCH) + ft.fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataOperation.SEARCH) .build(null, null); assertWarnings(ProvidedIdFieldMapper.ID_FIELD_DATA_DEPRECATION_MESSAGE); assertTrue(ft.isAggregatable()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java index 1185d894c4b79..917b00ee0100e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldMapperTests.java @@ -530,14 +530,14 @@ public void testFielddata() throws IOException { Exception e = expectThrows( IllegalArgumentException.class, () -> disabledMapper.fieldType("field") - .fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataType.SEARCH) + .fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataOperation.SEARCH) ); assertThat(e.getMessage(), containsString("Text fields are not optimised for operations that require per-document field data")); MapperService enabledMapper = createMapperService(fieldMapping(b -> b.field("type", "text").field("fielddata", true))); enabledMapper.fieldType("field") // no exception this time - .fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataType.SEARCH); + .fielddataBuilder("test", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataOperation.SEARCH); e = expectThrows( MapperParsingException.class, @@ -1200,7 +1200,7 @@ public void testIgnoreFieldDataOnLegacyIndex() throws IOException { expectThrows( IllegalArgumentException.class, () -> ((TextFieldMapper) finalMapperService.documentMapper().mappers().getMapper("field")).fieldType() - .fielddataBuilder("test", null, MappedFieldType.FielddataType.SEARCH) + .fielddataBuilder("test", null, MappedFieldType.FielddataOperation.SEARCH) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java index e4dad1e9ae1e9..28d2cbee36f33 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedIndexFieldDataTests.java @@ -74,7 +74,7 @@ public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) { fieldType1, "test", () -> { throw new UnsupportedOperationException("search lookup not available"); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ); assertTrue(ifd1 instanceof KeyedFlattenedFieldData); @@ -89,7 +89,7 @@ public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) { fieldType2, "test", () -> { throw new UnsupportedOperationException("search lookup not available"); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ); assertTrue(ifd2 instanceof KeyedFlattenedFieldData); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java index 6540effdf2bc0..25f519101a992 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/DenseVectorFieldTypeTests.java @@ -54,7 +54,7 @@ public void testIsAggregatable() { public void testFielddataBuilder() { DenseVectorFieldType ft = createFieldType(); assertNotNull( - ft.fielddataBuilder("index", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataType.SEARCH) + ft.fielddataBuilder("index", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataOperation.SEARCH) ); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java index 7a11a694dfc06..a58be009f1b71 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldTypeTests.java @@ -18,7 +18,10 @@ public class SparseVectorFieldTypeTests extends FieldTypeTestCase { public void testDocValuesDisabled() { MappedFieldType fieldType = new SparseVectorFieldMapper.SparseVectorFieldType("field", Collections.emptyMap()); assertFalse(fieldType.hasDocValues()); - expectThrows(IllegalArgumentException.class, () -> fieldType.fielddataBuilder("index", null, MappedFieldType.FielddataType.SEARCH)); + expectThrows( + IllegalArgumentException.class, + () -> fieldType.fielddataBuilder("index", null, MappedFieldType.FielddataOperation.SEARCH) + ); } public void testIsNotAggregatable() { diff --git a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java index fe4045bb563a5..1ae7ae3b3b34f 100644 --- a/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/SearchExecutionContextTests.java @@ -149,7 +149,7 @@ public void testClusterAlias() throws IOException { IndexFieldMapper mapper = new IndexFieldMapper(); - IndexFieldData forField = context.getForField(mapper.fieldType(), MappedFieldType.FielddataType.SEARCH); + IndexFieldData forField = context.getForField(mapper.fieldType(), MappedFieldType.FielddataOperation.SEARCH); String expected = clusterAlias == null ? context.getIndexSettings().getIndexMetadata().getIndex().getName() : clusterAlias + ":" + context.getIndexSettings().getIndex().getName(); @@ -489,7 +489,7 @@ private static RuntimeField runtimeField(String name, BiFunction searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { return (cache, breakerService) -> new IndexFieldData<>() { @Override @@ -606,9 +606,9 @@ private static List collect(String field, SearchExecutionContext searchE MappedFieldType fieldType = searchExecutionContext.getFieldType(field); IndexFieldData indexFieldData; if (randomBoolean()) { - indexFieldData = searchExecutionContext.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); + indexFieldData = searchExecutionContext.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); } else { - indexFieldData = searchExecutionContext.lookup().getForField(fieldType, MappedFieldType.FielddataType.SEARCH); + indexFieldData = searchExecutionContext.lookup().getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); } searcher.search(query, new Collector() { @Override diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index c93e68c44be53..f47203af7588d 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -2669,7 +2669,7 @@ public void testReaderWrapperWorksWithGlobalOrdinals() throws IOException { foo, "test", () -> { throw new UnsupportedOperationException("search lookup not available"); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ); FieldDataStats before = shard.fieldData().stats("foo"); assertThat(before.getMemorySizeInBytes(), equalTo(0L)); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index b8ce869a8bfc2..c2ca6dcb550fb 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -1239,7 +1239,12 @@ private static SearchExecutionContext newSearchExecutionContext(MapperService ma private static SearchExecutionContext newSearchExecutionContext( MapperService mapperService, - QuadFunction, MappedFieldType.FielddataType, IndexFieldData> indexFieldDataLookup + QuadFunction< + MappedFieldType, + String, + Supplier, + MappedFieldType.FielddataOperation, + IndexFieldData> indexFieldDataLookup ) { Settings settings = Settings.builder() .put("index.version.created", Version.CURRENT) diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java index ddb2ffbcb42cd..7c8921e9123a0 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java @@ -77,7 +77,7 @@ public void testFlattenedField() throws IOException { MappedFieldType fieldType1 = fieldType.getChildFieldType("key1"); MappedFieldType fieldType2 = fieldType.getChildFieldType("key2"); - BiFunction> fieldDataSupplier = (ft, fdt) -> { + BiFunction> fieldDataSupplier = (ft, fdt) -> { FlattenedFieldMapper.KeyedFlattenedFieldType keyedFieldType = (FlattenedFieldMapper.KeyedFlattenedFieldType) ft; return keyedFieldType.key().equals("key1") ? fieldData1 : fieldData2; }; diff --git a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java index 6005d4ac84bf2..4bf7b4d766911 100644 --- a/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/slice/SliceBuilderTests.java @@ -171,7 +171,7 @@ public Query existsQuery(SearchExecutionContext context) { when(context.getIndexSettings()).thenReturn(indexSettings); if (dvType != null) { IndexNumericFieldData fd = mock(IndexNumericFieldData.class); - when(context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH)).thenReturn(fd); + when(context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH)).thenReturn(fd); } return context; diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 13c8f875b6c3f..0db4ba9e415ac 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -193,7 +193,7 @@ protected final SearchExecutionContext createMockSearchExecutionContext(IndexSea MappedFieldType, String, Supplier, - MappedFieldType.FielddataType, + MappedFieldType.FielddataOperation, IndexFieldData> indexFieldDataLookup = (fieldType, fieldIndexName, searchLookup, fielddataType) -> { IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup, fielddataType); return builder.build(new IndexFieldDataCache.None(), null); diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 48adae3dc4880..6e77e3deac823 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -450,7 +450,7 @@ public Query filterQuery(Query query) { @Override protected IndexFieldData buildFieldData(MappedFieldType ft) { - return ft.fielddataBuilder("test", null, MappedFieldType.FielddataType.SEARCH) + return ft.fielddataBuilder("test", null, MappedFieldType.FielddataOperation.SEARCH) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } @@ -644,7 +644,9 @@ public void onRemoval(ShardId shardId, Accountable accountable) { ); } - protected TriFunction, MappedFieldType.FielddataType, IndexFieldData> fieldDataLookup() { + protected + TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> + fieldDataLookup() { return (mft, lookupSource, fdt) -> mft.fielddataBuilder("test", lookupSource, fdt) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()); } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java index 9ae917f679c59..b1b1fc7144235 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperTestCase.java @@ -326,7 +326,7 @@ protected final List fetchFromDocValues(MapperService mapperService, MappedFi fieldDataLookup(), mapperService.mappingLookup()::sourcePaths ); - ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft, MappedFieldType.FielddataType.SEARCH)); + ValueFetcher valueFetcher = new DocValueFetcher(format, lookup.getForField(ft, MappedFieldType.FielddataOperation.SEARCH)); IndexSearcher searcher = newSearcher(iw); LeafReaderContext context = searcher.getIndexReader().leaves().get(0); lookup.source().setSegmentAndDocument(context, 0); @@ -576,11 +576,11 @@ protected void registerDimensionChecks(ParameterChecker checker) throws IOExcept */ protected void assertFetch(MapperService mapperService, String field, Object value, String format) throws IOException { MappedFieldType ft = mapperService.fieldType(field); - MappedFieldType.FielddataType fdt = MappedFieldType.FielddataType.SEARCH; + MappedFieldType.FielddataOperation fdt = MappedFieldType.FielddataOperation.SEARCH; SourceToParse source = source(b -> b.field(ft.name(), value)); ValueFetcher docValueFetcher = new DocValueFetcher( ft.docValueFormat(format, null), - ft.fielddataBuilder("test", () -> null, MappedFieldType.FielddataType.SEARCH) + ft.fielddataBuilder("test", () -> null, MappedFieldType.FielddataOperation.SEARCH) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) ); SearchExecutionContext searchExecutionContext = mock(SearchExecutionContext.class); @@ -662,7 +662,7 @@ public final void testIndexTimeFieldData() throws IOException { DocValuesScriptFieldFactory docValuesFieldSource = fieldType.fielddataBuilder( "test", () -> { throw new UnsupportedOperationException(); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ).build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()).load(ctx).getScriptFieldFactory("test"); docValuesFieldSource.setNextDocId(0); @@ -671,7 +671,7 @@ public final void testIndexTimeFieldData() throws IOException { DocValuesScriptFieldFactory indexData = fieldType.fielddataBuilder( "test", () -> { throw new UnsupportedOperationException(); }, - MappedFieldType.FielddataType.SEARCH + MappedFieldType.FielddataOperation.SEARCH ) .build(new IndexFieldDataCache.None(), new NoneCircuitBreakerService()) .load(reader.getContext()) diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 895b05c367aff..0778cbb958e9d 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -292,12 +292,16 @@ protected AggregationContext createAggregationContext( .map(ft -> new FieldAliasMapper(ft.name() + "-alias", ft.name() + "-alias", ft.name())) .collect(toList()) ); - QuadFunction, MappedFieldType.FielddataType, IndexFieldData> fieldDataBuilder = ( - fieldType, - s, - searchLookup, - fdt) -> fieldType.fielddataBuilder(indexSettings.getIndex().getName(), searchLookup, fdt) - .build(new IndexFieldDataCache.None(), breakerService); + QuadFunction< + MappedFieldType, + String, + Supplier, + MappedFieldType.FielddataOperation, + IndexFieldData> fieldDataBuilder = (fieldType, s, searchLookup, fdt) -> fieldType.fielddataBuilder( + indexSettings.getIndex().getName(), + searchLookup, + fdt + ).build(new IndexFieldDataCache.None(), breakerService); BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, new BitsetFilterCache.Listener() { @Override public void onRemoval(ShardId shardId, Accountable accountable) {} @@ -1052,9 +1056,11 @@ public void testSupportedFieldTypes() throws IOException { } private ValuesSourceType fieldToVST(MappedFieldType fieldType) { - return fieldType.fielddataBuilder("", () -> { throw new UnsupportedOperationException(); }, MappedFieldType.FielddataType.SEARCH) - .build(null, null) - .getValuesSourceType(); + return fieldType.fielddataBuilder( + "", + () -> { throw new UnsupportedOperationException(); }, + MappedFieldType.FielddataOperation.SEARCH + ).build(null, null).getValuesSourceType(); } /** diff --git a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java index 61c4c416f218d..7ccb69f56146c 100644 --- a/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java +++ b/x-pack/plugin/analytics/src/main/java/org/elasticsearch/xpack/analytics/mapper/HistogramFieldMapper.java @@ -172,7 +172,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return (cache, breakerService) -> new IndexHistogramFieldData(name(), AnalyticsValuesSourceType.HISTOGRAM) { diff --git a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java index 4a6cfc3c5b464..06a88ca6903c7 100644 --- a/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java +++ b/x-pack/plugin/mapper-aggregate-metric/src/main/java/org/elasticsearch/xpack/aggregatemetric/mapper/AggregateDoubleMetricFieldMapper.java @@ -394,7 +394,7 @@ public Relation isFieldWithinQuery( public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { return (cache, breakerService) -> new IndexAggregateDoubleMetricFieldData( name(), diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index ba1c507619736..7e73c8b82a686 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -135,7 +135,7 @@ public String familyTypeName() { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { return new ConstantIndexFieldData.Builder( value, diff --git a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java index c0e40c7db5bc7..be593ad39aeba 100644 --- a/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java +++ b/x-pack/plugin/mapper-unsigned-long/src/main/java/org/elasticsearch/xpack/unsignedlong/UnsignedLongFieldMapper.java @@ -291,7 +291,7 @@ public Query rangeQuery( public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return (cache, breakerService) -> { diff --git a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java index 7248f537c68c5..ea3e91af1851c 100644 --- a/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java +++ b/x-pack/plugin/mapper-version/src/main/java/org/elasticsearch/xpack/versionfield/VersionStringFieldMapper.java @@ -282,7 +282,7 @@ protected BytesRef indexedValueForSearch(Object value) { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.KEYWORD, VersionStringDocValuesField::new); } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java index bad322592acee..5a4e80e762b49 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/v2/FieldValueFetcher.java @@ -109,7 +109,7 @@ static List build(SearchExecutionContext context, String[] fi if (fieldType == null) { throw new IllegalArgumentException("Unknown field: [" + field + "]"); } - IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataType.SEARCH); + IndexFieldData fieldData = context.getForField(fieldType, MappedFieldType.FielddataOperation.SEARCH); fetchers.add(new FieldValueFetcher(field, fieldType, fieldData, getValidator(field))); } return Collections.unmodifiableList(fetchers); diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java index 2dac8f0cdd32b..925daab0a9245 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/index/mapper/GeoShapeWithDocValuesFieldMapper.java @@ -184,7 +184,7 @@ public GeoShapeWithDocValuesFieldType( public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return new AbstractLatLonShapeIndexFieldData.Builder(name(), GeoShapeValuesSourceType.instance(), GeoShapeDocValuesField::new); diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index f8f0d10371bcc..48a82c653a4c3 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -835,7 +835,7 @@ public Query termsQuery(Collection values, SearchExecutionContext context) { public IndexFieldData.Builder fielddataBuilder( String fullyQualifiedIndexName, Supplier searchLookup, - MappedFieldType.FielddataType type + FielddataOperation operation ) { failIfNoDocValues(); return (cache, breakerService) -> new StringBinaryIndexFieldData( diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 8e2f3dce0b11b..2680bdab7bf36 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -1079,7 +1079,7 @@ protected final SearchExecutionContext createMockContext() { MappedFieldType, String, Supplier, - MappedFieldType.FielddataType, + MappedFieldType.FielddataOperation, IndexFieldData> indexFieldDataLookup = (fieldType, fieldIndexName, searchLookup, fielddataType) -> { IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup, fielddataType); return builder.build(new IndexFieldDataCache.None(), null); From aa4c75ae01d9e3570cba5614d3dc863a0d4990fd Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Fri, 22 Jul 2022 08:56:52 -0700 Subject: [PATCH 17/23] Update docs/changelog/88735.yaml --- docs/changelog/88735.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/88735.yaml diff --git a/docs/changelog/88735.yaml b/docs/changelog/88735.yaml new file mode 100644 index 0000000000000..91c8d1c3f4417 --- /dev/null +++ b/docs/changelog/88735.yaml @@ -0,0 +1,5 @@ +pr: 88735 +summary: Add source fallback for keyword fields using operation +area: Mapping +type: enhancement +issues: [] From 02ee1e31df1565f5b52416ff8dd1c209e24ffdc0 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Fri, 22 Jul 2022 08:57:49 -0700 Subject: [PATCH 18/23] Delete docs/changelog/88735.yaml --- docs/changelog/88735.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/88735.yaml diff --git a/docs/changelog/88735.yaml b/docs/changelog/88735.yaml deleted file mode 100644 index 91c8d1c3f4417..0000000000000 --- a/docs/changelog/88735.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 88735 -summary: Add source fallback for keyword fields using operation -area: Mapping -type: enhancement -issues: [] From f341afac968d3dd8014a548ed1416758da646bee Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Wed, 27 Jul 2022 09:36:19 -0700 Subject: [PATCH 19/23] add sourceValueFetcher methods for keyword and number field types --- .../index/mapper/KeywordFieldMapper.java | 18 +++++---------- .../index/mapper/NumberFieldMapper.java | 22 ++++++------------- 2 files changed, 13 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 599485f4656f5..32bfda76d6d2c 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -705,17 +705,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext return new SourceValueFetcherSortedBinaryIndexFieldData.Builder( name(), CoreValuesSourceType.KEYWORD, - new SourceValueFetcher(sourcePaths, nullValue) { - @Override - protected String parseSourceValue(Object value) { - String keywordValue = value.toString(); - if (keywordValue.length() > ignoreAbove) { - return null; - } - - return normalizeValue(normalizer(), name(), keywordValue); - } - }, + sourceValueFetcher(sourcePaths), searchLookup.source(), KeywordDocValuesField::new ); @@ -732,7 +722,11 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return new SourceValueFetcher(name(), context, nullValue) { + return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + } + + private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { + return new SourceValueFetcher(sourcePaths, nullValue) { @Override protected String parseSourceValue(Object value) { String keywordValue = value.toString(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index ec06ca5e1f533..7c4b4384ddb7e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -1432,26 +1432,14 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext } if ((operation == FielddataOperation.SEARCH || operation == FielddataOperation.SCRIPT) && hasDocValues()) { - return this.type.getFieldDataBuilder(name()); + return type.getFieldDataBuilder(name()); } if (operation == FielddataOperation.SCRIPT) { SearchLookup searchLookup = fieldDataContext.lookupSupplier().get(); Set sourcePaths = fieldDataContext.sourcePathsLookup().apply(name()); - return this.type.getValueFetcherFieldDataBuilder( - name(), - searchLookup.source(), - new SourceValueFetcher(sourcePaths, nullValue) { - @Override - protected Object parseSourceValue(Object value) { - if (value.equals("")) { - return nullValue; - } - return NumberFieldType.this.type.parse(value, coerce); - } - } - ); + return type.getValueFetcherFieldDataBuilder(name(), searchLookup.source(), sourceValueFetcher(sourcePaths)); } throw new IllegalStateException("unknown field data type [" + operation.name() + "]"); @@ -1473,7 +1461,11 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) if (this.scriptValues != null) { return FieldValues.valueFetcher(this.scriptValues, context); } - return new SourceValueFetcher(name(), context, nullValue) { + return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); + } + + private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { + return new SourceValueFetcher(sourcePaths, nullValue) { @Override protected Object parseSourceValue(Object value) { if (value.equals("")) { From b5258ad25f7d1409c54d157fd3ae569d93f5427e Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Wed, 27 Jul 2022 12:42:58 -0700 Subject: [PATCH 20/23] fix bad merge --- .../index/mapper/AbstractScriptFieldType.java | 2 +- .../index/mapper/GeoPointScriptFieldType.java | 35 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java index ae2ba6442c7ab..187d148387dff 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java @@ -172,7 +172,7 @@ protected final void applyScriptContext(SearchExecutionContext context) { } @Override - public final ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH)); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java index a746049a05e38..900a2daaa74ff 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java @@ -16,6 +16,7 @@ import org.apache.lucene.search.Query; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeoUtils; +import org.elasticsearch.common.geo.GeometryFormatterFactory; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.unit.DistanceUnit; @@ -28,12 +29,16 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.field.GeoPointDocValuesField; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.search.runtime.GeoPointScriptFieldDistanceFeatureQuery; import org.elasticsearch.search.runtime.GeoPointScriptFieldExistsQuery; import org.elasticsearch.search.runtime.GeoPointScriptFieldGeoShapeQuery; +import java.io.IOException; import java.time.ZoneId; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.function.Function; @@ -175,4 +180,34 @@ public void execute() { }; }; } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + GeoPointFieldScript.LeafFactory leafFactory = leafFactory(context.lookup()); + Function, List> formatter = GeoPointFieldMapper.GeoPointFieldType.GEO_FORMATTER_FACTORY.getFormatter( + format != null ? format : GeometryFormatterFactory.GEOJSON, + p -> new org.elasticsearch.geometry.Point(p.getLon(), p.getLat()) + ); + return new ValueFetcher() { + private GeoPointFieldScript script; + + @Override + public void setNextReader(LeafReaderContext context) { + script = leafFactory.newInstance(context); + } + + @Override + public List fetchValues(SourceLookup lookup, List ignoredValues) throws IOException { + script.runForDoc(lookup.docId()); + if (script.count() == 0) { + return List.of(); + } + List points = new ArrayList<>(script.count()); + for (int i = 0; i < script.count(); i++) { + points.add(new GeoPoint(script.lats()[i], script.lons()[i])); + } + return formatter.apply(points); + } + }; + } } From 8055416898d7cd1a049ebea80b31b90dae93bc9a Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Thu, 28 Jul 2022 08:49:49 -0700 Subject: [PATCH 21/23] response to pr comments --- .../java/org/elasticsearch/index/IndexService.java | 5 +---- .../java/org/elasticsearch/index/IndexSortConfig.java | 7 +++---- .../index/fielddata/FieldDataContext.java | 2 +- .../elasticsearch/index/IndexSortSettingsTests.java | 6 +++++- .../index/mapper/AbstractScriptFieldTypeTestCase.java | 4 ++-- .../index/mapper/BooleanScriptFieldTypeTests.java | 7 ++----- .../index/mapper/DateScriptFieldTypeTests.java | 7 ++----- .../index/mapper/DoubleScriptFieldTypeTests.java | 7 ++----- .../index/mapper/GeoPointScriptFieldTypeTests.java | 7 ++----- .../index/mapper/IpScriptFieldTypeTests.java | 7 ++----- .../index/mapper/KeywordScriptFieldTypeTests.java | 7 ++----- .../index/mapper/LongScriptFieldTypeTests.java | 11 +++-------- 12 files changed, 27 insertions(+), 50 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 5ab78d254797c..efc1075db0084 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -206,10 +206,7 @@ public IndexService( this.indexSortSupplier = () -> indexSettings.getIndexSortConfig() .buildIndexSort( mapperService::fieldType, - (fieldType, searchLookup, fielddataOperation) -> indexFieldData.getForField( - fieldType, - FieldDataContext.noRuntimeFields("index sort") - ) + (fieldType, searchLookup) -> indexFieldData.getForField(fieldType, FieldDataContext.noRuntimeFields("index sort")) ); } else { this.indexSortSupplier = () -> null; diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index 89014f9b3006c..0cf09d0d49f77 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -13,7 +13,6 @@ import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.SortedSetSortField; import org.elasticsearch.Version; -import org.elasticsearch.common.TriFunction; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Setting; @@ -30,6 +29,7 @@ import java.util.EnumSet; import java.util.List; import java.util.Objects; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.Supplier; @@ -205,7 +205,7 @@ public boolean hasPrimarySortOnField(String field) { */ public Sort buildIndexSort( Function fieldTypeLookup, - TriFunction, MappedFieldType.FielddataOperation, IndexFieldData> fieldDataLookup + BiFunction, IndexFieldData> fieldDataLookup ) { if (hasIndexSort() == false) { return null; @@ -249,8 +249,7 @@ public Sort buildIndexSort( try { fieldData = fieldDataLookup.apply( ft, - () -> { throw new UnsupportedOperationException("index sorting not supported on runtime field [" + ft.name() + "]"); }, - MappedFieldType.FielddataOperation.SEARCH + () -> { throw new UnsupportedOperationException("index sorting not supported on runtime field [" + ft.name() + "]"); } ); } catch (Exception e) { throw new IllegalArgumentException("docvalues not found for index sort field:[" + sortSpec.field + "]", e); diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java b/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java index 30b6b739f42b5..3cfab4e599015 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/FieldDataContext.java @@ -41,7 +41,7 @@ public static FieldDataContext noRuntimeFields(String reason) { return new FieldDataContext( "", () -> { throw new UnsupportedOperationException("Runtime fields not supported for [" + reason + "]"); }, - null, + Set::of, MappedFieldType.FielddataOperation.SEARCH ); } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index c921e35e6d0c8..6cbfb53c3f584 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -32,6 +32,7 @@ import java.util.Collections; import java.util.Map; +import java.util.Set; import static org.elasticsearch.index.IndexSettingsTests.newIndexMeta; import static org.hamcrest.Matchers.arrayWithSize; @@ -217,7 +218,10 @@ private Sort buildIndexSort(IndexSettings indexSettings, Map indexFieldDataService.getForField(ft, new FieldDataContext("test", s, null, fdo)) + (ft, s) -> indexFieldDataService.getForField( + ft, + new FieldDataContext("test", s, Set::of, MappedFieldType.FielddataOperation.SEARCH) + ) ); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index e87790fadc71c..e120e2e912608 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -186,8 +186,8 @@ protected static SearchExecutionContext mockContext() { return mockContext(true); } - protected static FieldDataContext mockFielddataContext(MappedFieldType.FielddataOperation fielddataOperation) { - return new FieldDataContext("test", mockContext()::lookup, mockContext()::sourcePath, fielddataOperation); + protected static FieldDataContext mockFielddataContext() { + return new FieldDataContext("test", mockContext()::lookup, mockContext()::sourcePath, MappedFieldType.FielddataOperation.SEARCH); } protected static SearchExecutionContext mockContext(boolean allowExpensiveQueries) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java index 6fe51669e69fd..5588333465eee 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/BooleanScriptFieldTypeTests.java @@ -68,8 +68,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); BooleanScriptFieldType ft = simpleMappedFieldType(); - BooleanScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH)) - .build(null, null); + BooleanScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext()).build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override public ScoreMode scoreMode() { @@ -106,9 +105,7 @@ public void testSort() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [false]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - BooleanScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( - mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH) - ).build(null, null); + BooleanScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder(mockFielddataContext()).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [false]}")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java index 8a037d52a5322..8fa5b95c93b98 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateScriptFieldTypeTests.java @@ -150,8 +150,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); DateScriptFieldType ft = build("add_days", Map.of("days", 1)); - DateScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH)) - .build(null, null); + DateScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext()).build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override public ScoreMode scoreMode() { @@ -189,9 +188,7 @@ public void testSort() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181356]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - DateScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( - mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH) - ).build(null, null); + DateScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder(mockFielddataContext()).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); assertThat(readSource(reader, docs.scoreDocs[0].doc), equalTo("{\"timestamp\": [1595432181351]}")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java index 3a75ae57758ea..9b1fba8915e37 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DoubleScriptFieldTypeTests.java @@ -65,8 +65,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); DoubleScriptFieldType ft = build("add_param", Map.of("param", 1)); - DoubleScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH)) - .build(null, null); + DoubleScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext()).build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override public ScoreMode scoreMode() { @@ -104,9 +103,7 @@ public void testSort() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2.1]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - DoubleScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( - mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH) - ).build(null, null); + DoubleScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder(mockFielddataContext()).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [1.1]}")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java index 037a7aefc8613..0ae0796e2d28e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/GeoPointScriptFieldTypeTests.java @@ -65,8 +65,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); GeoPointScriptFieldType ft = build("fromLatLon", Map.of()); - GeoPointScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH)) - .build(null, null); + GeoPointScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext()).build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override public ScoreMode scoreMode() { @@ -99,9 +98,7 @@ public void collect(int doc) throws IOException { @Override public void testSort() throws IOException { - GeoPointScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( - mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH) - ).build(null, null); + GeoPointScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder(mockFielddataContext()).build(null, null); Exception e = expectThrows(IllegalArgumentException.class, () -> ifd.sortField(null, MultiValueMode.MIN, null, false)); assertThat(e.getMessage(), equalTo("can't sort on geo_point field without using specific sorting feature, like geo_distance")); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java index ea3100c1be2a6..e1eb5cad3f1c9 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IpScriptFieldTypeTests.java @@ -67,8 +67,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); IpScriptFieldType ft = build("append_param", Map.of("param", ".1")); - BinaryScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH)) - .build(null, null); + BinaryScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext()).build(null, null); DocValueFormat format = ft.docValueFormat(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override @@ -107,9 +106,7 @@ public void testSort() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.0.2\"]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - BinaryScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( - mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH) - ).build(null, null); + BinaryScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder(mockFielddataContext()).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); assertThat( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java index 8ffdba6b4d55f..309dcca1ba05f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordScriptFieldTypeTests.java @@ -60,8 +60,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); KeywordScriptFieldType ft = build("append_param", Map.of("param", "-suffix")); - StringScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH)) - .build(null, null); + StringScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext()).build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override public ScoreMode scoreMode() { @@ -99,9 +98,7 @@ public void testSort() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"b\"]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - BinaryScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( - mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH) - ).build(null, null); + BinaryScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder(mockFielddataContext()).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [\"a\"]}")); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java index 931bc91b2bbae..7ad0a18c9d1a7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/LongScriptFieldTypeTests.java @@ -77,8 +77,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); LongScriptFieldType ft = build("add_param", Map.of("param", 1)); - LongScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH)) - .build(null, null); + LongScriptFieldData ifd = ft.fielddataBuilder(mockFielddataContext()).build(null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override public ScoreMode scoreMode() { @@ -116,9 +115,7 @@ public void testSort() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [2]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - LongScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder( - mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH) - ).build(null, null); + LongScriptFieldData ifd = simpleMappedFieldType().fielddataBuilder(mockFielddataContext()).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); assertThat(reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), equalTo("{\"foo\": [1]}")); @@ -135,9 +132,7 @@ public void testNow() throws IOException { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181356]}")))); try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); - LongScriptFieldData ifd = build("millis_ago", Map.of()).fielddataBuilder( - mockFielddataContext(MappedFieldType.FielddataOperation.SEARCH) - ).build(null, null); + LongScriptFieldData ifd = build("millis_ago", Map.of()).fielddataBuilder(mockFielddataContext()).build(null, null); SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); assertThat(readSource(reader, docs.scoreDocs[0].doc), equalTo("{\"timestamp\": [1595432181356]}")); From 5b41bb2f6adf4a1a4508a6c3814feafe71185cbf Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Thu, 28 Jul 2022 09:20:24 -0700 Subject: [PATCH 22/23] fix tests --- .../index/mapper/AbstractScriptFieldTypeTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java index e120e2e912608..7ef32966aa131 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/AbstractScriptFieldTypeTestCase.java @@ -187,7 +187,7 @@ protected static SearchExecutionContext mockContext() { } protected static FieldDataContext mockFielddataContext() { - return new FieldDataContext("test", mockContext()::lookup, mockContext()::sourcePath, MappedFieldType.FielddataOperation.SEARCH); + return new FieldDataContext("test", mockContext()::lookup, mockContext()::sourcePath, MappedFieldType.FielddataOperation.SCRIPT); } protected static SearchExecutionContext mockContext(boolean allowExpensiveQueries) { From 2dd57a716660342f49b0c0e1fd8b3780892a1000 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Thu, 28 Jul 2022 09:45:07 -0700 Subject: [PATCH 23/23] Update docs/changelog/88735.yaml --- docs/changelog/88735.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/88735.yaml diff --git a/docs/changelog/88735.yaml b/docs/changelog/88735.yaml new file mode 100644 index 0000000000000..91c8d1c3f4417 --- /dev/null +++ b/docs/changelog/88735.yaml @@ -0,0 +1,5 @@ +pr: 88735 +summary: Add source fallback for keyword fields using operation +area: Mapping +type: enhancement +issues: []