From ee0394d78aea7a10dbc006f2113bac00b27b49fb Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Fri, 18 Dec 2020 18:50:23 +0100 Subject: [PATCH] Sort field tiebreaker for PIT (point in time) readers (#66093) (#66588) This commit introduces a new sort field called `_shard_doc` that can be used in conjunction with a PIT to consistently tiebreak identical sort values. The sort value is a numeric long that is composed of the ordinal of the shard (assigned by the coordinating node) and the internal Lucene document ID. These two values are consistent within a PIT so this sort criteria can be used as the tiebreaker of any search requests. Since this sort criteria is stable we'd like to add it automatically to any sorted search requests that use a PIT but we also need to expose it explicitly in order to be able to: * Reverse the order of the tiebreaking, useful to search "before" `search_after`. * Force the primary sort to use it in order to benefit from the `search_after` optimization when sorting by index order (to be released in Lucene 8.8. I plan to add the documentation and the automatic configuration for PIT in a follow up since this change is already big. Relates #56828 --- .../common/DisableGraphQueryTests.java | 2 +- .../action/PainlessExecuteAction.java | 10 ++- .../painless/NeedsScoreTests.java | 2 +- .../PercolatorFieldMapperTests.java | 2 +- .../PercolatorQuerySearchTests.java | 2 +- .../test/search/90_search_after.yml | 24 ++++++ .../TransportSimulateIndexTemplateAction.java | 2 +- ...TransportFieldCapabilitiesIndexAction.java | 4 +- .../action/search/SearchRequest.java | 11 +++ .../metadata/MetadataCreateIndexService.java | 6 +- .../metadata/MetadataIndexAliasesService.java | 4 +- .../MetadataIndexTemplateService.java | 2 +- .../elasticsearch/common/lucene/Lucene.java | 27 +++--- .../org/elasticsearch/index/IndexService.java | 26 +++++- .../index/query/QueryShardContext.java | 37 +++++++- .../search/DefaultSearchContext.java | 3 +- .../elasticsearch/search/SearchService.java | 4 +- .../search/internal/ShardSearchRequest.java | 27 +++--- .../search/slice/SliceBuilder.java | 4 +- .../search/sort/FieldSortBuilder.java | 13 ++- .../search/sort/ShardDocSortField.java | 85 ++++++++++++++++++ .../search/sort/SortBuilders.java | 8 ++ .../MetadataCreateIndexServiceTests.java | 2 +- .../common/lucene/LuceneTests.java | 8 +- .../fielddata/AbstractFieldDataTestCase.java | 2 +- .../index/mapper/DateFieldTypeTests.java | 6 +- .../mapper/FieldNamesFieldTypeTests.java | 2 +- .../index/mapper/IndexFieldTypeTests.java | 2 +- .../index/mapper/NumberFieldTypeTests.java | 2 +- .../index/mapper/RangeFieldTypeTests.java | 2 +- .../index/query/QueryShardContextTests.java | 3 +- .../index/query/RangeQueryRewriteTests.java | 6 +- .../index/search/MultiMatchQueryTests.java | 77 ++++++++++++++--- .../index/search/NestedHelperTests.java | 9 +- .../search/nested/NestedSortingTests.java | 2 +- .../search/DefaultSearchContextTests.java | 21 +++-- .../fetch/subphase/FieldFetcherTests.java | 2 +- .../highlight/HighlightBuilderTests.java | 2 +- .../rescore/QueryRescorerBuilderTests.java | 4 +- .../search/sort/AbstractSortTestCase.java | 2 +- .../search/sort/FieldSortBuilderTests.java | 17 ++++ .../search/sort/SortBuilderTests.java | 13 ++- .../AbstractSuggestionBuilderTestCase.java | 4 +- .../aggregations/AggregatorTestCase.java | 1 + .../test/AbstractBuilderTestCase.java | 2 +- .../elasticsearch/test/TestSearchContext.java | 2 +- .../xpack/core/search/PointInTimeIT.java | 86 +++++++++++++++++++ .../DocumentSubsetBitsetCacheTests.java | 2 +- ...ityIndexReaderWrapperIntegrationTests.java | 4 +- .../action/EnrichShardMultiSearchAction.java | 1 + .../job/RollupIndexerIndexingTests.java | 2 +- .../xpack/security/Security.java | 1 + .../mapper/WildcardFieldMapperTests.java | 2 +- 53 files changed, 491 insertions(+), 105 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java index c7ac35d3febce..b4d5ddc55d0ed 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java @@ -85,7 +85,7 @@ public void setup() { indexService = createIndex("test", settings, "t", "text_shingle", "type=text,analyzer=text_shingle", "text_shingle_unigram", "type=text,analyzer=text_shingle_unigram"); - shardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()); + shardContext = indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap()); // parsed queries for "text_shingle_unigram:(foo bar baz)" with query parsers // that ignores position length attribute diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 1ad7fd02389c6..98476f561ef8e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -577,8 +577,14 @@ private static Response prepareRamIndex(Request request, final IndexSearcher searcher = new IndexSearcher(indexReader); searcher.setQueryCache(null); final long absoluteStartMillis = System.currentTimeMillis(); - QueryShardContext context = - indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null, emptyMap()); + QueryShardContext context = indexService.newQueryShardContext( + 0, + 0, + searcher, + () -> absoluteStartMillis, + null, + emptyMap() + ); return handler.apply(context, indexReader.leaves().get(0)); } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java index ca885cdfdff6a..5630973d66448 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java @@ -47,7 +47,7 @@ public void testNeedsScores() { contexts.put(NumberSortScript.CONTEXT, Whitelist.BASE_WHITELISTS); PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts); - QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null, emptyMap()); + QueryShardContext shardContext = index.newQueryShardContext(0, 0, null, () -> 0, null, emptyMap()); NumberSortScript.Factory factory = service.compile(null, "1.2", NumberSortScript.CONTEXT, Collections.emptyMap()); NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 397b915856002..4072713403923 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -534,7 +534,7 @@ public void testQueryWithRewrite() throws Exception { XContentType.JSON)); BytesRef qbSource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue(); QueryShardContext shardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { + randomInt(20), 0, null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); PlainActionFuture future = new PlainActionFuture<>(); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java index 4851c5fdf06c6..f1115f40f195f 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java @@ -259,7 +259,7 @@ public void testRangeQueriesWithNow() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { long[] currentTime = new long[] {System.currentTimeMillis()}; QueryShardContext queryShardContext = - indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null, emptyMap()); + indexService.newQueryShardContext(0, 0, searcher, () -> currentTime[0], null, emptyMap()); BytesReference source = BytesReference.bytes(jsonBuilder().startObject() .field("field1", "value") diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml index 9f0273fbc0213..1728006518a12 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml @@ -228,3 +228,27 @@ - match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828740" } - match: {hits.hits.0.sort: [1571617804828740000] } + +--- +"_shard_doc sort": + - skip: + version: " - 7.11.99" + reason: _shard_doc sort was added in 7.12 + + - do: + indices.create: + index: test + - do: + index: + index: test + id: 1 + body: { id: 1, foo: bar, age: 18 } + + - do: + catch: /\[_shard_doc\] sort field cannot be used without \[point in time\]/ + search: + index: test + body: + size: 1 + sort: ["_shard_doc"] + search_after: [ 0L ] diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index d835a71b27ede..615ea1b48c674 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -182,7 +182,7 @@ public static Template resolveTemplate(final String matchingTemplate, final Stri resolvedAliases, tempClusterState.metadata(), aliasValidator, xContentRegistry, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()))); + tempIndexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap()))); Map aliasesByName = aliases.stream().collect( Collectors.toMap(AliasMetadata::getAlias, Function.identity())); diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index a4c624d0c9541..44168de15f07c 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -117,8 +117,8 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd final IndexShard indexShard = indexService.getShard(request.shardId().getId()); try (Engine.Searcher searcher = indexShard.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE)) { - final QueryShardContext queryShardContext = indexService.newQueryShardContext(shardId.id(), searcher, - request::nowInMillis, null, Collections.emptyMap()); + final QueryShardContext queryShardContext = indexService.newQueryShardContext(shardId.id(), 0, + searcher, request::nowInMillis, null, Collections.emptyMap()); if (canMatchShard(request, queryShardContext) == false) { return new FieldCapabilitiesIndexResponse(request.index(), Collections.emptyMap(), false); diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index 5da95412bf9ad..87117b14c2bee 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -34,6 +34,9 @@ import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.ShardDocSortField; import org.elasticsearch.tasks.TaskId; import java.io.IOException; @@ -300,6 +303,14 @@ public ActionRequestValidationException validate() { if (scroll) { validationException = addValidationError("using [point in time] is not allowed in a scroll context", validationException); } + } else if (source != null && source.sorts() != null) { + for (SortBuilder sortBuilder : source.sorts()) { + if (sortBuilder instanceof FieldSortBuilder + && ShardDocSortField.NAME.equals(((FieldSortBuilder) sortBuilder).getFieldName())) { + validationException = addValidationError("[" + FieldSortBuilder.SHARD_DOC_FIELD_NAME + + "] sort field cannot be used without [point in time]", validationException); + } + } } return validationException; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index a22cfb6045845..8b70e00a39748 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -496,7 +496,7 @@ private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState c MetadataIndexTemplateService.resolveAliases(templates), currentState.metadata(), aliasValidator, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), + xContentRegistry, indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap())), templates.stream().map(IndexTemplateMetadata::getName).collect(toList()), metadataTransformer); } @@ -529,7 +529,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), currentState.metadata(), aliasValidator, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), + xContentRegistry, indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap())), Collections.singletonList(templateName), metadataTransformer); } @@ -580,7 +580,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterSt currentState.metadata(), aliasValidator, xContentRegistry, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), + indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap())), org.elasticsearch.common.collect.List.of(), metadataTransformer); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java index 40ea503c7f246..18c997b3cb3e2 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java @@ -149,8 +149,8 @@ public ClusterState applyAliasActions(ClusterState currentState, Iterable System.currentTimeMillis(), null, emptyMap()), xContentRegistry); + aliasValidator.validateAliasFilter(alias, filter, indexService.newQueryShardContext(0, 0, + null, () -> System.currentTimeMillis(), null, emptyMap()), xContentRegistry); } }; if (action.apply(newAliasValidator, metadata, index)) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index e762a0decfb7a..3a91fa5447378 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1093,7 +1093,7 @@ private static void validateCompositeTemplate(final ClusterState state, new AliasValidator(), // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())); + xContentRegistry, tempIndexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap())); // triggers inclusion of _timestamp field and its validation: String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index 6ffbd8aa7ee4f..2dd224b3e8ac8 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -93,6 +93,7 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.search.sort.ShardDocSortField; import java.io.IOException; import java.math.BigInteger; @@ -574,29 +575,35 @@ public static void writeSortType(StreamOutput out, SortField.Type sortType) thro out.writeVInt(sortType.ordinal()); } - public static void writeSortField(StreamOutput out, SortField sortField) throws IOException { + /** + * Returns the generic version of the provided {@link SortField} that + * can be used to merge documents coming from different shards. + */ + private static SortField rewriteMergeSortField(SortField sortField) { if (sortField.getClass() == GEO_DISTANCE_SORT_TYPE_CLASS) { - // for geo sorting, we replace the SortField with a SortField that assumes a double field. - // this works since the SortField is only used for merging top docs SortField newSortField = new SortField(sortField.getField(), SortField.Type.DOUBLE); newSortField.setMissingValue(sortField.getMissingValue()); - sortField = newSortField; + return newSortField; } else if (sortField.getClass() == SortedSetSortField.class) { - // for multi-valued sort field, we replace the SortedSetSortField with a simple SortField. - // It works because the sort field is only used to merge results from different shards. SortField newSortField = new SortField(sortField.getField(), SortField.Type.STRING, sortField.getReverse()); newSortField.setMissingValue(sortField.getMissingValue()); - sortField = newSortField; + return newSortField; } else if (sortField.getClass() == SortedNumericSortField.class) { - // for multi-valued sort field, we replace the SortedSetSortField with a simple SortField. - // It works because the sort field is only used to merge results from different shards. SortField newSortField = new SortField(sortField.getField(), ((SortedNumericSortField) sortField).getNumericType(), sortField.getReverse()); newSortField.setMissingValue(sortField.getMissingValue()); - sortField = newSortField; + return newSortField; + } else if (sortField.getClass() == ShardDocSortField.class) { + SortField newSortField = new SortField(sortField.getField(), SortField.Type.LONG, sortField.getReverse()); + return newSortField; + } else { + return sortField; } + } + public static void writeSortField(StreamOutput out, SortField sortField) throws IOException { + sortField = rewriteMergeSortField(sortField); if (sortField.getClass() != SortField.class) { throw new IllegalArgumentException("Cannot serialize SortField impl [" + sortField + "]"); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index d554395a1d80e..4ac8f6d01d0f6 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -198,7 +198,7 @@ public IndexService( assert indexAnalyzers != null; this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, // we parse all percolator queries as they would be parsed on shard 0 - () -> newQueryShardContext(0, null, System::currentTimeMillis, null, emptyMap()), idFieldDataEnabled, scriptService); + () -> newQueryShardContext(0, 0, null, System::currentTimeMillis, null, emptyMap()), idFieldDataEnabled, scriptService); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { // we delay the actual creation of the sort order for this index because the mapping has not been merged yet. @@ -598,6 +598,7 @@ public IndexSettings getIndexSettings() { */ public QueryShardContext newQueryShardContext( int shardId, + int shardRequestIndex, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias, @@ -606,9 +607,26 @@ public QueryShardContext newQueryShardContext( final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(index().getName(), clusterAlias, clusterService, expressionResolver); return new QueryShardContext( - shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(), - similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, - indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, runtimeMappings); + shardId, + shardRequestIndex, + indexSettings, + bigArrays, + indexCache.bitsetFilterCache(), + indexFieldData::getForField, + mapperService(), + similarityService(), + scriptService, + xContentRegistry, + namedWriteableRegistry, + client, + searcher, + nowInMillis, + clusterAlias, + indexNameMatcher, + allowExpensiveQueries, + valuesSourceRegistry, + runtimeMappings + ); } /** diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index 96323d619a82a..6143c4923b924 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -103,6 +103,7 @@ public class QueryShardContext extends QueryRewriteContext { private final BitsetFilterCache bitsetFilterCache; private final TriFunction, IndexFieldData> indexFieldDataService; private final int shardId; + private final int shardRequestIndex; private final IndexSearcher searcher; private String[] types = Strings.EMPTY_ARRAY; private boolean cacheable = true; @@ -132,6 +133,7 @@ public String[] getTypes() { */ public QueryShardContext( int shardId, + int shardRequestIndex, IndexSettings indexSettings, BigArrays bigArrays, BitsetFilterCache bitsetFilterCache, @@ -152,6 +154,7 @@ public QueryShardContext( ) { this( shardId, + shardRequestIndex, indexSettings, bigArrays, bitsetFilterCache, @@ -176,13 +179,30 @@ public QueryShardContext( } public QueryShardContext(QueryShardContext source) { - this(source.shardId, source.indexSettings, source.bigArrays, source.bitsetFilterCache, source.indexFieldDataService, - source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(), - source.getWriteableRegistry(), source.client, source.searcher, source.nowInMillis, source.indexNameMatcher, - source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry, source.runtimeMappings); + this( + source.shardId, + source.shardRequestIndex, + source.indexSettings, + source.bigArrays, + source.bitsetFilterCache, + source.indexFieldDataService, + source.mapperService, + source.similarityService, + source.scriptService, + source.getXContentRegistry(), + source.getWriteableRegistry(), + source.client, source.searcher, + source.nowInMillis, + source.indexNameMatcher, + source.fullyQualifiedIndex, + source.allowExpensiveQueries, + source.valuesSourceRegistry, + source.runtimeMappings + ); } private QueryShardContext(int shardId, + int shardRequestIndex, IndexSettings indexSettings, BigArrays bigArrays, BitsetFilterCache bitsetFilterCache, @@ -202,6 +222,7 @@ private QueryShardContext(int shardId, Map runtimeMappings) { super(xContentRegistry, namedWriteableRegistry, client, nowInMillis); this.shardId = shardId; + this.shardRequestIndex = shardRequestIndex; this.similarityService = similarityService; this.mapperService = mapperService; this.bigArrays = bigArrays; @@ -562,6 +583,14 @@ public int getShardId() { return shardId; } + /** + * Returns the shard request ordinal that is used by the main search request + * to reference this shard. + */ + public int getShardRequestIndex() { + return shardRequestIndex; + } + @Override public final long nowInMillis() { failIfFrozen(); diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index c1170cde613ff..813a66134d298 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -172,7 +172,8 @@ final class DefaultSearchContext extends SearchContext { this.minNodeVersion = minNodeVersion; queryShardContext = indexService.newQueryShardContext( request.shardId().id(), - this.searcher, + request.shardRequestIndex(), + searcher, request::nowInMillis, shardTarget.getClusterAlias(), request.getRuntimeMappings() diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 179446a4a70b4..a151091447f49 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1186,8 +1186,8 @@ private CanMatchResponse canMatch(ShardSearchRequest request, boolean checkRefre } try (Releasable ignored2 = canMatchSearcher) { - QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), canMatchSearcher, - request::nowInMillis, request.getClusterAlias(), request.getRuntimeMappings()); + QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), 0, + canMatchSearcher, request::nowInMillis, request.getClusterAlias(), request.getRuntimeMappings()); final boolean canMatch = queryStillMatchesAfterRewrite(request, context); final MinAndMax minMax; if (canMatch || hasRefreshPending) { diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index b86f4f7ce3f02..d55bda988e22b 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -71,7 +71,7 @@ public class ShardSearchRequest extends TransportRequest implements IndicesRequest { private final String clusterAlias; private final ShardId shardId; - private final int shardIndex; + private final int shardRequestIndex; private final int numberOfShards; private final SearchType searchType; private final Scroll scroll; @@ -94,20 +94,20 @@ public class ShardSearchRequest extends TransportRequest implements IndicesReque public ShardSearchRequest(OriginalIndices originalIndices, SearchRequest searchRequest, ShardId shardId, - int shardIndex, + int shardRequestIndex, int numberOfShards, AliasFilter aliasFilter, float indexBoost, long nowInMillis, @Nullable String clusterAlias) { - this(originalIndices, searchRequest, shardId, shardIndex, numberOfShards, aliasFilter, + this(originalIndices, searchRequest, shardId, shardRequestIndex, numberOfShards, aliasFilter, indexBoost, nowInMillis, clusterAlias, null, null); } public ShardSearchRequest(OriginalIndices originalIndices, SearchRequest searchRequest, ShardId shardId, - int shardIndex, + int shardRequestIndex, int numberOfShards, AliasFilter aliasFilter, float indexBoost, @@ -117,7 +117,7 @@ public ShardSearchRequest(OriginalIndices originalIndices, TimeValue keepAlive) { this(originalIndices, shardId, - shardIndex, + shardRequestIndex, numberOfShards, searchRequest.searchType(), searchRequest.source(), @@ -146,7 +146,7 @@ public ShardSearchRequest(ShardId shardId, private ShardSearchRequest(OriginalIndices originalIndices, ShardId shardId, - int shardIndex, + int shardRequestIndex, int numberOfShards, SearchType searchType, SearchSourceBuilder source, @@ -161,7 +161,7 @@ private ShardSearchRequest(OriginalIndices originalIndices, ShardSearchContextId readerId, TimeValue keepAlive) { this.shardId = shardId; - this.shardIndex = shardIndex; + this.shardRequestIndex = shardRequestIndex; this.numberOfShards = numberOfShards; this.searchType = searchType; this.source = source; @@ -183,7 +183,7 @@ public ShardSearchRequest(StreamInput in) throws IOException { super(in); shardId = new ShardId(in); searchType = SearchType.fromId(in.readByte()); - shardIndex = in.getVersion().onOrAfter(Version.V_7_11_0) ? in.readVInt() : -1; + shardRequestIndex = in.getVersion().onOrAfter(Version.V_7_11_0) ? in.readVInt() : -1; numberOfShards = in.readVInt(); scroll = in.readOptionalWriteable(Scroll::new); source = in.readOptionalWriteable(SearchSourceBuilder::new); @@ -224,7 +224,7 @@ public ShardSearchRequest(StreamInput in) throws IOException { public ShardSearchRequest(ShardSearchRequest clone) { this.shardId = clone.shardId; - this.shardIndex = clone.shardIndex; + this.shardRequestIndex = clone.shardRequestIndex; this.searchType = clone.searchType; this.numberOfShards = clone.numberOfShards; this.scroll = clone.scroll; @@ -255,7 +255,7 @@ protected final void innerWriteTo(StreamOutput out, boolean asKey) throws IOExce out.writeByte(searchType.id()); if (asKey == false) { if (out.getVersion().onOrAfter(Version.V_7_11_0)) { - out.writeVInt(shardIndex); + out.writeVInt(shardRequestIndex); } out.writeVInt(numberOfShards); } @@ -329,10 +329,11 @@ public void source(SearchSourceBuilder source) { } /** - * Returns the index of the shard that is used to tiebreak documents with identical sort values. + * Returns the shard request ordinal that is used by the main search request + * to reference this shard. */ - public int shardIndex() { - return shardIndex; + public int shardRequestIndex() { + return shardRequestIndex; } public int numberOfShards() { diff --git a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java index 51d3e9085ef44..8b51f03edfc97 100644 --- a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java @@ -215,8 +215,8 @@ public Query toFilter(ShardSearchRequest request, QueryShardContext context) { throw new IllegalArgumentException("field " + field + " not found"); } - int shardIndex = request.shardIndex() != -1 ? request.shardIndex() : request.shardId().id(); - int numShards = request.shardIndex() != -1 ? request.numberOfShards() : context.getIndexSettings().getNumberOfShards(); + int shardIndex = request.shardRequestIndex() != -1 ? request.shardRequestIndex() : request.shardId().id(); + int numShards = request.shardRequestIndex() != -1 ? request.numberOfShards() : context.getIndexSettings().getNumberOfShards(); String field = this.field; boolean useTermQuery = false; if ("_uid".equals(field)) { diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index c2e8f5425d013..1a4719c3ffcd6 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -81,6 +81,12 @@ public class FieldSortBuilder extends SortBuilder { * special field name to sort by index order */ public static final String DOC_FIELD_NAME = "_doc"; + + /** + * special field name to sort by index order + */ + public static final String SHARD_DOC_FIELD_NAME = ShardDocSortField.NAME; + private static final SortFieldAndFormat SORT_DOC = new SortFieldAndFormat( new SortField(null, SortField.Type.DOC), DocValueFormat.RAW); private static final SortFieldAndFormat SORT_DOC_REVERSE = new SortFieldAndFormat( @@ -386,8 +392,12 @@ private static NumericType resolveNumericType(String value) { @Override public SortFieldAndFormat build(QueryShardContext context) throws IOException { + final boolean reverse = order == SortOrder.DESC; + if (DOC_FIELD_NAME.equals(fieldName)) { - return order == SortOrder.DESC ? SORT_DOC_REVERSE : SORT_DOC; + return reverse ? SORT_DOC_REVERSE : SORT_DOC; + } else if (SHARD_DOC_FIELD_NAME.equals(fieldName)) { + return new SortFieldAndFormat(new ShardDocSortField(context.getShardRequestIndex(), reverse), DocValueFormat.RAW); } MappedFieldType fieldType = context.getFieldType(fieldName); @@ -396,7 +406,6 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException { fieldType = resolveUnmappedType(context); } - boolean reverse = order == SortOrder.DESC; IndexFieldData fieldData = context.getForField(fieldType); if (fieldData instanceof IndexNumericFieldData == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) { diff --git a/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java b/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java new file mode 100644 index 0000000000000..47907875d8a8d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.sort; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.comparators.DocComparator; + + /** + * A {@link SortField} that first compares the shard index and then uses the document number (_doc) + * to tiebreak if the value is the same. + **/ +public class ShardDocSortField extends SortField { + public static final String NAME = "_shard_doc"; + + private final int shardRequestIndex; + + public ShardDocSortField(int shardRequestIndex, boolean reverse) { + super(NAME, Type.LONG, reverse); + assert shardRequestIndex >= 0; + this.shardRequestIndex = shardRequestIndex; + } + + int getShardRequestIndex() { + return shardRequestIndex; + } + + @Override + public FieldComparator getComparator(int numHits, int sortPos) { + final DocComparator delegate = new DocComparator(numHits, false, sortPos); + + return new FieldComparator() { + @Override + public int compare(int slot1, int slot2) { + return delegate.compare(slot1, slot2); + } + + @Override + public int compareValues(Long first, Long second) { + return Long.compare(first, second); + } + + @Override + public void setTopValue(Long value) { + int topShardIndex = (int) (value >> 32); + if (shardRequestIndex == topShardIndex) { + delegate.setTopValue(value.intValue()); + } else if (shardRequestIndex < topShardIndex) { + delegate.setTopValue(Integer.MAX_VALUE); + } else { + delegate.setTopValue(-1); + } + } + + @Override + public Long value(int slot) { + return (((long) shardRequestIndex) << 32) | (delegate.value(slot) & 0xFFFFFFFFL); + } + + @Override + public LeafFieldComparator getLeafComparator(LeafReaderContext context) { + return delegate.getLeafComparator(context); + } + }; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/sort/SortBuilders.java b/server/src/main/java/org/elasticsearch/search/sort/SortBuilders.java index 3eae9b8d01960..7f5d98dee82ec 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/SortBuilders.java +++ b/server/src/main/java/org/elasticsearch/search/sort/SortBuilders.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.script.Script; +import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType; /** @@ -46,6 +47,13 @@ public static FieldSortBuilder fieldSort(String field) { return new FieldSortBuilder(field); } + /** + * Constructs a sort tiebreaker that can be used within a point in time reader {@link PointInTimeBuilder}. + */ + public static FieldSortBuilder pitTiebreaker() { + return new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME); + } + /** * Constructs a new script based sort. * diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 8d2b7df7fffdb..02eb3ff2cbcf0 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -130,7 +130,7 @@ public void setupCreateIndexRequestAndAliasValidator() { request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); Settings indexSettings = Settings.builder().put(SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); - queryShardContext = new QueryShardContext(0, + queryShardContext = new QueryShardContext(0, 0, new IndexSettings(IndexMetadata.builder("test").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> randomNonNegativeLong(), null, null, () -> true, null, emptyMap()); diff --git a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java index a716350638d9d..2da356426e43e 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java @@ -68,6 +68,7 @@ import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource; import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.sort.ShardDocSortField; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -716,7 +717,7 @@ private static Tuple randomSortFieldCustomComparatorSource private static Tuple randomCustomSortField() { String field = randomAlphaOfLengthBetween(3, 10); - switch(randomIntBetween(0, 2)) { + switch(randomIntBetween(0, 3)) { case 0: { SortField sortField = LatLonDocValuesField.newDistanceSort(field, 0, 0); SortField expected = new SortField(field, SortField.Type.DOUBLE); @@ -742,6 +743,11 @@ private static Tuple randomCustomSortField() { } return Tuple.tuple(sortField, expected); } + case 3: { + ShardDocSortField sortField = new ShardDocSortField(randomIntBetween(0, 100), randomBoolean()); + SortField expected = new SortField(ShardDocSortField.NAME, SortField.Type.LONG, sortField.getReverse()); + return Tuple.tuple(sortField, expected); + } default: throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 0438577b219c6..292d7fb034917 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -139,7 +139,7 @@ public void setup() throws Exception { writer = new IndexWriter( new ByteBuffersDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()) ); - shardContext = indexService.newQueryShardContext(0, null, () -> 0, null, emptyMap()); + shardContext = indexService.newQueryShardContext(0, 0, null, () -> 0, null, emptyMap()); } protected final List refreshReader() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index e860de2a7da89..8ec0762209b40 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -164,7 +164,7 @@ public void testValueForSearch() { public void testTermQuery() { Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); - QueryShardContext context = new QueryShardContext(0, + QueryShardContext context = new QueryShardContext(0, 0, new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); @@ -186,7 +186,7 @@ public void testTermQuery() { public void testRangeQuery() throws IOException { Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); - QueryShardContext context = new QueryShardContext(0, + QueryShardContext context = new QueryShardContext(0, 0, new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); @@ -231,7 +231,7 @@ public void testRangeQueryWithIndexSort() { .build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); - QueryShardContext context = new QueryShardContext(0, indexSettings, + QueryShardContext context = new QueryShardContext(0, 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> 0L, null, null, () -> true, null, emptyMap()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java index 7e93b0e79c63c..c78346f469439 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java @@ -50,7 +50,7 @@ public void testTermQuery() { when(mapperService.fieldType("field_name")).thenReturn(fieldType); when(mapperService.simpleMatchToFullName("field_name")).thenReturn(Collections.singleton("field_name")); - QueryShardContext queryShardContext = new QueryShardContext(0, + QueryShardContext queryShardContext = new QueryShardContext(0, 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, null, null, null, null, () -> 0L, null, null, () -> true, null, emptyMap()); Query termQuery = fieldNamesFieldType.termQuery("field_name", queryShardContext); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java index b858929e3abc4..cd24c8bb48d11 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java @@ -69,7 +69,7 @@ private QueryShardContext createContext() { IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); Predicate indexNameMatcher = pattern -> Regex.simpleMatch(pattern, "index"); - return new QueryShardContext(0, indexSettings, null, null, null, null, null, null, xContentRegistry(), writableRegistry(), + return new QueryShardContext(0, 0, indexSettings, null, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, System::currentTimeMillis, null, indexNameMatcher, () -> true, null, emptyMap()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index 6d4ab1321476b..d7ff6bb537299 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -473,7 +473,7 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS DirectoryReader reader = DirectoryReader.open(w); IndexSearcher searcher = newSearcher(reader); - QueryShardContext context = new QueryShardContext(0, indexSettings, + QueryShardContext context = new QueryShardContext(0, 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> 0L, null, null, () -> true, null, emptyMap()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index b183efe2f9a73..fb8733953faf8 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -213,7 +213,7 @@ private QueryShardContext createContext() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); - return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, + return new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); } diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index dd41933e32388..04f6313a371b9 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -196,6 +196,7 @@ public void testIndexSortedOnField() { IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); QueryShardContext context = new QueryShardContext( + 0, 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, @@ -377,7 +378,7 @@ private static QueryShardContext createQueryShardContext( MapperService mapperService = createMapperService(indexUuid, fieldTypeLookup, mapperPlugins); final long nowInMillis = randomNonNegativeLong(); return new QueryShardContext( - 0, mapperService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, + 0, 0, mapperService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), null, null, () -> nowInMillis, clusterAlias, null, () -> true, null, runtimeMappings); diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java index 9ab1e80f09bdc..d7f6cb3a6c61d 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java @@ -41,7 +41,7 @@ public class RangeQueryRewriteTests extends ESSingleNodeTestCase { public void testRewriteMissingField() throws Exception { IndexService indexService = createIndex("test"); IndexReader reader = new MultiReader(); - QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, + QueryRewriteContext context = new QueryShardContext(0, 0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), null, new IndexSearcher(reader), null, null, null, () -> true, null, emptyMap()); RangeQueryBuilder range = new RangeQueryBuilder("foo"); @@ -59,7 +59,7 @@ public void testRewriteMissingReader() throws Exception { .endObject().endObject()); indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); - QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, null, + QueryRewriteContext context = new QueryShardContext(0, 0, indexService.getIndexSettings(), null, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), null, null, null, null, null, () -> true, null, emptyMap()); RangeQueryBuilder range = new RangeQueryBuilder("foo"); @@ -79,7 +79,7 @@ public void testRewriteEmptyReader() throws Exception { indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); IndexReader reader = new MultiReader(); - QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, + QueryRewriteContext context = new QueryShardContext(0, 0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), null, new IndexSearcher(reader), null, null, null, () -> true, null, emptyMap()); RangeQueryBuilder range = new RangeQueryBuilder("foo"); diff --git a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java index 8410636583024..737ae519dd5b0 100644 --- a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java @@ -100,7 +100,13 @@ public void setup() throws IOException { public void testCrossFieldMultiMatchQuery() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ); queryShardContext.setAllowUnmappedFields(true); for (float tieBreaker : new float[] {0.0f, 0.5f}) { Query parsedQuery = multiMatchQuery("banon") @@ -127,7 +133,14 @@ public void testBlendTerms() { float[] boosts = new float[] {2, 3}; Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + indexService.newQueryShardContext( + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ), new BytesRef("baz"), null, 1f, @@ -146,7 +159,14 @@ public void testBlendTermsWithFieldBoosts() { float[] boosts = new float[] {200, 30}; Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + indexService.newQueryShardContext( + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ), new BytesRef("baz"), null, 1f, @@ -171,7 +191,14 @@ public Query termQuery(Object value, QueryShardContext context) { BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f) ), 1f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + indexService.newQueryShardContext( + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ), new BytesRef("baz"), null, 1f, @@ -189,7 +216,13 @@ public Query termQuery(Object value, QueryShardContext context) { } }; expectThrows(IllegalArgumentException.class, () -> MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + indexService.newQueryShardContext( + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap()), new BytesRef("baz"), null, 1f, false, Arrays.asList(new FieldAndBoost(ft, 1)))); } @@ -211,7 +244,14 @@ public Query termQuery(Object value, QueryShardContext context) { expectedDisjunct1 ), 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + indexService.newQueryShardContext( + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ), new BytesRef("baz"), null, 1f, @@ -223,7 +263,13 @@ public Query termQuery(Object value, QueryShardContext context) { public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); parser.setAnalyzer(new MockSynonymAnalyzer()); @@ -255,7 +301,13 @@ public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { public void testMultiMatchCrossFieldsWithSynonymsPhrase() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); parser.setAnalyzer(new MockSynonymAnalyzer()); Map fieldNames = new HashMap<>(); @@ -322,9 +374,12 @@ public void testKeywordSplitQueriesOnWhitespace() throws IOException { .endObject().endObject()); mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { - throw new UnsupportedOperationException(); - }, null, emptyMap()); + randomInt(20), + 0, + null, () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); Map fieldNames = new HashMap<>(); fieldNames.put("field", 1.0f); diff --git a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java index 4cd75d8b04bea..93609360b8b97 100644 --- a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java @@ -335,7 +335,14 @@ public void testConjunction() { } public void testNested() throws IOException { - QueryShardContext context = indexService.newQueryShardContext(0, new IndexSearcher(new MultiReader()), () -> 0, null, emptyMap()); + QueryShardContext context = indexService.newQueryShardContext( + 0, + 0, + new IndexSearcher(new MultiReader()), + () -> 0, + null, + emptyMap() + ); NestedQueryBuilder queryBuilder = new NestedQueryBuilder("nested1", new MatchAllQueryBuilder(), ScoreMode.Avg); ESToParentBlockJoinQuery query = (ESToParentBlockJoinQuery) queryBuilder.toQuery(context); diff --git a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java index 1c354050ab0b6..caa5c307999bc 100644 --- a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java @@ -614,7 +614,7 @@ public void testMultiLevelNestedSorting() throws IOException { DirectoryReader reader = DirectoryReader.open(writer); reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId(indexService.index(), 0)); IndexSearcher searcher = new IndexSearcher(reader); - QueryShardContext queryShardContext = indexService.newQueryShardContext(0, searcher, () -> 0L, null, emptyMap()); + QueryShardContext queryShardContext = indexService.newQueryShardContext(0, 0, searcher, () -> 0L, null, emptyMap()); FieldSortBuilder sortBuilder = new FieldSortBuilder("chapters.paragraphs.word_count"); sortBuilder.setNestedSort(new NestedSortBuilder("chapters").setNestedSort(new NestedSortBuilder("chapters.paragraphs"))); diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index e2f570fb89345..659e62697c26e 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -80,7 +80,7 @@ public void testPreProcess() throws Exception { ShardId shardId = new ShardId("index", UUID.randomUUID().toString(), 1); when(shardSearchRequest.shardId()).thenReturn(shardId); when(shardSearchRequest.types()).thenReturn(new String[]{}); - when(shardSearchRequest.shardIndex()).thenReturn(shardId.id()); + when(shardSearchRequest.shardRequestIndex()).thenReturn(shardId.id()); when(shardSearchRequest.numberOfShards()).thenReturn(2); ThreadPool threadPool = new TestThreadPool(this.getClass().getName()); @@ -107,9 +107,8 @@ public void testPreProcess() throws Exception { when(indexCache.query()).thenReturn(queryCache); when(indexService.cache()).thenReturn(indexCache); QueryShardContext queryShardContext = mock(QueryShardContext.class); - when(indexService.newQueryShardContext(eq(shardId.id()), anyObject(), anyObject(), anyString(), anyObject())).thenReturn( - queryShardContext - ); + when(indexService.newQueryShardContext(eq(shardId.id()), eq(shardId.id()), anyObject(), anyObject(), anyString(), anyObject())) + .thenReturn(queryShardContext); MapperService mapperService = mock(MapperService.class); when(mapperService.hasNested()).thenReturn(randomBoolean()); when(indexService.mapperService()).thenReturn(mapperService); @@ -240,8 +239,11 @@ public void testClearQueryCancellationsOnClose() throws IOException { TimeValue timeout = new TimeValue(randomIntBetween(1, 100)); ShardSearchRequest shardSearchRequest = mock(ShardSearchRequest.class); when(shardSearchRequest.searchType()).thenReturn(SearchType.DEFAULT); + when(shardSearchRequest.types()).thenReturn(new String[]{}); ShardId shardId = new ShardId("index", UUID.randomUUID().toString(), 1); when(shardSearchRequest.shardId()).thenReturn(shardId); + when(shardSearchRequest.shardRequestIndex()).thenReturn(shardId.id()); + when(shardSearchRequest.numberOfShards()).thenReturn(2); ThreadPool threadPool = new TestThreadPool(this.getClass().getName()); IndexShard indexShard = mock(IndexShard.class); @@ -251,9 +253,14 @@ public void testClearQueryCancellationsOnClose() throws IOException { IndexService indexService = mock(IndexService.class); QueryShardContext queryShardContext = mock(QueryShardContext.class); - when(indexService.newQueryShardContext(eq(shardId.id()), anyObject(), anyObject(), anyString(), anyObject())).thenReturn( - queryShardContext - ); + when(indexService.newQueryShardContext( + eq(shardId.id()), + eq(shardId.id()), + anyObject(), + anyObject(), + anyString(), + anyObject()) + ).thenReturn(queryShardContext); try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 1f92c773021f6..20063b6be772e 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -738,7 +738,7 @@ private static QueryShardContext newQueryShardContext(MapperService mapperServic .put(IndexMetadata.SETTING_INDEX_UUID, "uuid").build(); IndexMetadata indexMetadata = new IndexMetadata.Builder("index").settings(settings).build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); - return new QueryShardContext(0, indexSettings, null, null, null, mapperService, null, null, null, null, null, null, null, null, + return new QueryShardContext(0, 0, indexSettings, null, null, null, mapperService, null, null, null, null, null, null, null, null, null, null, null, emptyMap()); } } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index 39043ae669501..159abfd374622 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -278,7 +278,7 @@ public void testBuildSearchContextHighlight() throws IOException { Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_"); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, null, null, System::currentTimeMillis, null, null, () -> true, null, emptyMap()) { @Override diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java index 87f35b5375ab2..05040eeb6741d 100644 --- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java @@ -142,7 +142,7 @@ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()) { @Override @@ -186,7 +186,7 @@ public void testRewritingKeepsSettings() throws IOException { .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()) { @Override diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 2afbb53a5b651..3865affb3bb33 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -201,7 +201,7 @@ protected final QueryShardContext createMockShardContext(IndexSearcher searcher) IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null); }; - return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, + return new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, null, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, searcher, () -> randomNonNegativeLong(), null, null, () -> true, null, emptyMap()) { diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index 4c46094be287c..06f989c24bd70 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -73,6 +73,7 @@ import static org.elasticsearch.search.sort.FieldSortBuilder.getMinMaxOrNull; import static org.elasticsearch.search.sort.FieldSortBuilder.getPrimaryFieldSortOrNull; import static org.elasticsearch.search.sort.NestedSortBuilderTests.createRandomNestedSort; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; public class FieldSortBuilderTests extends AbstractSortTestCase { @@ -179,6 +180,8 @@ protected void sortFieldAssertions(FieldSortBuilder builder, SortField sortField SortField.Type expectedType; if (builder.getFieldName().equals(FieldSortBuilder.DOC_FIELD_NAME)) { expectedType = SortField.Type.DOC; + } else if (builder.getFieldName().equals(FieldSortBuilder.SHARD_DOC_FIELD_NAME)) { + expectedType = SortField.Type.LONG; } else { expectedType = SortField.Type.CUSTOM; } @@ -347,6 +350,20 @@ public void testUnknownOptionFails() throws IOException { } } + public void testShardDocSort() throws IOException { + QueryShardContext shardContextMock = createMockShardContext(); + + boolean reverse = randomBoolean(); + FieldSortBuilder sortBuilder = new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME) + .order(reverse ? SortOrder.DESC : SortOrder.ASC); + SortFieldAndFormat sortAndFormat = sortBuilder.build(shardContextMock); + assertThat(sortAndFormat.field.getClass(), equalTo(ShardDocSortField.class)); + ShardDocSortField sortField = (ShardDocSortField) sortAndFormat.field; + assertThat(sortField.getShardRequestIndex(), equalTo(shardContextMock.getShardRequestIndex())); + assertThat(sortField.getReverse(), equalTo(reverse)); + assertThat(sortAndFormat.format, equalTo(DocValueFormat.RAW)); + } + @Override protected MappedFieldType provideMappedFieldType(String name) { if (name.equals(MAPPED_STRING_FIELDNAME)) { diff --git a/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java index 5f5ea5e869450..8e79e1c15f03d 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java @@ -200,20 +200,25 @@ public static List> randomSortBuilderList() { int size = randomIntBetween(1, 5); List> list = new ArrayList<>(size); for (int i = 0; i < size; i++) { - switch (randomIntBetween(0, 3)) { + switch (randomIntBetween(0, 5)) { case 0: list.add(new ScoreSortBuilder()); break; case 1: - String fieldName = rarely() ? FieldSortBuilder.DOC_FIELD_NAME : randomAlphaOfLengthBetween(1, 10); - list.add(new FieldSortBuilder(fieldName)); + list.add(new FieldSortBuilder( randomAlphaOfLengthBetween(1, 10))); break; case 2: - list.add(GeoDistanceSortBuilderTests.randomGeoDistanceSortBuilder()); + list.add(SortBuilders.fieldSort(FieldSortBuilder.DOC_FIELD_NAME)); break; case 3: + list.add(GeoDistanceSortBuilderTests.randomGeoDistanceSortBuilder()); + break; + case 4: list.add(ScriptSortBuilderTests.randomScriptSortBuilder()); break; + case 5: + list.add(SortBuilders.pitTiebreaker()); + break; default: throw new IllegalStateException("unexpected randomization in tests"); } diff --git a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java index ca3cf6e8ae030..2ad3719cafa27 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -179,7 +179,7 @@ public NamedAnalyzer get(Object key) { when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); when(scriptService.compile(any(Script.class), any())).then(invocation -> new TestTemplateService.MockTemplateScript.Factory( ((Script) invocation.getArguments()[0]).getIdOrCode())); - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, null, System::currentTimeMillis, null, null, () -> true, null, emptyMap()); @@ -219,7 +219,7 @@ public void testBuildWithUnmappedField() { when(mapperService.getNamedAnalyzer(any(String.class))).then( invocation -> new NamedAnalyzer((String) invocation.getArguments()[0], AnalyzerScope.INDEX, new SimpleAnalyzer())); - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, null, System::currentTimeMillis, null, null, () -> true, null, emptyMap()); if (randomBoolean()) { diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 1b78fbdef7b61..165e1681806ad 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -264,6 +264,7 @@ protected AggregationContext createAggregationContext(IndexSearcher indexSearche .build(new IndexFieldDataCache.None(), breakerService); QueryShardContext queryShardContext = new QueryShardContext( 0, + -1, indexSettings, bigArrays, null, diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index d8ac25e40aecb..0c5e4631d6d98 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -416,7 +416,7 @@ public void close() throws IOException { } QueryShardContext createShardContext(IndexSearcher searcher) { - return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, + return new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataService::getForField, mapperService, similarityService, scriptService, xContentRegistry, namedWriteableRegistry, this.client, searcher, () -> nowInMillis, null, indexNameMatcher(), () -> true, null, emptyMap()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index a410df8a2aa1e..29b64da0122cf 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -96,7 +96,7 @@ public TestSearchContext(IndexService indexService) { this.indexService = indexService; this.fixedBitSetFilterCache = indexService.cache().bitsetFilterCache(); this.indexShard = indexService.getShardOrNull(0); - queryShardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()); + queryShardContext = indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap()); } public TestSearchContext(QueryShardContext queryShardContext) { diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/search/PointInTimeIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/search/PointInTimeIT.java index 9bd317f7b4a80..f2a997de88288 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/search/PointInTimeIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/search/PointInTimeIT.java @@ -16,6 +16,8 @@ import org.elasticsearch.xpack.core.search.action.ClosePointInTimeAction; import org.elasticsearch.xpack.core.search.action.ClosePointInTimeRequest; import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.search.ShardSearchFailure; @@ -31,8 +33,12 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.search.SearchContextMissingException; +import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.builder.PointInTimeBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xpack.core.search.action.OpenPointInTimeAction; import org.elasticsearch.xpack.core.search.action.OpenPointInTimeRequest; @@ -40,6 +46,7 @@ import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -382,6 +389,85 @@ public void testPartialResults() throws Exception { } } + public void testPITTiebreak() throws Exception { + assertAcked(client().admin().indices().prepareDelete("index-*").get()); + int numIndex = randomIntBetween(2, 10); + int expectedNumDocs = 0; + for (int i = 0; i < numIndex; i++) { + String index = "index-" + i; + createIndex(index, Settings.builder().put("index.number_of_shards", 1).build()); + int numDocs = randomIntBetween(3, 20); + for (int j = 0; j < numDocs; j++) { + client().prepareIndex(index, "_doc").setSource("value", randomIntBetween(0, 2)).get(); + expectedNumDocs ++; + } + } + refresh("index-*"); + String pit = openPointInTime(new String[] { "index-*" }, TimeValue.timeValueHours(1)); + try { + for (int size = 1; size <= numIndex; size++) { + SortOrder order = randomBoolean() ? SortOrder.ASC : SortOrder.DESC; + assertPagination(new PointInTimeBuilder(pit), expectedNumDocs, size, + SortBuilders.pitTiebreaker().order(order)); + assertPagination(new PointInTimeBuilder(pit), expectedNumDocs, size, + SortBuilders.fieldSort("value"), SortBuilders.pitTiebreaker().order(order)); + } + } finally { + closePointInTime(pit); + } + } + + private void assertPagination(PointInTimeBuilder pit, int expectedNumDocs, int size, SortBuilder... sort) throws Exception { + Set seen = new HashSet<>(); + SearchRequestBuilder builder = client().prepareSearch() + .setSize(size) + .setPointInTime(pit); + + final int[] reverseMuls = new int[sort.length]; + for (int i = 0; i < sort.length; i++) { + builder.addSort(sort[i]); + reverseMuls[i] = sort[i].order() == SortOrder.ASC ? 1 : -1; + } + final SearchRequest searchRequest = builder.request(); + SearchResponse response = client().search(searchRequest).get(); + Object[] lastSortValues = null; + while (response.getHits().getHits().length > 0) { + Object[] lastHitSortValues = null; + for (SearchHit hit : response.getHits().getHits()) { + assertTrue(seen.add(hit.getIndex() + hit.getId())); + + if (lastHitSortValues != null) { + for (int i = 0; i < sort.length; i++) { + Comparable value = (Comparable) hit.getRawSortValues()[i]; + int cmp = value.compareTo(lastHitSortValues[i]) * reverseMuls[i]; + if (cmp != 0) { + assertThat(cmp, equalTo(1)); + break; + } + } + } + lastHitSortValues = hit.getRawSortValues(); + } + int len = response.getHits().getHits().length; + SearchHit last = response.getHits().getHits()[len - 1]; + if (lastSortValues != null) { + for (int i = 0; i < sort.length; i++) { + Comparable value = (Comparable) last.getSortValues()[i]; + int cmp = value.compareTo(lastSortValues[i]) * reverseMuls[i]; + if (cmp != 0) { + assertThat(cmp, equalTo(1)); + break; + } + } + } + assertThat(last.getSortValues().length, equalTo(sort.length)); + lastSortValues = last.getSortValues(); + searchRequest.source().searchAfter(last.getSortValues()); + response = client().search(searchRequest).get(); + } + assertThat(seen.size(), equalTo(expectedNumDocs)); + } + private String openPointInTime(String[] indices, TimeValue keepAlive) { OpenPointInTimeRequest request = new OpenPointInTimeRequest( indices, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java index 436273e9b2a5d..f777097f23906 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java @@ -564,7 +564,7 @@ private TestIndexContext testIndex(MapperService mapperService, Client client) t directoryReader = DirectoryReader.open(directory); final LeafReaderContext leaf = directoryReader.leaves().get(0); - final QueryShardContext shardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, + final QueryShardContext shardContext = new QueryShardContext(shardId.id(), 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), client, new IndexSearcher(directoryReader), () -> nowInMillis, null, null, () -> true, null, emptyMap()); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java index 74042155ffd93..774c0d9694448 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java @@ -90,7 +90,7 @@ public void testDLS() throws Exception { Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); final long nowInMillis = randomNonNegativeLong(); - QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); QueryShardContext queryShardContext = spy(realQueryShardContext); @@ -222,7 +222,7 @@ public void testDLSWithLimitedPermissions() throws Exception { Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); final long nowInMillis = randomNonNegativeLong(); - QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); QueryShardContext queryShardContext = spy(realQueryShardContext); diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index 9eca0ce4cc24d..c47204a56f283 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -244,6 +244,7 @@ protected MultiSearchResponse shardOperation(Request request, ShardId shardId) t Map runtimeFields = emptyMap(); final QueryShardContext context = indexService.newQueryShardContext( shardId.id(), + 0, searcher, () -> { throw new UnsupportedOperationException(); }, null, diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index 85195ff608599..1868c2288b8a6 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -90,7 +90,7 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase { @Before private void setup() { settings = createIndexSettings(); - queryShardContext = new QueryShardContext(0, settings, + queryShardContext = new QueryShardContext(0, 0, settings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, null, null, null, null, () -> 0L, null, null, () -> true, null, emptyMap()); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index 0d2d76693335e..26a634946dbd5 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -772,6 +772,7 @@ public void onIndexModule(IndexModule module) { module.setReaderWrapper(indexService -> new SecurityIndexReaderWrapper( shardId -> indexService.newQueryShardContext(shardId.id(), + 0, // we pass a null index reader, which is legal and will disable rewrite optimizations // based on index statistics, which is probably safer... null, diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 412df7d51aeb5..2b58dba7524c7 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -897,7 +897,7 @@ protected final QueryShardContext createMockShardContext() { IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null); }; - return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, + return new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, null, null, null, xContentRegistry(), null, null, null, () -> randomNonNegativeLong(), null, null, () -> true, null, emptyMap()) {