diff --git a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java index c7ac35d3febce..b4d5ddc55d0ed 100644 --- a/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java +++ b/modules/analysis-common/src/test/java/org/elasticsearch/analysis/common/DisableGraphQueryTests.java @@ -85,7 +85,7 @@ public void setup() { indexService = createIndex("test", settings, "t", "text_shingle", "type=text,analyzer=text_shingle", "text_shingle_unigram", "type=text,analyzer=text_shingle_unigram"); - shardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()); + shardContext = indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap()); // parsed queries for "text_shingle_unigram:(foo bar baz)" with query parsers // that ignores position length attribute diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java index 1810a1eff384a..4b6109f63df7b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/action/PainlessExecuteAction.java @@ -556,7 +556,7 @@ private static Response prepareRamIndex(Request request, searcher.setQueryCache(null); final long absoluteStartMillis = System.currentTimeMillis(); QueryShardContext context = - indexService.newQueryShardContext(0, searcher, () -> absoluteStartMillis, null, emptyMap()); + indexService.newQueryShardContext(0, 0, searcher, () -> absoluteStartMillis, null, emptyMap()); return handler.apply(context, indexReader.leaves().get(0)); } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java index ca885cdfdff6a..5630973d66448 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/NeedsScoreTests.java @@ -47,7 +47,7 @@ public void testNeedsScores() { contexts.put(NumberSortScript.CONTEXT, Whitelist.BASE_WHITELISTS); PainlessScriptEngine service = new PainlessScriptEngine(Settings.EMPTY, contexts); - QueryShardContext shardContext = index.newQueryShardContext(0, null, () -> 0, null, emptyMap()); + QueryShardContext shardContext = index.newQueryShardContext(0, 0, null, () -> 0, null, emptyMap()); NumberSortScript.Factory factory = service.compile(null, "1.2", NumberSortScript.CONTEXT, Collections.emptyMap()); NumberSortScript.LeafFactory ss = factory.newFactory(Collections.emptyMap(), shardContext.lookup()); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java index 45a836562913a..9da5505623bb1 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java @@ -514,7 +514,7 @@ public void testQueryWithRewrite() throws Exception { XContentType.JSON)); BytesRef qbSource = doc.rootDoc().getFields(fieldType.queryBuilderField.name())[0].binaryValue(); QueryShardContext shardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { + randomInt(20), 0, null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); PlainActionFuture future = new PlainActionFuture<>(); diff --git a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java index ff5d6dc7c4b7b..c9414059bed6e 100644 --- a/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java +++ b/modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorQuerySearchTests.java @@ -259,7 +259,7 @@ public void testRangeQueriesWithNow() throws Exception { try (Engine.Searcher searcher = indexService.getShard(0).acquireSearcher("test")) { long[] currentTime = new long[] {System.currentTimeMillis()}; QueryShardContext queryShardContext = - indexService.newQueryShardContext(0, searcher, () -> currentTime[0], null, emptyMap()); + indexService.newQueryShardContext(0, 0, searcher, () -> currentTime[0], null, emptyMap()); BytesReference source = BytesReference.bytes(jsonBuilder().startObject() .field("field1", "value") diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml index 36ac1f0c7548f..41c8ef3500a85 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml @@ -156,8 +156,8 @@ --- "date_nanos": - skip: - version: " - 7.99.99" - reason: fixed in 8.0.0 to be backported to 7.10.0 + version: " - 7.9.99" + reason: fixed in 7.10.0 - do: indices.create: @@ -218,3 +218,27 @@ - match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828740" } - match: {hits.hits.0.sort: [1571617804828740000] } + +--- +"_shard_doc sort": + - skip: + version: " - 7.99.99" + reason: _shard_doc sort was added in 8.0 (TODO adapt version after backport) + + - do: + indices.create: + index: test + - do: + index: + index: test + id: 1 + body: { id: 1, foo: bar, age: 18 } + + - do: + catch: /\[_shard_doc\] sort field cannot be used without \[point in time\]/ + search: + index: test + body: + size: 1 + sort: [{ _shard_doc }] + search_after: [ 0L ] diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index db129a9479934..2ee2684530ae3 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -183,7 +183,7 @@ public static Template resolveTemplate(final String matchingTemplate, final Stri resolvedAliases, tempClusterState.metadata(), aliasValidator, xContentRegistry, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()))); + tempIndexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap()))); Map aliasesByName = aliases.stream().collect( Collectors.toMap(AliasMetadata::getAlias, Function.identity())); diff --git a/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java b/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java index a84d5885a73c3..687adde514839 100644 --- a/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java +++ b/server/src/main/java/org/elasticsearch/action/search/QueryPhaseResultConsumer.java @@ -290,9 +290,13 @@ long ramBytesUsedQueryResult(QuerySearchResult result) { if (hasAggs == false) { return 0; } - return result.aggregations() - .asSerialized(InternalAggregations::readFrom, namedWriteableRegistry) - .ramBytesUsed(); + if (result.aggregations().isSerialized()) { + return result.aggregations() + .asSerialized(InternalAggregations::readFrom, namedWriteableRegistry) + .ramBytesUsed(); + } else { + return result.aggregations().expand().getSerializedSize(); + } } /** diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java index e6e07dbc67904..1b8275af4ade0 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequest.java @@ -34,6 +34,9 @@ import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.SearchContext; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.ShardDocSortField; import org.elasticsearch.tasks.TaskId; import java.io.IOException; @@ -280,6 +283,14 @@ public ActionRequestValidationException validate() { if (scroll) { validationException = addValidationError("using [point in time] is not allowed in a scroll context", validationException); } + } else if (source != null && source.sorts() != null) { + for (SortBuilder sortBuilder : source.sorts()) { + if (sortBuilder instanceof FieldSortBuilder + && ShardDocSortField.NAME.equals(((FieldSortBuilder) sortBuilder).getFieldName())) { + validationException = addValidationError("[" + FieldSortBuilder.SHARD_DOC_FIELD_NAME + + "] sort field cannot be used without [point in time]", validationException); + } + } } return validationException; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 0b5210baabfd2..fec1f15962d24 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -487,7 +487,7 @@ private ClusterState applyCreateIndexRequestWithV1Templates(final ClusterState c MetadataIndexTemplateService.resolveAliases(templates), currentState.metadata(), aliasValidator, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), + xContentRegistry, indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap())), templates.stream().map(IndexTemplateMetadata::getName).collect(toList()), metadataTransformer); } @@ -520,7 +520,7 @@ private ClusterState applyCreateIndexRequestWithV2Template(final ClusterState cu MetadataIndexTemplateService.resolveAliases(currentState.metadata(), templateName), currentState.metadata(), aliasValidator, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), + xContentRegistry, indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap())), Collections.singletonList(templateName), metadataTransformer); } @@ -566,7 +566,7 @@ private ClusterState applyCreateIndexRequestWithExistingMetadata(final ClusterSt currentState.metadata(), aliasValidator, xContentRegistry, // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())), + indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap())), List.of(), metadataTransformer); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java index 3ced1267655f1..605c569a86dbf 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexAliasesService.java @@ -149,8 +149,8 @@ public ClusterState applyAliasActions(ClusterState currentState, Iterable System.currentTimeMillis(), null, emptyMap()), xContentRegistry); + aliasValidator.validateAliasFilter(alias, filter, indexService.newQueryShardContext(0, -1, + null, () -> System.currentTimeMillis(), null, emptyMap()), xContentRegistry); } }; if (action.apply(newAliasValidator, metadata, index)) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index ddab81145eff3..3f40294737206 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1118,7 +1118,7 @@ private static void validateCompositeTemplate(final ClusterState state, new AliasValidator(), // the context is only used for validation so it's fine to pass fake values for the // shard id and the current timestamp - xContentRegistry, tempIndexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap())); + xContentRegistry, tempIndexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap())); // triggers inclusion of _timestamp field and its validation: String indexName = DataStream.BACKING_INDEX_PREFIX + temporaryIndexName; diff --git a/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java index 1e58f12985c17..1ced33ce6ed7f 100644 --- a/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java +++ b/server/src/main/java/org/elasticsearch/common/io/stream/DelayableWriteable.java @@ -78,7 +78,7 @@ private DelayableWriteable() {} * {@code true} if the {@linkplain Writeable} is being stored in * serialized form, {@code false} otherwise. */ - abstract boolean isSerialized(); + public abstract boolean isSerialized(); private static class Referencing extends DelayableWriteable { private final T reference; @@ -109,7 +109,7 @@ public Serialized asSerialized(Reader reader, NamedWriteableRegistry regis } @Override - boolean isSerialized() { + public boolean isSerialized() { return false; } @@ -179,7 +179,7 @@ public Serialized asSerialized(Reader reader, NamedWriteableRegistry regis } @Override - boolean isSerialized() { + public boolean isSerialized() { return true; } diff --git a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java index b9f8bb98b4273..865bb793b4438 100644 --- a/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java +++ b/server/src/main/java/org/elasticsearch/common/lucene/Lucene.java @@ -93,6 +93,7 @@ import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.search.sort.ShardDocSortField; import java.io.IOException; import java.math.BigInteger; @@ -566,29 +567,35 @@ public static void writeSortType(StreamOutput out, SortField.Type sortType) thro out.writeVInt(sortType.ordinal()); } - public static void writeSortField(StreamOutput out, SortField sortField) throws IOException { + /** + * Returns the generic version of the provided {@link SortField} that + * can be used to merge documents coming from different shards. + */ + private static SortField rewriteMergeSortField(SortField sortField) { if (sortField.getClass() == GEO_DISTANCE_SORT_TYPE_CLASS) { - // for geo sorting, we replace the SortField with a SortField that assumes a double field. - // this works since the SortField is only used for merging top docs SortField newSortField = new SortField(sortField.getField(), SortField.Type.DOUBLE); newSortField.setMissingValue(sortField.getMissingValue()); - sortField = newSortField; + return newSortField; } else if (sortField.getClass() == SortedSetSortField.class) { - // for multi-valued sort field, we replace the SortedSetSortField with a simple SortField. - // It works because the sort field is only used to merge results from different shards. SortField newSortField = new SortField(sortField.getField(), SortField.Type.STRING, sortField.getReverse()); newSortField.setMissingValue(sortField.getMissingValue()); - sortField = newSortField; + return newSortField; } else if (sortField.getClass() == SortedNumericSortField.class) { - // for multi-valued sort field, we replace the SortedSetSortField with a simple SortField. - // It works because the sort field is only used to merge results from different shards. SortField newSortField = new SortField(sortField.getField(), ((SortedNumericSortField) sortField).getNumericType(), sortField.getReverse()); newSortField.setMissingValue(sortField.getMissingValue()); - sortField = newSortField; + return sortField; + } else if (sortField.getClass() == ShardDocSortField.class) { + SortField newSortField = new SortField(sortField.getField(), SortField.Type.LONG, sortField.getReverse()); + return newSortField; + } else { + return sortField; } + } + public static void writeSortField(StreamOutput out, SortField sortField) throws IOException { + sortField = rewriteMergeSortField(sortField); if (sortField.getClass() != SortField.class) { throw new IllegalArgumentException("Cannot serialize SortField impl [" + sortField + "]"); } diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index bec23def8b1c3..18c8bfa5d77c0 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -192,7 +192,7 @@ public IndexService( assert indexAnalyzers != null; this.mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, // we parse all percolator queries as they would be parsed on shard 0 - () -> newQueryShardContext(0, null, System::currentTimeMillis, null, emptyMap()), idFieldDataEnabled, scriptService); + () -> newQueryShardContext(0, 0, null, System::currentTimeMillis, null, emptyMap()), idFieldDataEnabled, scriptService); this.indexFieldData = new IndexFieldDataService(indexSettings, indicesFieldDataCache, circuitBreakerService, mapperService); if (indexSettings.getIndexSortConfig().hasIndexSort()) { // we delay the actual creation of the sort order for this index because the mapping has not been merged yet. @@ -588,6 +588,7 @@ public IndexSettings getIndexSettings() { */ public QueryShardContext newQueryShardContext( int shardId, + int shardRequestIndex, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias, @@ -596,9 +597,26 @@ public QueryShardContext newQueryShardContext( final SearchIndexNameMatcher indexNameMatcher = new SearchIndexNameMatcher(index().getName(), clusterAlias, clusterService, expressionResolver); return new QueryShardContext( - shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(), - similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias, - indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry, runtimeMappings); + shardId, + shardRequestIndex, + indexSettings, + bigArrays, + indexCache.bitsetFilterCache(), + indexFieldData::getForField, + mapperService(), + similarityService(), + scriptService, + xContentRegistry, + namedWriteableRegistry, + client, + searcher, + nowInMillis, + clusterAlias, + indexNameMatcher, + allowExpensiveQueries, + valuesSourceRegistry, + runtimeMappings + ); } /** diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index d6a358f5aad1d..2df7b86f36dce 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -94,6 +94,7 @@ public class QueryShardContext extends QueryRewriteContext { private final BitsetFilterCache bitsetFilterCache; private final TriFunction, IndexFieldData> indexFieldDataService; private final int shardId; + private final int shardRequestIndex; private final IndexSearcher searcher; private boolean cacheable = true; private final SetOnce frozen = new SetOnce<>(); @@ -114,6 +115,7 @@ public class QueryShardContext extends QueryRewriteContext { */ public QueryShardContext( int shardId, + int shardRequestIndex, IndexSettings indexSettings, BigArrays bigArrays, BitsetFilterCache bitsetFilterCache, @@ -134,6 +136,7 @@ public QueryShardContext( ) { this( shardId, + shardRequestIndex, indexSettings, bigArrays, bitsetFilterCache, @@ -158,13 +161,30 @@ public QueryShardContext( } public QueryShardContext(QueryShardContext source) { - this(source.shardId, source.indexSettings, source.bigArrays, source.bitsetFilterCache, source.indexFieldDataService, - source.mapperService, source.similarityService, source.scriptService, source.getXContentRegistry(), - source.getWriteableRegistry(), source.client, source.searcher, source.nowInMillis, source.indexNameMatcher, - source.fullyQualifiedIndex, source.allowExpensiveQueries, source.valuesSourceRegistry, source.runtimeMappings); + this( + source.shardId, + source.shardRequestIndex, + source.indexSettings, + source.bigArrays, + source.bitsetFilterCache, + source.indexFieldDataService, + source.mapperService, + source.similarityService, + source.scriptService, + source.getXContentRegistry(), + source.getWriteableRegistry(), + source.client, source.searcher, + source.nowInMillis, + source.indexNameMatcher, + source.fullyQualifiedIndex, + source.allowExpensiveQueries, + source.valuesSourceRegistry, + source.runtimeMappings + ); } private QueryShardContext(int shardId, + int shardRequestIndex, IndexSettings indexSettings, BigArrays bigArrays, BitsetFilterCache bitsetFilterCache, @@ -184,6 +204,7 @@ private QueryShardContext(int shardId, Map runtimeMappings) { super(xContentRegistry, namedWriteableRegistry, client, nowInMillis); this.shardId = shardId; + this.shardRequestIndex = shardRequestIndex; this.similarityService = similarityService; this.mapperService = mapperService; this.bigArrays = bigArrays; @@ -520,6 +541,14 @@ public int getShardId() { return shardId; } + /** + * Returns the shard request ordinal that is used by the main search request + * to reference this shard. + */ + public int getShardRequestIndex() { + return shardRequestIndex; + } + @Override public final long nowInMillis() { failIfFrozen(); diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 6184b18048be9..a04607d5752ac 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -166,7 +166,8 @@ final class DefaultSearchContext extends SearchContext { this.timeout = timeout; queryShardContext = indexService.newQueryShardContext( request.shardId().id(), - this.searcher, + request.shardRequestIndex(), + searcher, request::nowInMillis, shardTarget.getClusterAlias(), request.getRuntimeMappings() diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 8c416d9b0912f..90f8dc4cbbd15 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1174,8 +1174,8 @@ private CanMatchResponse canMatch(ShardSearchRequest request, boolean checkRefre } try (canMatchSearcher) { - QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), canMatchSearcher, - request::nowInMillis, request.getClusterAlias(), request.getRuntimeMappings()); + QueryShardContext context = indexService.newQueryShardContext(request.shardId().id(), 0, + canMatchSearcher, request::nowInMillis, request.getClusterAlias(), request.getRuntimeMappings()); Rewriteable.rewrite(request.getRewriteable(), context, false); final boolean aliasFilterCanMatch = request.getAliasFilter() .getQueryBuilder() instanceof MatchNoneQueryBuilder == false; diff --git a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java index 1d768ab8c7b63..7a3a6a145ea04 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/search/internal/ShardSearchRequest.java @@ -72,7 +72,7 @@ public class ShardSearchRequest extends TransportRequest implements IndicesRequest { private final String clusterAlias; private final ShardId shardId; - private final int shardIndex; + private final int shardRequestIndex; private final int numberOfShards; private final SearchType searchType; private final Scroll scroll; @@ -94,20 +94,20 @@ public class ShardSearchRequest extends TransportRequest implements IndicesReque public ShardSearchRequest(OriginalIndices originalIndices, SearchRequest searchRequest, ShardId shardId, - int shardIndex, + int shardRequestIndex, int numberOfShards, AliasFilter aliasFilter, float indexBoost, long nowInMillis, @Nullable String clusterAlias) { - this(originalIndices, searchRequest, shardId, shardIndex, numberOfShards, aliasFilter, + this(originalIndices, searchRequest, shardId, shardRequestIndex, numberOfShards, aliasFilter, indexBoost, nowInMillis, clusterAlias, null, null); } public ShardSearchRequest(OriginalIndices originalIndices, SearchRequest searchRequest, ShardId shardId, - int shardIndex, + int shardRequestIndex, int numberOfShards, AliasFilter aliasFilter, float indexBoost, @@ -117,7 +117,7 @@ public ShardSearchRequest(OriginalIndices originalIndices, TimeValue keepAlive) { this(originalIndices, shardId, - shardIndex, + shardRequestIndex, numberOfShards, searchRequest.searchType(), searchRequest.source(), @@ -144,7 +144,7 @@ public ShardSearchRequest(ShardId shardId, private ShardSearchRequest(OriginalIndices originalIndices, ShardId shardId, - int shardIndex, + int shardRequestIndex, int numberOfShards, SearchType searchType, SearchSourceBuilder source, @@ -158,7 +158,7 @@ private ShardSearchRequest(OriginalIndices originalIndices, ShardSearchContextId readerId, TimeValue keepAlive) { this.shardId = shardId; - this.shardIndex = shardIndex; + this.shardRequestIndex = shardRequestIndex; this.numberOfShards = numberOfShards; this.searchType = searchType; this.source = source; @@ -179,7 +179,7 @@ public ShardSearchRequest(StreamInput in) throws IOException { super(in); shardId = new ShardId(in); searchType = SearchType.fromId(in.readByte()); - shardIndex = in.getVersion().onOrAfter(Version.V_7_11_0) ? in.readVInt() : -1; + shardRequestIndex = in.getVersion().onOrAfter(Version.V_7_11_0) ? in.readVInt() : -1; numberOfShards = in.readVInt(); scroll = in.readOptionalWriteable(Scroll::new); source = in.readOptionalWriteable(SearchSourceBuilder::new); @@ -218,7 +218,7 @@ public ShardSearchRequest(StreamInput in) throws IOException { public ShardSearchRequest(ShardSearchRequest clone) { this.shardId = clone.shardId; - this.shardIndex = clone.shardIndex; + this.shardRequestIndex = clone.shardRequestIndex; this.searchType = clone.searchType; this.numberOfShards = clone.numberOfShards; this.scroll = clone.scroll; @@ -248,7 +248,7 @@ protected final void innerWriteTo(StreamOutput out, boolean asKey) throws IOExce out.writeByte(searchType.id()); if (asKey == false) { if (out.getVersion().onOrAfter(Version.V_7_11_0)) { - out.writeVInt(shardIndex); + out.writeVInt(shardRequestIndex); } out.writeVInt(numberOfShards); } @@ -315,10 +315,11 @@ public void source(SearchSourceBuilder source) { } /** - * Returns the index of the shard that is used to tiebreak documents with identical sort values. + * Returns the shard request ordinal that is used by the main search request + * to reference this shard. */ - public int shardIndex() { - return shardIndex; + public int shardRequestIndex() { + return shardRequestIndex; } public int numberOfShards() { diff --git a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java index 29e95e4b7d680..a6d25efece134 100644 --- a/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/slice/SliceBuilder.java @@ -203,8 +203,8 @@ public Query toFilter(ShardSearchRequest request, QueryShardContext context) { throw new IllegalArgumentException("field " + field + " not found"); } - int shardIndex = request.shardIndex() != -1 ? request.shardIndex() : request.shardId().id(); - int numShards = request.shardIndex() != -1 ? request.numberOfShards() : context.getIndexSettings().getNumberOfShards(); + int shardIndex = request.shardRequestIndex() != -1 ? request.shardRequestIndex() : request.shardId().id(); + int numShards = request.shardRequestIndex() != -1 ? request.numberOfShards() : context.getIndexSettings().getNumberOfShards(); String field = this.field; boolean useTermQuery = false; if (IdFieldMapper.NAME.equals(field)) { diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index 7f197d3398038..d7eb3d5aaf024 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -81,6 +81,12 @@ public class FieldSortBuilder extends SortBuilder { * special field name to sort by index order */ public static final String DOC_FIELD_NAME = "_doc"; + + /** + * special field name to sort by index order + */ + public static final String SHARD_DOC_FIELD_NAME = ShardDocSortField.NAME; + private static final SortFieldAndFormat SORT_DOC = new SortFieldAndFormat( new SortField(null, SortField.Type.DOC), DocValueFormat.RAW); private static final SortFieldAndFormat SORT_DOC_REVERSE = new SortFieldAndFormat( @@ -321,8 +327,15 @@ private static NumericType resolveNumericType(String value) { @Override public SortFieldAndFormat build(QueryShardContext context) throws IOException { + final boolean reverse = order == SortOrder.DESC; + if (DOC_FIELD_NAME.equals(fieldName)) { - return order == SortOrder.DESC ? SORT_DOC_REVERSE : SORT_DOC; + return reverse ? SORT_DOC_REVERSE : SORT_DOC; + } else if (SHARD_DOC_FIELD_NAME.equals(fieldName)) { + if (context.getShardRequestIndex() == -1) { + throw new IllegalArgumentException(""); + } + return new SortFieldAndFormat(new ShardDocSortField(context.getShardRequestIndex(), reverse), DocValueFormat.RAW); } MappedFieldType fieldType = context.getFieldType(fieldName); @@ -331,7 +344,6 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException { fieldType = resolveUnmappedType(context); } - boolean reverse = order == SortOrder.DESC; IndexFieldData fieldData = context.getForField(fieldType); if (fieldData instanceof IndexNumericFieldData == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) { diff --git a/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java b/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java new file mode 100644 index 0000000000000..47907875d8a8d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/search/sort/ShardDocSortField.java @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.search.sort; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.FieldComparator; +import org.apache.lucene.search.LeafFieldComparator; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.comparators.DocComparator; + + /** + * A {@link SortField} that first compares the shard index and then uses the document number (_doc) + * to tiebreak if the value is the same. + **/ +public class ShardDocSortField extends SortField { + public static final String NAME = "_shard_doc"; + + private final int shardRequestIndex; + + public ShardDocSortField(int shardRequestIndex, boolean reverse) { + super(NAME, Type.LONG, reverse); + assert shardRequestIndex >= 0; + this.shardRequestIndex = shardRequestIndex; + } + + int getShardRequestIndex() { + return shardRequestIndex; + } + + @Override + public FieldComparator getComparator(int numHits, int sortPos) { + final DocComparator delegate = new DocComparator(numHits, false, sortPos); + + return new FieldComparator() { + @Override + public int compare(int slot1, int slot2) { + return delegate.compare(slot1, slot2); + } + + @Override + public int compareValues(Long first, Long second) { + return Long.compare(first, second); + } + + @Override + public void setTopValue(Long value) { + int topShardIndex = (int) (value >> 32); + if (shardRequestIndex == topShardIndex) { + delegate.setTopValue(value.intValue()); + } else if (shardRequestIndex < topShardIndex) { + delegate.setTopValue(Integer.MAX_VALUE); + } else { + delegate.setTopValue(-1); + } + } + + @Override + public Long value(int slot) { + return (((long) shardRequestIndex) << 32) | (delegate.value(slot) & 0xFFFFFFFFL); + } + + @Override + public LeafFieldComparator getLeafComparator(LeafReaderContext context) { + return delegate.getLeafComparator(context); + } + }; + } +} diff --git a/server/src/main/java/org/elasticsearch/search/sort/SortBuilders.java b/server/src/main/java/org/elasticsearch/search/sort/SortBuilders.java index 3eae9b8d01960..7f5d98dee82ec 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/SortBuilders.java +++ b/server/src/main/java/org/elasticsearch/search/sort/SortBuilders.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.script.Script; +import org.elasticsearch.search.builder.PointInTimeBuilder; import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType; /** @@ -46,6 +47,13 @@ public static FieldSortBuilder fieldSort(String field) { return new FieldSortBuilder(field); } + /** + * Constructs a sort tiebreaker that can be used within a point in time reader {@link PointInTimeBuilder}. + */ + public static FieldSortBuilder pitTiebreaker() { + return new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME); + } + /** * Constructs a new script based sort. * diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 4c5f8c08abedc..376de20cabda2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -123,7 +123,7 @@ public void setupCreateIndexRequestAndAliasValidator() { request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test"); Settings indexSettings = Settings.builder().put(SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); - queryShardContext = new QueryShardContext(0, + queryShardContext = new QueryShardContext(0, 0, new IndexSettings(IndexMetadata.builder("test").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> randomNonNegativeLong(), null, null, () -> true, null, emptyMap()); diff --git a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java index a716350638d9d..2da356426e43e 100644 --- a/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java +++ b/server/src/test/java/org/elasticsearch/common/lucene/LuceneTests.java @@ -68,6 +68,7 @@ import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource; import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.search.sort.ShardDocSortField; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -716,7 +717,7 @@ private static Tuple randomSortFieldCustomComparatorSource private static Tuple randomCustomSortField() { String field = randomAlphaOfLengthBetween(3, 10); - switch(randomIntBetween(0, 2)) { + switch(randomIntBetween(0, 3)) { case 0: { SortField sortField = LatLonDocValuesField.newDistanceSort(field, 0, 0); SortField expected = new SortField(field, SortField.Type.DOUBLE); @@ -742,6 +743,11 @@ private static Tuple randomCustomSortField() { } return Tuple.tuple(sortField, expected); } + case 3: { + ShardDocSortField sortField = new ShardDocSortField(randomIntBetween(0, 100), randomBoolean()); + SortField expected = new SortField(ShardDocSortField.NAME, SortField.Type.LONG, sortField.getReverse()); + return Tuple.tuple(sortField, expected); + } default: throw new UnsupportedOperationException(); } diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java index 0438577b219c6..292d7fb034917 100644 --- a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java @@ -139,7 +139,7 @@ public void setup() throws Exception { writer = new IndexWriter( new ByteBuffersDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()) ); - shardContext = indexService.newQueryShardContext(0, null, () -> 0, null, emptyMap()); + shardContext = indexService.newQueryShardContext(0, 0, null, () -> 0, null, emptyMap()); } protected final List refreshReader() throws Exception { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index d7ce0a3460e23..06d42b41af0ad 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -165,7 +165,7 @@ public void testValueForSearch() { public void testTermQuery() { Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); - QueryShardContext context = new QueryShardContext(0, + QueryShardContext context = new QueryShardContext(0, 0, new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); @@ -187,7 +187,7 @@ public void testTermQuery() { public void testRangeQuery() throws IOException { Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build(); - QueryShardContext context = new QueryShardContext(0, + QueryShardContext context = new QueryShardContext(0, 0, new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); @@ -232,7 +232,7 @@ public void testRangeQueryWithIndexSort() { .build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); - QueryShardContext context = new QueryShardContext(0, indexSettings, + QueryShardContext context = new QueryShardContext(0, 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> 0L, null, null, () -> true, null, emptyMap()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java index 7e93b0e79c63c..c78346f469439 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/FieldNamesFieldTypeTests.java @@ -50,7 +50,7 @@ public void testTermQuery() { when(mapperService.fieldType("field_name")).thenReturn(fieldType); when(mapperService.simpleMatchToFullName("field_name")).thenReturn(Collections.singleton("field_name")); - QueryShardContext queryShardContext = new QueryShardContext(0, + QueryShardContext queryShardContext = new QueryShardContext(0, 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, null, null, null, null, () -> 0L, null, null, () -> true, null, emptyMap()); Query termQuery = fieldNamesFieldType.termQuery("field_name", queryShardContext); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java index b858929e3abc4..cd24c8bb48d11 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/IndexFieldTypeTests.java @@ -69,7 +69,7 @@ private QueryShardContext createContext() { IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY); Predicate indexNameMatcher = pattern -> Regex.simpleMatch(pattern, "index"); - return new QueryShardContext(0, indexSettings, null, null, null, null, null, null, xContentRegistry(), writableRegistry(), + return new QueryShardContext(0, 0, indexSettings, null, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, System::currentTimeMillis, null, indexNameMatcher, () -> true, null, emptyMap()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java index ac763d35a0c3f..397823d8347a7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NumberFieldTypeTests.java @@ -473,7 +473,7 @@ public void doTestIndexSortRangeQueries(NumberType type, Supplier valueS DirectoryReader reader = DirectoryReader.open(w); IndexSearcher searcher = newSearcher(reader); - QueryShardContext context = new QueryShardContext(0, indexSettings, + QueryShardContext context = new QueryShardContext(0, 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> 0L, null, null, () -> true, null, emptyMap()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java index a4c026f6d9799..7c4465c03c1d1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/RangeFieldTypeTests.java @@ -213,7 +213,7 @@ private QueryShardContext createContext() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); - return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, + return new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); } diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java index 4bd5bce9f479b..47f955fc4a48a 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryShardContextTests.java @@ -195,6 +195,7 @@ public void testIndexSortedOnField() { IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); QueryShardContext context = new QueryShardContext( + 0, 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, @@ -374,7 +375,7 @@ private static QueryShardContext createQueryShardContext( MapperService mapperService = createMapperService(indexUuid, fieldTypeLookup, mapperPlugins); final long nowInMillis = randomNonNegativeLong(); return new QueryShardContext( - 0, mapperService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, + 0, 0, mapperService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, (mappedFieldType, idxName, searchLookup) -> mappedFieldType.fielddataBuilder(idxName, searchLookup).build(null, null), mapperService, null, null, NamedXContentRegistry.EMPTY, new NamedWriteableRegistry(Collections.emptyList()), null, null, () -> nowInMillis, clusterAlias, null, () -> true, null, runtimeMappings); diff --git a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java index 9ab1e80f09bdc..d7f6cb3a6c61d 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RangeQueryRewriteTests.java @@ -41,7 +41,7 @@ public class RangeQueryRewriteTests extends ESSingleNodeTestCase { public void testRewriteMissingField() throws Exception { IndexService indexService = createIndex("test"); IndexReader reader = new MultiReader(); - QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, + QueryRewriteContext context = new QueryShardContext(0, 0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), null, new IndexSearcher(reader), null, null, null, () -> true, null, emptyMap()); RangeQueryBuilder range = new RangeQueryBuilder("foo"); @@ -59,7 +59,7 @@ public void testRewriteMissingReader() throws Exception { .endObject().endObject()); indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); - QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), null, null, null, + QueryRewriteContext context = new QueryShardContext(0, 0, indexService.getIndexSettings(), null, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), null, null, null, null, null, () -> true, null, emptyMap()); RangeQueryBuilder range = new RangeQueryBuilder("foo"); @@ -79,7 +79,7 @@ public void testRewriteEmptyReader() throws Exception { indexService.mapperService().merge("type", new CompressedXContent(mapping), MergeReason.MAPPING_UPDATE); IndexReader reader = new MultiReader(); - QueryRewriteContext context = new QueryShardContext(0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, + QueryRewriteContext context = new QueryShardContext(0, 0, indexService.getIndexSettings(), BigArrays.NON_RECYCLING_INSTANCE, null, null, indexService.mapperService(), null, null, xContentRegistry(), writableRegistry(), null, new IndexSearcher(reader), null, null, null, () -> true, null, emptyMap()); RangeQueryBuilder range = new RangeQueryBuilder("foo"); diff --git a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java index d633b7248e2cb..840a20ccf82ad 100644 --- a/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/MultiMatchQueryTests.java @@ -100,7 +100,13 @@ public void setup() throws IOException { public void testCrossFieldMultiMatchQuery() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ); queryShardContext.setAllowUnmappedFields(true); for (float tieBreaker : new float[] {0.0f, 0.5f}) { Query parsedQuery = multiMatchQuery("banon") @@ -127,7 +133,14 @@ public void testBlendTerms() { float[] boosts = new float[] {2, 3}; Query expected = BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + indexService.newQueryShardContext( + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ), new BytesRef("baz"), 1f, false, @@ -151,7 +164,14 @@ public Query termQuery(Object value, QueryShardContext context) { BlendedTermQuery.dismaxBlendedQuery(terms, boosts, 1.0f) ), 1f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + indexService.newQueryShardContext( + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ), new BytesRef("baz"), 1f, true, @@ -168,7 +188,14 @@ public Query termQuery(Object value, QueryShardContext context) { } }; expectThrows(IllegalArgumentException.class, () -> MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + indexService.newQueryShardContext( + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ), new BytesRef("baz"), 1f, false, Arrays.asList(new FieldAndBoost(ft, 1)))); } @@ -190,7 +217,14 @@ public Query termQuery(Object value, QueryShardContext context) { expectedDisjunct1 ), 1.0f); Query actual = MultiMatchQuery.blendTerm( - indexService.newQueryShardContext(randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()), + indexService.newQueryShardContext( + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ), new BytesRef("baz"), 1f, false, @@ -201,7 +235,13 @@ public Query termQuery(Object value, QueryShardContext context) { public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); parser.setAnalyzer(new MockSynonymAnalyzer()); @@ -233,7 +273,13 @@ public void testMultiMatchCrossFieldsWithSynonyms() throws IOException { public void testMultiMatchCrossFieldsWithSynonymsPhrase() throws IOException { QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null, emptyMap()); + randomInt(20), + 0, + null, + () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); parser.setAnalyzer(new MockSynonymAnalyzer()); Map fieldNames = new HashMap<>(); @@ -300,9 +346,12 @@ public void testKeywordSplitQueriesOnWhitespace() throws IOException { .endObject().endObject()); mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); QueryShardContext queryShardContext = indexService.newQueryShardContext( - randomInt(20), null, () -> { - throw new UnsupportedOperationException(); - }, null, emptyMap()); + randomInt(20), + 0, + null, () -> { throw new UnsupportedOperationException(); }, + null, + emptyMap() + ); MultiMatchQuery parser = new MultiMatchQuery(queryShardContext); Map fieldNames = new HashMap<>(); fieldNames.put("field", 1.0f); diff --git a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java index d8f7405ef3387..6386d90d0a788 100644 --- a/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/NestedHelperTests.java @@ -335,7 +335,14 @@ public void testConjunction() { } public void testNested() throws IOException { - QueryShardContext context = indexService.newQueryShardContext(0, new IndexSearcher(new MultiReader()), () -> 0, null, emptyMap()); + QueryShardContext context = indexService.newQueryShardContext( + 0, + 0, + new IndexSearcher(new MultiReader()), + () -> 0, + null, + emptyMap() + ); NestedQueryBuilder queryBuilder = new NestedQueryBuilder("nested1", new MatchAllQueryBuilder(), ScoreMode.Avg); ESToParentBlockJoinQuery query = (ESToParentBlockJoinQuery) queryBuilder.toQuery(context); diff --git a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java index e4b0573b06cca..cd7399ea3256d 100644 --- a/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java +++ b/server/src/test/java/org/elasticsearch/index/search/nested/NestedSortingTests.java @@ -608,7 +608,7 @@ public void testMultiLevelNestedSorting() throws IOException { DirectoryReader reader = DirectoryReader.open(writer); reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId(indexService.index(), 0)); IndexSearcher searcher = new IndexSearcher(reader); - QueryShardContext queryShardContext = indexService.newQueryShardContext(0, searcher, () -> 0L, null, emptyMap()); + QueryShardContext queryShardContext = indexService.newQueryShardContext(0, 0, searcher, () -> 0L, null, emptyMap()); FieldSortBuilder sortBuilder = new FieldSortBuilder("chapters.paragraphs.word_count"); sortBuilder.setNestedSort(new NestedSortBuilder("chapters").setNestedSort(new NestedSortBuilder("chapters.paragraphs"))); diff --git a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java index e4384bd344a36..b426df9e21895 100644 --- a/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java +++ b/server/src/test/java/org/elasticsearch/search/DefaultSearchContextTests.java @@ -79,7 +79,7 @@ public void testPreProcess() throws Exception { when(shardSearchRequest.searchType()).thenReturn(SearchType.DEFAULT); ShardId shardId = new ShardId("index", UUID.randomUUID().toString(), 1); when(shardSearchRequest.shardId()).thenReturn(shardId); - when(shardSearchRequest.shardIndex()).thenReturn(shardId.id()); + when(shardSearchRequest.shardRequestIndex()).thenReturn(shardId.id()); when(shardSearchRequest.numberOfShards()).thenReturn(2); ThreadPool threadPool = new TestThreadPool(this.getClass().getName()); @@ -106,7 +106,7 @@ public void testPreProcess() throws Exception { when(indexCache.query()).thenReturn(queryCache); when(indexService.cache()).thenReturn(indexCache); QueryShardContext queryShardContext = mock(QueryShardContext.class); - when(indexService.newQueryShardContext(eq(shardId.id()), anyObject(), anyObject(), anyString(), anyObject())).thenReturn( + when(indexService.newQueryShardContext(eq(shardId.id()), 0, anyObject(), anyObject(), anyString(), anyObject())).thenReturn( queryShardContext ); MapperService mapperService = mock(MapperService.class); diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index 00d17464f4e70..49193f482d831 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -710,7 +710,7 @@ private static QueryShardContext newQueryShardContext(MapperService mapperServic .put(IndexMetadata.SETTING_INDEX_UUID, "uuid").build(); IndexMetadata indexMetadata = new IndexMetadata.Builder("index").settings(settings).build(); IndexSettings indexSettings = new IndexSettings(indexMetadata, settings); - return new QueryShardContext(0, indexSettings, null, null, null, mapperService, null, null, null, null, null, null, null, null, + return new QueryShardContext(0, 0, indexSettings, null, null, null, mapperService, null, null, null, null, null, null, null, null, null, null, null, emptyMap()); } } diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java index f30158f2fbad0..6bf9fb5ee116f 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/highlight/HighlightBuilderTests.java @@ -278,7 +278,7 @@ public void testBuildSearchContextHighlight() throws IOException { Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_"); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in highlighter - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, null, null, System::currentTimeMillis, null, null, () -> true, null, emptyMap()) { @Override diff --git a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java index 4c2749c5c0c7b..6e04efbd5d170 100644 --- a/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/rescore/QueryRescorerBuilderTests.java @@ -142,7 +142,7 @@ public void testBuildRescoreSearchContext() throws ElasticsearchParseException, .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()) { @Override @@ -186,7 +186,7 @@ public void testRewritingKeepsSettings() throws IOException { .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build(); IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(randomAlphaOfLengthBetween(1, 10), indexSettings); // shard context will only need indicesQueriesRegistry for building Query objects nested in query rescorer - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), namedWriteableRegistry, null, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()) { @Override diff --git a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java index 38a721fd117dc..e9cb1ce7aff37 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/sort/AbstractSortTestCase.java @@ -201,7 +201,7 @@ protected final QueryShardContext createMockShardContext(IndexSearcher searcher) IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null); }; - return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, + return new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, null, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, searcher, () -> randomNonNegativeLong(), null, null, () -> true, null, emptyMap()) { diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index 89d6aa235f1df..c30fb30306753 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -72,6 +72,7 @@ import static org.elasticsearch.search.sort.FieldSortBuilder.getMinMaxOrNull; import static org.elasticsearch.search.sort.FieldSortBuilder.getPrimaryFieldSortOrNull; import static org.elasticsearch.search.sort.NestedSortBuilderTests.createRandomNestedSort; +import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; public class FieldSortBuilderTests extends AbstractSortTestCase { @@ -160,6 +161,8 @@ protected void sortFieldAssertions(FieldSortBuilder builder, SortField sortField SortField.Type expectedType; if (builder.getFieldName().equals(FieldSortBuilder.DOC_FIELD_NAME)) { expectedType = SortField.Type.DOC; + } else if (builder.getFieldName().equals(FieldSortBuilder.SHARD_DOC_FIELD_NAME)) { + expectedType = SortField.Type.LONG; } else { expectedType = SortField.Type.CUSTOM; } @@ -324,6 +327,20 @@ public void testUnknownOptionFails() throws IOException { } } + public void testShardDocSort() throws IOException { + QueryShardContext shardContextMock = createMockShardContext(); + + boolean reverse = randomBoolean(); + FieldSortBuilder sortBuilder = new FieldSortBuilder(FieldSortBuilder.SHARD_DOC_FIELD_NAME) + .order(reverse ? SortOrder.DESC : SortOrder.ASC); + SortFieldAndFormat sortAndFormat = sortBuilder.build(shardContextMock); + assertThat(sortAndFormat.field.getClass(), equalTo(ShardDocSortField.class)); + ShardDocSortField sortField = (ShardDocSortField) sortAndFormat.field; + assertThat(sortField.getShardRequestIndex(), equalTo(shardContextMock.getShardRequestIndex())); + assertThat(sortField.getReverse(), equalTo(reverse)); + assertThat(sortAndFormat.format, equalTo(DocValueFormat.RAW)); + } + @Override protected MappedFieldType provideMappedFieldType(String name) { if (name.equals(MAPPED_STRING_FIELDNAME)) { diff --git a/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java index 7b1e56d5c69f2..6c27c23b78573 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/sort/SortBuilderTests.java @@ -190,20 +190,25 @@ public static List> randomSortBuilderList() { int size = randomIntBetween(1, 5); List> list = new ArrayList<>(size); for (int i = 0; i < size; i++) { - switch (randomIntBetween(0, 3)) { + switch (randomIntBetween(0, 5)) { case 0: list.add(new ScoreSortBuilder()); break; case 1: - String fieldName = rarely() ? FieldSortBuilder.DOC_FIELD_NAME : randomAlphaOfLengthBetween(1, 10); - list.add(new FieldSortBuilder(fieldName)); + list.add(new FieldSortBuilder( randomAlphaOfLengthBetween(1, 10))); break; case 2: - list.add(GeoDistanceSortBuilderTests.randomGeoDistanceSortBuilder()); + list.add(SortBuilders.fieldSort(FieldSortBuilder.DOC_FIELD_NAME)); break; case 3: + list.add(GeoDistanceSortBuilderTests.randomGeoDistanceSortBuilder()); + break; + case 4: list.add(ScriptSortBuilderTests.randomScriptSortBuilder()); break; + case 5: + list.add(SortBuilders.pitTiebreaker()); + break; default: throw new IllegalStateException("unexpected randomization in tests"); } diff --git a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java index f9892be86028a..3ac6396e580d0 100644 --- a/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java +++ b/server/src/test/java/org/elasticsearch/search/suggest/AbstractSuggestionBuilderTestCase.java @@ -179,7 +179,7 @@ public NamedAnalyzer get(Object key) { when(mapperService.getIndexAnalyzers()).thenReturn(indexAnalyzers); when(scriptService.compile(any(Script.class), any())).then(invocation -> new TestTemplateService.MockTemplateScript.Factory( ((Script) invocation.getArguments()[0]).getIdOrCode())); - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, null, System::currentTimeMillis, null, null, () -> true, null, emptyMap()); @@ -219,7 +219,7 @@ public void testBuildWithUnmappedField() { when(mapperService.getNamedAnalyzer(any(String.class))).then( invocation -> new NamedAnalyzer((String) invocation.getArguments()[0], AnalyzerScope.INDEX, new SimpleAnalyzer())); - QueryShardContext mockShardContext = new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, + QueryShardContext mockShardContext = new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, null, System::currentTimeMillis, null, null, () -> true, null, emptyMap()); if (randomBoolean()) { diff --git a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java index 44f982df2beae..bca84b1c3cf67 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/search/aggregations/AggregatorTestCase.java @@ -271,6 +271,7 @@ protected AggregationContext createAggregationContext(IndexSearcher indexSearche searchLookup) -> fieldType.fielddataBuilder(mapperService.getIndexSettings().getIndex().getName(), searchLookup) .build(new IndexFieldDataCache.None(), breakerService); QueryShardContext queryShardContext = new QueryShardContext( + 0, 0, indexSettings, bigArrays, @@ -370,6 +371,17 @@ protected MapperService mapperServiceMock() { } /** + * Sub-tests that need a more complex index field data provider can override this + */ + protected TriFunction, IndexFieldData> getIndexFieldDataLookup( + MapperService mapperService, CircuitBreakerService circuitBreakerService) { + return (fieldType, s, searchLookup) -> fieldType.fielddataBuilder( + mapperService.getIndexSettings().getIndex().getName(), searchLookup) + .build(new IndexFieldDataCache.None(), circuitBreakerService); + } + + /** +>>>>>>> Stashed changes * Sub-tests that need scripting can override this method to provide a script service and pre-baked scripts */ protected ScriptService getMockScriptService() { diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java index f44cc47924576..b66156c28cc76 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractBuilderTestCase.java @@ -411,7 +411,7 @@ public void close() throws IOException { } QueryShardContext createShardContext(IndexSearcher searcher) { - return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, + return new QueryShardContext(0, 0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataService::getForField, mapperService, similarityService, scriptService, xContentRegistry, namedWriteableRegistry, this.client, searcher, () -> nowInMillis, null, indexNameMatcher(), () -> true, null, emptyMap()); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 1a15be52bb1ba..4533a18f391a3 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -96,7 +96,7 @@ public TestSearchContext(IndexService indexService) { this.indexService = indexService; this.fixedBitSetFilterCache = indexService.cache().bitsetFilterCache(); this.indexShard = indexService.getShardOrNull(0); - queryShardContext = indexService.newQueryShardContext(0, null, () -> 0L, null, emptyMap()); + queryShardContext = indexService.newQueryShardContext(0, 0, null, () -> 0L, null, emptyMap()); } public TestSearchContext(QueryShardContext queryShardContext) { diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/search/PointInTimeIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/search/PointInTimeIT.java index e39632966b440..75679a437de10 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/search/PointInTimeIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/search/PointInTimeIT.java @@ -9,6 +9,8 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.admin.indices.stats.CommonStats; import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.action.search.ShardSearchFailure; @@ -25,8 +27,12 @@ import org.elasticsearch.indices.IndicesService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchContextMissingException; +import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.builder.PointInTimeBuilder; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.search.action.ClosePointInTimeAction; @@ -37,6 +43,7 @@ import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -370,6 +377,85 @@ public void testPartialResults() throws Exception { } } + public void testPITTiebreak() throws Exception { + assertAcked(client().admin().indices().prepareDelete("index-*").get()); + int numIndex = randomIntBetween(2, 10); + int expectedNumDocs = 0; + for (int i = 0; i < numIndex; i++) { + String index = "index-" + i; + createIndex(index, Settings.builder().put("index.number_of_shards", 1).build()); + int numDocs = randomIntBetween(3, 20); + for (int j = 0; j < numDocs; j++) { + client().prepareIndex(index).setSource("value", randomIntBetween(0, 2)).get(); + expectedNumDocs ++; + } + } + refresh("index-*"); + String pit = openPointInTime(new String[] { "index-*" }, TimeValue.timeValueHours(1)); + try { + for (int size = 1; size <= numIndex; size++) { + SortOrder order = randomBoolean() ? SortOrder.ASC : SortOrder.DESC; + assertPagination(new PointInTimeBuilder(pit), expectedNumDocs, size, + SortBuilders.pitTiebreaker().order(order)); + assertPagination(new PointInTimeBuilder(pit), expectedNumDocs, size, + SortBuilders.fieldSort("value"), SortBuilders.pitTiebreaker().order(order)); + } + } finally { + closePointInTime(pit); + } + } + + private void assertPagination(PointInTimeBuilder pit, int expectedNumDocs, int size, SortBuilder... sort) throws Exception { + Set seen = new HashSet<>(); + SearchRequestBuilder builder = client().prepareSearch() + .setSize(size) + .setPointInTime(pit); + + final int[] reverseMuls = new int[sort.length]; + for (int i = 0; i < sort.length; i++) { + builder.addSort(sort[i]); + reverseMuls[i] = sort[i].order() == SortOrder.ASC ? 1 : -1; + } + final SearchRequest searchRequest = builder.request(); + SearchResponse response = client().search(searchRequest).get(); + Object[] lastSortValues = null; + while (response.getHits().getHits().length > 0) { + Object[] lastHitSortValues = null; + for (SearchHit hit : response.getHits().getHits()) { + assertTrue(seen.add(hit.getIndex() + hit.getId())); + + if (lastHitSortValues != null) { + for (int i = 0; i < sort.length; i++) { + Comparable value = (Comparable) hit.getRawSortValues()[i]; + int cmp = value.compareTo(lastHitSortValues[i]) * reverseMuls[i]; + if (cmp != 0) { + assertThat(cmp, equalTo(1)); + break; + } + } + } + lastHitSortValues = hit.getRawSortValues(); + } + int len = response.getHits().getHits().length; + SearchHit last = response.getHits().getHits()[len - 1]; + if (lastSortValues != null) { + for (int i = 0; i < sort.length; i++) { + Comparable value = (Comparable) last.getSortValues()[i]; + int cmp = value.compareTo(lastSortValues[i]) * reverseMuls[i]; + if (cmp != 0) { + assertThat(cmp, equalTo(1)); + break; + } + } + } + assertThat(last.getSortValues().length, equalTo(sort.length)); + lastSortValues = last.getSortValues(); + searchRequest.source().searchAfter(last.getSortValues()); + response = client().search(searchRequest).get(); + } + assertThat(seen.size(), equalTo(expectedNumDocs)); + } + private String openPointInTime(String[] indices, TimeValue keepAlive) { OpenPointInTimeRequest request = new OpenPointInTimeRequest( indices, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java index 55a56949b7dce..14d87dcb6a803 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/DocumentSubsetBitsetCacheTests.java @@ -563,7 +563,7 @@ private TestIndexContext testIndex(MapperService mapperService, Client client) t directoryReader = DirectoryReader.open(directory); final LeafReaderContext leaf = directoryReader.leaves().get(0); - final QueryShardContext shardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, + final QueryShardContext shardContext = new QueryShardContext(shardId.id(), 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), client, new IndexSearcher(directoryReader), () -> nowInMillis, null, null, () -> true, null, emptyMap()); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java index 74042155ffd93..774c0d9694448 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexReaderWrapperIntegrationTests.java @@ -90,7 +90,7 @@ public void testDLS() throws Exception { Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); final long nowInMillis = randomNonNegativeLong(); - QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); QueryShardContext queryShardContext = spy(realQueryShardContext); @@ -222,7 +222,7 @@ public void testDLSWithLimitedPermissions() throws Exception { Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); final long nowInMillis = randomNonNegativeLong(); - QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), indexSettings, BigArrays.NON_RECYCLING_INSTANCE, + QueryShardContext realQueryShardContext = new QueryShardContext(shardId.id(), 0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, mapperService, null, null, xContentRegistry(), writableRegistry(), client, null, () -> nowInMillis, null, null, () -> true, null, emptyMap()); QueryShardContext queryShardContext = spy(realQueryShardContext); diff --git a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java index 7cd1e85366989..25c73ad5b581a 100644 --- a/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java +++ b/x-pack/plugin/enrich/src/main/java/org/elasticsearch/xpack/enrich/action/EnrichShardMultiSearchAction.java @@ -241,6 +241,7 @@ protected MultiSearchResponse shardOperation(Request request, ShardId shardId) t Map runtimeFields = emptyMap(); final QueryShardContext context = indexService.newQueryShardContext( shardId.id(), + 0, searcher, () -> { throw new UnsupportedOperationException(); }, null, diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java index ab0a57d6d3e10..90e57da2f39cb 100644 --- a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/job/RollupIndexerIndexingTests.java @@ -88,7 +88,7 @@ public class RollupIndexerIndexingTests extends AggregatorTestCase { @Before private void setup() { settings = createIndexSettings(); - queryShardContext = new QueryShardContext(0, settings, + queryShardContext = new QueryShardContext(0, 0, settings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, null, null, null, null, () -> 0L, null, null, () -> true, null, emptyMap()); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index d496756e19ee4..e4518dadf4e65 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -730,6 +730,7 @@ public void onIndexModule(IndexModule module) { module.setReaderWrapper(indexService -> new SecurityIndexReaderWrapper( shardId -> indexService.newQueryShardContext(shardId.id(), + 0, // we pass a null index reader, which is legal and will disable rewrite optimizations // based on index statistics, which is probably safer... null, diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index eb5b9b06bfb13..848fcc16eb8b2 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -897,7 +897,7 @@ protected final QueryShardContext createMockShardContext() { IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup); return builder.build(new IndexFieldDataCache.None(), null); }; - return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, + return new QueryShardContext(0, -1, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, null, null, null, xContentRegistry(), null, null, null, () -> randomNonNegativeLong(), null, null, () -> true, null, emptyMap()) {