From 705f28ca2a40a32413de605830f5b7d5ec7b1f1b Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Fri, 28 Dec 2018 16:07:09 +0100 Subject: [PATCH] Add ability to suggest shard_size on coord node rewrite With #36997 and #37000 we added the ability to provide a cluster alias with a SearchRequest and to perform non-final reduction when a cluster alias is set to the incoming search request. With CCS soon supporting local reduction on each remote cluster, we also need to prevent shard_size of terms aggs to be adjusted on the data nodes. We may end up querying a single shard per cluster, but later we will have to reduce results coming from multiple cluster hence we still have to collect more buckets than needed to guarantee some level of precision. This is done by adding the ability to override the shard_size on the coordinating node as part of the rewrite phase. This will be done only when searching against multiple clusters, meaning when using CCS with alternate execution mode. Setting the shard_size explicitly on the coord node means that it will not be overridden later on the data nodes. The shard_size will still be set on the data nodes like before in all other cases (local search, or CCS with ordinary execution mode). Relates to #32125 --- .../query/TransportValidateQueryAction.java | 5 ++- .../action/search/TransportSearchAction.java | 5 ++- .../index/query/QueryRewriteContext.java | 16 +++++-- .../index/query/QueryShardContext.java | 2 +- .../elasticsearch/indices/IndicesService.java | 4 +- .../elasticsearch/search/SearchService.java | 6 +-- .../aggregations/AggregationBuilder.java | 2 +- .../filter/FilterAggregationBuilder.java | 4 +- .../filter/FiltersAggregationBuilder.java | 4 +- .../geogrid/GeoGridAggregationBuilder.java | 36 ++++++++++----- .../SignificantTermsAggregationBuilder.java | 45 +++++++++++++++---- .../SignificantTermsAggregatorFactory.java | 31 +++---------- .../SignificantTextAggregationBuilder.java | 34 +++++++++----- .../SignificantTextAggregatorFactory.java | 19 ++------ .../bucket/terms/TermsAggregationBuilder.java | 34 +++++++++++++- .../bucket/terms/TermsAggregator.java | 4 +- .../bucket/terms/TermsAggregatorFactory.java | 15 ++----- .../index/mapper/DateFieldTypeTests.java | 6 +-- .../index/query/RewriteableTests.java | 6 +-- .../AggregatorFactoriesTests.java | 4 +- .../aggregations/bucket/FiltersTests.java | 15 +++---- .../builder/SearchSourceBuilderTests.java | 2 +- .../SecurityIndexSearcherWrapper.java | 4 +- .../RewriteCachingDirectoryReaderTests.java | 2 +- ...SecurityIndexSearcherWrapperUnitTests.java | 5 +-- 25 files changed, 183 insertions(+), 127 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java index 7016d1b42894f..eb38ea797bd9a 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/validate/query/TransportValidateQueryAction.java @@ -39,6 +39,7 @@ import org.elasticsearch.common.lease.Releasables; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.ParsedQuery; +import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.QueryShardException; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.indices.IndexClosedException; @@ -108,8 +109,8 @@ protected void doExecute(Task task, ValidateQueryRequest request, ActionListener if (request.query() == null) { rewriteListener.onResponse(request.query()); } else { - Rewriteable.rewriteAndFetch(request.query(), searchService.getRewriteContext(timeProvider), - rewriteListener); + QueryRewriteContext rewriteContext = searchService.getRewriteContext(timeProvider, false); + Rewriteable.rewriteAndFetch(request.query(), rewriteContext, rewriteListener); } } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 9df930544e624..bb381bb4e4648 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -40,6 +40,7 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.index.Index; +import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.search.SearchService; @@ -214,8 +215,8 @@ protected void doExecute(Task task, SearchRequest searchRequest, ActionListener< if (searchRequest.source() == null) { rewriteListener.onResponse(searchRequest.source()); } else { - Rewriteable.rewriteAndFetch(searchRequest.source(), searchService.getRewriteContext(timeProvider::getAbsoluteStartMillis), - rewriteListener); + QueryRewriteContext rewriteContext = searchService.getRewriteContext(timeProvider::getAbsoluteStartMillis, false); + Rewriteable.rewriteAndFetch(searchRequest.source(), rewriteContext, rewriteListener); } } diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index b275088d89441..982510cfa28c2 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -36,18 +36,18 @@ public class QueryRewriteContext { private final NamedXContentRegistry xContentRegistry; private final NamedWriteableRegistry writeableRegistry; + private final boolean multipleClusters; protected final Client client; protected final LongSupplier nowInMillis; private final List>> asyncActions = new ArrayList<>(); - public QueryRewriteContext( - NamedXContentRegistry xContentRegistry, NamedWriteableRegistry writeableRegistry,Client client, - LongSupplier nowInMillis) { - + public QueryRewriteContext(NamedXContentRegistry xContentRegistry, NamedWriteableRegistry writeableRegistry, + Client client, LongSupplier nowInMillis, boolean multipleClusters) { this.xContentRegistry = xContentRegistry; this.writeableRegistry = writeableRegistry; this.client = client; this.nowInMillis = nowInMillis; + this.multipleClusters = multipleClusters; } /** @@ -68,6 +68,14 @@ public NamedWriteableRegistry getWriteableRegistry() { return writeableRegistry; } + /** + * Returns whether the request being rewritten is part of a cross-cluster search request where each cluster performs + * its own local reduction, and the targeted clusters are more than one. + */ + public boolean isMultipleClusters() { + return multipleClusters; + } + /** * Returns an instance of {@link QueryShardContext} if available of null otherwise */ diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index f7f1d29f53098..705fe9c03371c 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -110,7 +110,7 @@ public QueryShardContext(int shardId, IndexSettings indexSettings, BitsetFilterC SimilarityService similarityService, ScriptService scriptService, NamedXContentRegistry xContentRegistry, NamedWriteableRegistry namedWriteableRegistry, Client client, IndexReader reader, LongSupplier nowInMillis, String clusterAlias) { - super(xContentRegistry, namedWriteableRegistry,client, nowInMillis); + super(xContentRegistry, namedWriteableRegistry,client, nowInMillis, false); this.shardId = shardId; this.similarityService = similarityService; this.mapperService = mapperService; diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 19388a2b63d4d..fb545089febd5 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -1357,8 +1357,8 @@ public AliasFilter buildAliasFilter(ClusterState state, String index, String... /** * Returns a new {@link QueryRewriteContext} with the given {@code now} provider */ - public QueryRewriteContext getRewriteContext(LongSupplier nowInMillis) { - return new QueryRewriteContext(xContentRegistry, namedWriteableRegistry, client, nowInMillis); + public QueryRewriteContext getRewriteContext(LongSupplier nowInMillis, boolean multipleClusters) { + return new QueryRewriteContext(xContentRegistry, namedWriteableRegistry, client, nowInMillis, multipleClusters); } /** diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 500e70a65b486..a3ffa3092aa76 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -1097,14 +1097,14 @@ protected void doRun() { // we also do rewrite on the coordinating node (TransportSearchService) but we also need to do it here for BWC as well as // AliasFilters that might need to be rewritten. These are edge-cases but we are every efficient doing the rewrite here so it's not // adding a lot of overhead - Rewriteable.rewriteAndFetch(request.getRewriteable(), indicesService.getRewriteContext(request::nowInMillis), actionListener); + Rewriteable.rewriteAndFetch(request.getRewriteable(), getRewriteContext(request::nowInMillis, false), actionListener); } /** * Returns a new {@link QueryRewriteContext} with the given {@code now} provider */ - public QueryRewriteContext getRewriteContext(LongSupplier nowInMillis) { - return indicesService.getRewriteContext(nowInMillis); + public QueryRewriteContext getRewriteContext(LongSupplier nowInMillis, boolean multipleClusters) { + return indicesService.getRewriteContext(nowInMillis, multipleClusters); } public IndicesService getIndicesService() { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java index 359c8dd571e25..74ab6a3bf44db 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/AggregationBuilder.java @@ -125,7 +125,7 @@ public final AggregationBuilder rewrite(QueryRewriteContext context) throws IOEx * identity reference must be returned otherwise the builder will be * rewritten infinitely. */ - protected AggregationBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException { + protected AggregationBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { return this; } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java index 71c1c8fa3deb2..1d4f553305277 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FilterAggregationBuilder.java @@ -84,8 +84,8 @@ protected void doWriteTo(StreamOutput out) throws IOException { } @Override - protected AggregationBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException { - QueryBuilder result = Rewriteable.rewrite(filter, queryShardContext); + protected AggregationBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { + QueryBuilder result = Rewriteable.rewrite(filter, queryRewriteContext); if (result != filter) { return new FilterAggregationBuilder(getName(), result); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java index 810126e851251..c7d88ecc042c6 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/filter/FiltersAggregationBuilder.java @@ -198,11 +198,11 @@ public String otherBucketKey() { } @Override - protected AggregationBuilder doRewrite(QueryRewriteContext queryShardContext) throws IOException { + protected AggregationBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { List rewrittenFilters = new ArrayList<>(filters.size()); boolean changed = false; for (KeyedFilter kf : filters) { - QueryBuilder result = Rewriteable.rewrite(kf.filter(), queryShardContext); + QueryBuilder result = Rewriteable.rewrite(kf.filter(), queryRewriteContext); rewrittenFilters.add(new KeyedFilter(kf.key(), result)); if (result != kf.filter()) { changed = true; diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java index 353f391f213d6..8ee2c498e8783 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/geogrid/GeoGridAggregationBuilder.java @@ -34,6 +34,7 @@ import org.elasticsearch.index.fielddata.MultiGeoPointValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; +import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; @@ -90,7 +91,7 @@ protected GeoGridAggregationBuilder(GeoGridAggregationBuilder clone, Builder fac } @Override - protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { + protected GeoGridAggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new GeoGridAggregationBuilder(this, factoriesBuilder, metaData); } @@ -150,26 +151,41 @@ public int shardSize() { protected ValuesSourceAggregatorFactory innerBuild(SearchContext context, ValuesSourceConfig config, AggregatorFactory parent, Builder subFactoriesBuilder) throws IOException { - int shardSize = this.shardSize; - - int requiredSize = this.requiredSize; + int shardSize = suggestShardSize(name, this.shardSize, requiredSize, context.numberOfShards() == 1); + return new GeoHashGridAggregatorFactory(name, config, precision, requiredSize, shardSize, context, parent, + subFactoriesBuilder, metaData); + } + static int suggestShardSize(String name, int shardSize, int requiredSize, boolean singleShard) { if (shardSize < 0) { - // Use default heuristic to avoid any wrong-ranking caused by - // distributed counting - shardSize = BucketUtils.suggestShardSideQueueSize(requiredSize, context.numberOfShards() == 1); + // Use default heuristic to avoid any wrong-ranking caused by distributed counting + shardSize = BucketUtils.suggestShardSideQueueSize(requiredSize, singleShard); } if (requiredSize <= 0 || shardSize <= 0) { throw new ElasticsearchException( - "parameters [required_size] and [shard_size] must be >0 in geohash_grid aggregation [" + name + "]."); + "parameters [required_size] and [shard_size] must be >0 in geohash_grid aggregation [" + name + "]."); } if (shardSize < requiredSize) { shardSize = requiredSize; } - return new GeoHashGridAggregatorFactory(name, config, precision, requiredSize, shardSize, context, parent, - subFactoriesBuilder, metaData); + return shardSize; + } + + @Override + protected AggregationBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { + if (queryRewriteContext.isMultipleClusters()) { + assert queryRewriteContext.convertToShardContext() == null; + // We are coordinating a cross-cluster search request across more than one cluster with local reduction on each remote cluster. + // We need to set shard_size explicitly to make sure that we don't optimize later for a single shard; although we may end up + // searching on a single shard per cluster, we will reduce buckets coming from multiple shards as we have multiple clusters. + int updatedShardSize = suggestShardSize(name, shardSize, requiredSize, false); + GeoGridAggregationBuilder aggregationBuilder = shallowCopy(factoriesBuilder, metaData); + aggregationBuilder.shardSize(updatedShardSize); + return aggregationBuilder; + } + return super.doRewrite(queryRewriteContext); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java index 02216c5a5cb35..7c95e25b0ca0b 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregationBuilder.java @@ -24,12 +24,13 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ParseFieldRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.bucket.BucketUtils; import org.elasticsearch.search.aggregations.bucket.MultiBucketAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.JLHScore; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic; @@ -100,12 +101,8 @@ public static Aggregator.Parser getParser(ParseFieldRegistry aggregationParser.parse(parser, + new SignificantTermsAggregationBuilder(aggregationName, null), null); } private IncludeExclude includeExclude = null; @@ -141,7 +138,7 @@ protected SignificantTermsAggregationBuilder(SignificantTermsAggregationBuilder } @Override - protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { + protected SignificantTermsAggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new SignificantTermsAggregationBuilder(this, factoriesBuilder, metaData); } @@ -175,6 +172,38 @@ public SignificantTermsAggregationBuilder bucketCountThresholds(TermsAggregator. return this; } + @Override + protected AggregationBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { + if (queryRewriteContext.isMultipleClusters()) { + assert queryRewriteContext.convertToShardContext() == null; + // We are coordinating a cross-cluster search request across more than one cluster with local reduction on each remote cluster. + // We need to set shard_size explicitly to make sure that we don't optimize later for a single shard; although we may end up + // searching on a single shard per cluster, we will reduce buckets coming from multiple shards as we have multiple clusters. + BucketCountThresholds bucketCountThresholds = suggestShardSize( + DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize(), this.bucketCountThresholds, false); + if (bucketCountThresholds != this.bucketCountThresholds) { + SignificantTermsAggregationBuilder aggregationBuilder = shallowCopy(factoriesBuilder, metaData); + aggregationBuilder.bucketCountThresholds = bucketCountThresholds; + return aggregationBuilder; + } + } + return super.doRewrite(queryRewriteContext); + } + + static BucketCountThresholds suggestShardSize(int defaultShardSize, BucketCountThresholds bucketCountThresholds, boolean singleShard) { + // The user has not made a shardSize selection. Use default heuristic to avoid any wrong-ranking caused by distributed counting + // but request double the usual amount. We typically need more than the number of "top" terms requested by other aggregations + // as the significance algorithm is in less of a position to down-select at shard-level - some of the things we want to find have + // only one occurrence on each shard and as such are impossible to differentiate from non-significant terms at that early stage. + if (bucketCountThresholds.getShardSize() == defaultShardSize) { + int newShardSize = BucketUtils.suggestShardSideQueueSize(2 * bucketCountThresholds.getRequiredSize(), singleShard); + BucketCountThresholds updatedBucketCountThresholds = new BucketCountThresholds(bucketCountThresholds); + updatedBucketCountThresholds.setShardSize(newShardSize); + return updatedBucketCountThresholds; + } + return bucketCountThresholds; + } + /** * Sets the size - indicating how many term buckets should be returned * (defaults to 10) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java index 7fe41407af4ca..72f71f3a6f517 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTermsAggregatorFactory.java @@ -43,7 +43,6 @@ import org.elasticsearch.search.aggregations.AggregatorFactory; import org.elasticsearch.search.aggregations.InternalAggregation; import org.elasticsearch.search.aggregations.NonCollectingAggregator; -import org.elasticsearch.search.aggregations.bucket.BucketUtils; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic; import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator; @@ -181,36 +180,18 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, Aggregator pare } numberOfAggregatorsCreated++; - BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(this.bucketCountThresholds); - if (bucketCountThresholds.getShardSize() == SignificantTermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) { - // The user has not made a shardSize selection . - // Use default heuristic to avoid any wrong-ranking caused by - // distributed counting - // but request double the usual amount. - // We typically need more than the number of "top" terms requested - // by other aggregations - // as the significance algorithm is in less of a position to - // down-select at shard-level - - // some of the things we want to find have only one occurrence on - // each shard and as - // such are impossible to differentiate from non-significant terms - // at that early stage. - bucketCountThresholds.setShardSize(2 * BucketUtils.suggestShardSideQueueSize(bucketCountThresholds.getRequiredSize(), - context.numberOfShards() == 1)); - } - + BucketCountThresholds bucketCountThresholds = SignificantTermsAggregationBuilder.suggestShardSize( + SignificantTermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize(), + this.bucketCountThresholds, context.numberOfShards() == 1); if (valuesSource instanceof ValuesSource.Bytes) { - ExecutionMode execution = null; + final ExecutionMode execution; if (executionHint != null) { execution = ExecutionMode.fromString(executionHint, deprecationLogger); - } - if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals == false) { + } else if (valuesSource instanceof ValuesSource.Bytes.WithOrdinals == false) { execution = ExecutionMode.MAP; - } - if (execution == null) { + } else { execution = ExecutionMode.GLOBAL_ORDINALS; } - assert execution != null; DocValueFormat format = config.format(); if ((includeExclude != null) && (includeExclude.isRegexBased()) && format != DocValueFormat.RAW) { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java index f0b85f979c233..0a7fb4b13c932 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregationBuilder.java @@ -25,9 +25,9 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ParseFieldRegistry; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationInitializationException; @@ -114,14 +114,8 @@ public static Aggregator.Parser getParser( return significanceHeuristicParser.parse(p); }, new ParseField(name)); } - return new Aggregator.Parser() { - @Override - public AggregationBuilder parse(String aggregationName, XContentParser parser) - throws IOException { - return PARSER.parse(parser, - new SignificantTextAggregationBuilder(aggregationName, null), null); - } - }; + return (aggregationName, parser) -> PARSER.parse(parser, + new SignificantTextAggregationBuilder(aggregationName, null), null); } protected SignificantTextAggregationBuilder(SignificantTextAggregationBuilder clone, @@ -137,7 +131,7 @@ protected SignificantTextAggregationBuilder(SignificantTextAggregationBuilder cl } @Override - protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { + protected SignificantTextAggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new SignificantTextAggregationBuilder(this, factoriesBuilder, metaData); } @@ -209,7 +203,6 @@ public SignificantTextAggregationBuilder fieldName(String fieldName) { return this; } - /** * Selects the fields to load from _source JSON and analyze. * If none are specified, the indexed "fieldName" value is assumed @@ -220,7 +213,6 @@ public SignificantTextAggregationBuilder sourceFieldNames(List names) { return this; } - /** * Control if duplicate paragraphs of text should try be filtered from the * statistical text analysis. Can improve results but slows down analysis. @@ -349,6 +341,24 @@ protected AggregatorFactory doBuild(SearchContext context, AggregatorFactory< fieldName, sourceFieldNames, filterDuplicateText, metaData); } + @Override + protected AggregationBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { + if (queryRewriteContext.isMultipleClusters()) { + assert queryRewriteContext.convertToShardContext() == null; + // We are coordinating a cross-cluster search request across more than one cluster with local reduction on each remote cluster. + // We need to set shard_size explicitly to make sure that we don't optimize later for a single shard; although we may end up + // searching on a single shard per cluster, we will reduce buckets coming from multiple shards as we have multiple clusters. + BucketCountThresholds bucketCountThresholds = SignificantTermsAggregationBuilder.suggestShardSize( + DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize(), this.bucketCountThresholds, false); + if (bucketCountThresholds != this.bucketCountThresholds) { + SignificantTextAggregationBuilder aggregationBuilder = shallowCopy(factoriesBuilder, metaData); + aggregationBuilder.bucketCountThresholds = bucketCountThresholds; + return aggregationBuilder; + } + } + return super.doRewrite(queryRewriteContext); + } + @Override protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java index a51a33defdd00..f17c3af8e0467 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/significant/SignificantTextAggregatorFactory.java @@ -37,7 +37,6 @@ import org.elasticsearch.search.aggregations.Aggregator; import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.AggregatorFactory; -import org.elasticsearch.search.aggregations.bucket.BucketUtils; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic; import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator; @@ -143,7 +142,6 @@ public long getBackgroundFrequency(BytesRef termBytes) throws IOException { return getBackgroundFrequency(value); } - @Override public void close() { try { @@ -164,20 +162,9 @@ protected Aggregator createInternal(Aggregator parent, boolean collectsFromSingl } numberOfAggregatorsCreated++; - BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(this.bucketCountThresholds); - if (bucketCountThresholds.getShardSize() == SignificantTextAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) { - // The user has not made a shardSize selection. - // Use default heuristic to avoid any wrong-ranking caused by - // distributed counting but request double the usual amount. - // We typically need more than the number of "top" terms requested - // by other aggregations as the significance algorithm is in less - // of a position to down-select at shard-level - some of the things - // we want to find have only one occurrence on each shard and as - // such are impossible to differentiate from non-significant terms - // at that early stage. - bucketCountThresholds.setShardSize(2 * BucketUtils.suggestShardSideQueueSize(bucketCountThresholds.getRequiredSize(), - context.numberOfShards() == 1)); - } + BucketCountThresholds bucketCountThresholds = SignificantTermsAggregationBuilder.suggestShardSize( + SignificantTextAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize(), + this.bucketCountThresholds, context.numberOfShards() == 1); // TODO - need to check with mapping that this is indeed a text field.... diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java index 5887f6b525891..09c600ef66955 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregationBuilder.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.Aggregator.SubAggCollectionMode; import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; @@ -32,6 +33,7 @@ import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.InternalOrder; import org.elasticsearch.search.aggregations.InternalOrder.CompoundOrder; +import org.elasticsearch.search.aggregations.bucket.BucketUtils; import org.elasticsearch.search.aggregations.bucket.MultiBucketAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds; import org.elasticsearch.search.aggregations.support.ValueType; @@ -122,7 +124,7 @@ protected TermsAggregationBuilder(TermsAggregationBuilder clone, Builder factori } @Override - protected AggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { + protected TermsAggregationBuilder shallowCopy(Builder factoriesBuilder, Map metaData) { return new TermsAggregationBuilder(this, factoriesBuilder, metaData); } @@ -154,6 +156,36 @@ protected void innerWriteTo(StreamOutput out) throws IOException { out.writeBoolean(showTermDocCountError); } + @Override + protected AggregationBuilder doRewrite(QueryRewriteContext queryRewriteContext) throws IOException { + if (queryRewriteContext.isMultipleClusters()) { + assert queryRewriteContext.convertToShardContext() == null; + // We are coordinating a cross-cluster search request across more than one cluster with local reduction on each remote cluster. + // We need to set shard_size explicitly to make sure that we don't optimize later for a single shard; although we may end up + // searching on a single shard per cluster, we will reduce buckets coming from multiple shards as we have multiple clusters. + BucketCountThresholds bucketCountThresholds = suggestShardSize(order, this.bucketCountThresholds, false); + if (bucketCountThresholds != this.bucketCountThresholds) { + TermsAggregationBuilder termsAggregationBuilder = shallowCopy(factoriesBuilder, metaData); + termsAggregationBuilder.bucketCountThresholds = bucketCountThresholds; + return termsAggregationBuilder; + } + } + return super.doRewrite(queryRewriteContext); + } + + static BucketCountThresholds suggestShardSize(BucketOrder order, BucketCountThresholds bucketCountThresholds, boolean singleShard) { + // The user has not made a shardSize selection. Use default heuristic to avoid any wrong-ranking caused by distributed counting + if (InternalOrder.isKeyOrder(order) == false + && bucketCountThresholds.getShardSize() == DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) { + int newShardSize = BucketUtils.suggestShardSideQueueSize(bucketCountThresholds.getRequiredSize(), singleShard); + BucketCountThresholds updatedBucketCountThresholds = new BucketCountThresholds(bucketCountThresholds); + updatedBucketCountThresholds.setShardSize(newShardSize); + updatedBucketCountThresholds.ensureValidity(); + return updatedBucketCountThresholds; + } + return bucketCountThresholds; + } + /** * Sets the size - indicating how many term buckets should be returned * (defaults to 10) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java index 189c2ee796eb4..5bf7ca82c2c86 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregator.java @@ -54,7 +54,7 @@ public abstract class TermsAggregator extends DeferableBucketAggregator { - public static class BucketCountThresholds implements Writeable, ToXContentFragment { + public static final class BucketCountThresholds implements Writeable, ToXContentFragment { private long minDocCount; private long shardMinDocCount; private int requiredSize; @@ -90,7 +90,7 @@ public BucketCountThresholds(BucketCountThresholds bucketCountThresholds) { bucketCountThresholds.shardSize); } - public void ensureValidity() { + void ensureValidity() { // shard_size cannot be smaller than size as we need to at least fetch entries from every shards in order to return // diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java index 25f552075dead..4808a7be20d5a 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorFactory.java @@ -34,7 +34,6 @@ import org.elasticsearch.search.aggregations.InternalOrder; import org.elasticsearch.search.aggregations.InternalOrder.CompoundOrder; import org.elasticsearch.search.aggregations.NonCollectingAggregator; -import org.elasticsearch.search.aggregations.bucket.BucketUtils; import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregator.BucketCountThresholds; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.support.ValuesSource; @@ -115,16 +114,10 @@ protected Aggregator doCreateInternal(ValuesSource valuesSource, Aggregator pare if (collectsFromSingleBucket == false) { return asMultiBucketAggregator(this, context, parent); } - BucketCountThresholds bucketCountThresholds = new BucketCountThresholds(this.bucketCountThresholds); - if (InternalOrder.isKeyOrder(order) == false - && bucketCountThresholds.getShardSize() == TermsAggregationBuilder.DEFAULT_BUCKET_COUNT_THRESHOLDS.getShardSize()) { - // The user has not made a shardSize selection. Use default - // heuristic to avoid any wrong-ranking caused by distributed - // counting - bucketCountThresholds.setShardSize(BucketUtils.suggestShardSideQueueSize(bucketCountThresholds.getRequiredSize(), - context.numberOfShards() == 1)); - } - bucketCountThresholds.ensureValidity(); + + BucketCountThresholds bucketCountThresholds = TermsAggregationBuilder.suggestShardSize(order, + this.bucketCountThresholds, context.numberOfShards() == 1); + if (valuesSource instanceof ValuesSource.Bytes) { ExecutionMode execution = null; if (executionHint != null) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 072170aff09dd..a70e91afae37a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -74,7 +74,7 @@ public void modify(MappedFieldType ft) { } public void testIsFieldWithinQueryEmptyReader() throws IOException { - QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> nowInMillis); + QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> nowInMillis, false); IndexReader reader = new MultiReader(); DateFieldType ft = new DateFieldType(); ft.setName("my_date"); @@ -84,7 +84,7 @@ public void testIsFieldWithinQueryEmptyReader() throws IOException { private void doTestIsFieldWithinQuery(DateFieldType ft, DirectoryReader reader, DateTimeZone zone, DateMathParser alternateFormat) throws IOException { - QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> nowInMillis); + QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> nowInMillis, false); assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02", randomBoolean(), randomBoolean(), null, null, context)); assertEquals(Relation.INTERSECTS, ft.isFieldWithinQuery(reader, "2016-01-02", "2016-06-20", @@ -131,7 +131,7 @@ public void testIsFieldWithinQuery() throws IOException { DateFieldType ft2 = new DateFieldType(); ft2.setName("my_date2"); - QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> nowInMillis); + QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> nowInMillis, false); assertEquals(Relation.DISJOINT, ft2.isFieldWithinQuery(reader, "2015-10-09", "2016-01-02", false, false, null, null, context)); IOUtils.close(reader, w, dir); } diff --git a/server/src/test/java/org/elasticsearch/index/query/RewriteableTests.java b/server/src/test/java/org/elasticsearch/index/query/RewriteableTests.java index 254f9b3fcad5f..5149f4e7056cc 100644 --- a/server/src/test/java/org/elasticsearch/index/query/RewriteableTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/RewriteableTests.java @@ -32,7 +32,7 @@ public class RewriteableTests extends ESTestCase { public void testRewrite() throws IOException { - QueryRewriteContext context = new QueryRewriteContext(null, null, null, null); + QueryRewriteContext context = new QueryRewriteContext(null, null, null, null, false); TestRewriteable rewrite = Rewriteable.rewrite(new TestRewriteable(randomIntBetween(0, Rewriteable.MAX_REWRITE_ROUNDS)), context, randomBoolean()); assertEquals(rewrite.numRewrites, 0); @@ -47,7 +47,7 @@ public void testRewrite() throws IOException { } public void testRewriteAndFetch() throws ExecutionException, InterruptedException { - QueryRewriteContext context = new QueryRewriteContext(null, null, null, null); + QueryRewriteContext context = new QueryRewriteContext(null, null, null, null, false); PlainActionFuture future = new PlainActionFuture<>(); Rewriteable.rewriteAndFetch(new TestRewriteable(randomIntBetween(0, Rewriteable.MAX_REWRITE_ROUNDS), true), context, future); TestRewriteable rewrite = future.get(); @@ -65,7 +65,7 @@ public void testRewriteAndFetch() throws ExecutionException, InterruptedExceptio } public void testRewriteList() throws IOException { - QueryRewriteContext context = new QueryRewriteContext(null, null, null, null); + QueryRewriteContext context = new QueryRewriteContext(null, null, null, null, false); List rewriteableList = new ArrayList<>(); int numInstances = randomIntBetween(1, 10); rewriteableList.add(new TestRewriteable(randomIntBetween(1, Rewriteable.MAX_REWRITE_ROUNDS))); diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java index 7a4e0fb705918..3b8ba16f6b59c 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/AggregatorFactoriesTests.java @@ -210,7 +210,7 @@ public void testRewrite() throws Exception { AggregatorFactories.Builder builder = new AggregatorFactories.Builder().addAggregator(filterAggBuilder) .addPipelineAggregator(pipelineAgg); AggregatorFactories.Builder rewritten = builder - .rewrite(new QueryRewriteContext(xContentRegistry, null, null, () -> 0L)); + .rewrite(new QueryRewriteContext(xContentRegistry, null, null, () -> 0L, false)); assertNotSame(builder, rewritten); Collection aggregatorFactories = rewritten.getAggregatorFactories(); assertEquals(1, aggregatorFactories.size()); @@ -224,7 +224,7 @@ public void testRewrite() throws Exception { // Check that a further rewrite returns the same aggregation factories builder AggregatorFactories.Builder secondRewritten = rewritten - .rewrite(new QueryRewriteContext(xContentRegistry, null, null, () -> 0L)); + .rewrite(new QueryRewriteContext(xContentRegistry, null, null, () -> 0L, false)); assertSame(rewritten, secondRewritten); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersTests.java index 4c7fdccb64b00..060ec60d6ecfd 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/FiltersTests.java @@ -131,12 +131,12 @@ public void testRewrite() throws IOException { // test non-keyed filter that doesn't rewrite AggregationBuilder original = new FiltersAggregationBuilder("my-agg", new MatchAllQueryBuilder()); original.setMetaData(Collections.singletonMap(randomAlphaOfLengthBetween(1, 20), randomAlphaOfLengthBetween(1, 20))); - AggregationBuilder rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); + AggregationBuilder rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L, false)); assertSame(original, rewritten); // test non-keyed filter that does rewrite original = new FiltersAggregationBuilder("my-agg", new BoolQueryBuilder()); - rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); + rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L, false)); assertNotSame(original, rewritten); assertThat(rewritten, instanceOf(FiltersAggregationBuilder.class)); assertEquals("my-agg", ((FiltersAggregationBuilder) rewritten).getName()); @@ -147,12 +147,12 @@ public void testRewrite() throws IOException { // test keyed filter that doesn't rewrite original = new FiltersAggregationBuilder("my-agg", new KeyedFilter("my-filter", new MatchAllQueryBuilder())); - rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); + rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L, false)); assertSame(original, rewritten); // test non-keyed filter that does rewrite original = new FiltersAggregationBuilder("my-agg", new KeyedFilter("my-filter", new BoolQueryBuilder())); - rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); + rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L, false)); assertNotSame(original, rewritten); assertThat(rewritten, instanceOf(FiltersAggregationBuilder.class)); assertEquals("my-agg", ((FiltersAggregationBuilder) rewritten).getName()); @@ -166,7 +166,7 @@ public void testRewrite() throws IOException { .subAggregation( new FiltersAggregationBuilder("my-agg", new KeyedFilter("my-filter", new BoolQueryBuilder())) ); - rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L)); + rewritten = original.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L, false)); assertNotSame(original, rewritten); assertNotEquals(original, rewritten); assertThat(rewritten, instanceOf(TermsAggregationBuilder.class)); @@ -176,7 +176,7 @@ public void testRewrite() throws IOException { assertNotSame(original.getSubAggregations().iterator().next(), subAgg); assertEquals("my-agg", subAgg.getName()); assertSame(rewritten, - rewritten.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L))); + rewritten.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L, false))); } public void testRewritePreservesOtherBucket() throws IOException { @@ -184,8 +184,7 @@ public void testRewritePreservesOtherBucket() throws IOException { originalFilters.otherBucket(randomBoolean()); originalFilters.otherBucketKey(randomAlphaOfLength(10)); - AggregationBuilder rewritten = originalFilters.rewrite(new QueryRewriteContext(xContentRegistry(), - null, null, () -> 0L)); + AggregationBuilder rewritten = originalFilters.rewrite(new QueryRewriteContext(xContentRegistry(), null, null, () -> 0L, false)); assertThat(rewritten, instanceOf(FiltersAggregationBuilder.class)); FiltersAggregationBuilder rewrittenFilters = (FiltersAggregationBuilder) rewritten; diff --git a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java index 12c3e487ff124..4d8522bd5db8f 100644 --- a/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/builder/SearchSourceBuilderTests.java @@ -463,6 +463,6 @@ private void assertIndicesBoostParseErrorMessage(String restContent, String expe private SearchSourceBuilder rewrite(SearchSourceBuilder searchSourceBuilder) throws IOException { return Rewriteable.rewrite(searchSourceBuilder, new QueryRewriteContext(xContentRegistry(), writableRegistry(), - null, Long.valueOf(1)::longValue)); + null, Long.valueOf(1)::longValue, false)); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java index a8651701448d2..3f7917f8eb82c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapper.java @@ -5,8 +5,8 @@ */ package org.elasticsearch.xpack.core.security.authz.accesscontrol; -import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.BooleanQuery; @@ -365,7 +365,7 @@ static void verifyRoleQuery(QueryBuilder queryBuilder) throws IOException { static void failIfQueryUsesClient(QueryBuilder queryBuilder, QueryRewriteContext original) throws IOException { QueryRewriteContext copy = new QueryRewriteContext( - original.getXContentRegistry(), original.getWriteableRegistry(), null, original::nowInMillis); + original.getXContentRegistry(), original.getWriteableRegistry(), null, original::nowInMillis, false); Rewriteable.rewrite(queryBuilder, copy); if (copy.hasAsyncActions()) { throw new IllegalStateException("role queries are not allowed to execute additional requests"); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/RewriteCachingDirectoryReaderTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/RewriteCachingDirectoryReaderTests.java index 6812aca474749..bb6f8056188b4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/RewriteCachingDirectoryReaderTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/index/engine/RewriteCachingDirectoryReaderTests.java @@ -90,7 +90,7 @@ public void testIsWithinQuery() throws IOException { DateFieldMapper.Builder b = new DateFieldMapper.Builder("test"); DateFieldMapper.DateFieldType dateFieldType = b.fieldType(); dateFieldType.setName("test"); - QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> 0); + QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), null, () -> 0, false); MappedFieldType.Relation relation = dateFieldType.isFieldWithinQuery(cachingDirectoryReader, 0, 10, true, true, DateTimeZone.UTC, null, context); assertEquals(relation, MappedFieldType.Relation.WITHIN); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java index 06838ac6ffae1..76a0ac0dbb590 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SecurityIndexSearcherWrapperUnitTests.java @@ -34,7 +34,6 @@ import org.apache.lucene.util.Accountable; import org.apache.lucene.util.BitSet; import org.apache.lucene.util.FixedBitSet; -import org.elasticsearch.core.internal.io.IOUtils; import org.apache.lucene.util.SparseFixedBitSet; import org.elasticsearch.client.Client; import org.elasticsearch.common.Strings; @@ -43,6 +42,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; @@ -664,8 +664,7 @@ public void testFailIfQueryUsesClient() throws Exception { Client client = mock(Client.class); when(client.settings()).thenReturn(Settings.EMPTY); final long nowInMillis = randomNonNegativeLong(); - QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), client, - () -> nowInMillis); + QueryRewriteContext context = new QueryRewriteContext(xContentRegistry(), writableRegistry(), client, () -> nowInMillis, false); QueryBuilder queryBuilder1 = new TermsQueryBuilder("field", "val1", "val2"); SecurityIndexSearcherWrapper.failIfQueryUsesClient(queryBuilder1, context);