From bfbf1342c327aab4e6a1ef50cc6c2712323b8a86 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Mon, 26 Nov 2018 18:45:37 +0100 Subject: [PATCH] Add rest_total_hits_as_int in the search APIs (#35848) This change adds the support for a search query parameter named `rest_total_hits_as_int`. This parameter will be used in the next major version (7.0) to restore the total hits as a number in the response. This param is added in this version (6.x) to handle mixed cluster queries where nodes are in multiple versions (7.0 and 6.latest). The param does not change anything in 6x since total hits is always a number in 6x so it is just ignored. --- .../RestMultiSearchTemplateAction.java | 11 +++- .../mustache/RestSearchTemplateAction.java | 11 +++- .../test/lang_mustache/30_search_template.yml | 27 ++++++++ .../50_multi_search_template.yml | 44 +++++++++++++ .../resources/rest-api-spec/api/msearch.json | 5 ++ .../rest-api-spec/api/msearch_template.json | 5 ++ .../resources/rest-api-spec/api/scroll.json | 5 ++ .../resources/rest-api-spec/api/search.json | 5 ++ .../rest-api-spec/api/search_template.json | 5 ++ .../rest-api-spec/test/msearch/10_basic.yml | 25 ++++++++ .../rest-api-spec/test/scroll/10_basic.yml | 63 +++++++++++++++++++ .../test/search/20_default_values.yml | 10 +++ .../action/search/RestMultiSearchAction.java | 11 +++- .../rest/action/search/RestSearchAction.java | 9 ++- .../action/search/RestSearchScrollAction.java | 9 +++ .../rollup/rest/RestRollupSearchAction.java | 9 +++ .../test/rollup/rollup_search.yml | 27 ++++++++ 17 files changed, 277 insertions(+), 4 deletions(-) diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java index 3aef9c32dc3d7..ab9bd388fcf8c 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java @@ -29,7 +29,9 @@ import org.elasticsearch.rest.action.search.RestSearchAction; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -37,7 +39,14 @@ public class RestMultiSearchTemplateAction extends BaseRestHandler { - private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM); + private static final Set RESPONSE_PARAMS; + + static { + final Set responseParams = new HashSet<>( + Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HIT_AS_INT_PARAM) + ); + RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); + } private final boolean allowExplicitIndex; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java index e9bbc7aeeecf2..b352f4aadb082 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java @@ -30,7 +30,9 @@ import org.elasticsearch.rest.action.search.RestSearchAction; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -38,7 +40,14 @@ public class RestSearchTemplateAction extends BaseRestHandler { - private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM); + private static final Set RESPONSE_PARAMS; + + static { + final Set responseParams = new HashSet<>( + Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HIT_AS_INT_PARAM) + ); + RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); + } public RestSearchTemplateAction(Settings settings, RestController controller) { super(settings); diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml index 89205e973fa04..f14004d3ebd55 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml @@ -124,3 +124,30 @@ - match: { hits.total: 1 } - length: { hits.hits: 1 } - length: { profile: 1 } + +--- +"Test with rest_total_hits_as_int": + - skip: + version: " - 6.5.99" + reason: rest_total_hits_as_int was introduced in 6.6.0 + + - do: + index: + index: test + type: type + id: 1 + body: {} + + - do: + put_script: + id: "template_1" + body: { "script": { "lang": "mustache", "source": { "query": { "match_all": {} } } } } + + - match: { acknowledged: true } + + - do: + search_template: + rest_total_hits_as_int: true + body: { "id": "template_1", "params": {} } + + - match: { hits.total: 0 } diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml index 0d0b16cdbfcf9..ae026d9d7a03e 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml @@ -156,3 +156,47 @@ setup: - match: { responses.0.hits.total: 2 } - match: { responses.1.hits.total: 1 } - match: { responses.2.hits.total: 1 } + + + - do: + put_script: + id: stored_template_1 + body: { "script": { "lang" : "mustache", "source": { "query": {"match": {"{{field}}": "{{value}}" }}}}} + - match: { acknowledged: true } + +--- +"Test with rest_total_hits_as_int": + - skip: + version: " - 6.5.99" + reason: rest_total_hits_as_int was introduced in 6.6.0 + + - do: + put_script: + id: stored_template_1 + body: { "script": { "lang": "mustache", "source": { "query": {"match": {"{{field}}": "{{value}}" }}}}} + - match: { acknowledged: true } + + - do: + msearch_template: + rest_total_hits_as_int: true + body: + - index: index_* + - id: stored_template_1 + params: + field: "foo" + value: "foo" + - index: _all + - id: stored_template_1 + params: + field: "foo" + value: "bar" + - index: index_2 + - id: stored_template_1 + params: + field: "foo" + value: "foo" + + - match: { responses.0.hits.total: 2 } + - match: { responses.1.hits.total: 1 } + - match: { responses.2.hits.total: 1 } + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json index 13281a2a232f4..4acc769a97e26 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json @@ -38,6 +38,11 @@ "type" : "number", "description" : "The number of concurrent shard requests each sub search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests", "default" : "The default grows with the number of nodes in the cluster but is at most 256." + }, + "rest_total_hits_as_int" : { + "type" : "boolean", + "description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number", + "default" : false } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json index e742093293255..dd02ed80786ca 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json @@ -28,6 +28,11 @@ "max_concurrent_searches" : { "type" : "number", "description" : "Controls the maximum number of concurrent searches the multi search api will execute" + }, + "rest_total_hits_as_int" : { + "type" : "boolean", + "description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number", + "default" : false } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json b/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json index 699ddcc9e00fc..f03653fa2f433 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json @@ -19,6 +19,11 @@ "scroll_id": { "type" : "string", "description" : "The scroll ID for scrolled search" + }, + "rest_total_hits_as_int" : { + "type" : "boolean", + "description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number", + "default" : false } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json index 12e13aace4ed0..84d3e4f15f8f0 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json @@ -182,6 +182,11 @@ "type" : "number", "description" : "A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.", "default" : 128 + }, + "rest_total_hits_as_int" : { + "type" : "boolean", + "description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number", + "default" : false } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json index df1fc1b079923..c5c9770222b3e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json @@ -62,6 +62,11 @@ "typed_keys": { "type" : "boolean", "description" : "Specify whether aggregation and suggester names should be prefixed by their respective types in the response" + }, + "rest_total_hits_as_int" : { + "type" : "boolean", + "description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number", + "default" : false } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml index c5d389e56bae3..325ca6a61563c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml @@ -96,3 +96,28 @@ setup: - match: { responses.3.error.root_cause.0.reason: "/no.such.index/" } - match: { responses.3.error.root_cause.0.index: index_3 } - match: { responses.4.hits.total: 4 } + +--- +"Search with rest_total_hits_as_int": + - skip: + version: " - 6.5.99" + reason: rest_total_hits_as_int was introduced in 6.6.0 + + - do: + msearch: + rest_total_hits_as_int: true + body: + - index: index_* + - query: + match: {foo: foo} + - index: index_2 + - query: + match_all: {} + - index: index_1 + - query: + match: {foo: foo} + + - match: { responses.0.hits.total: 2 } + - match: { responses.1.hits.total: 1 } + - match: { responses.2.hits.total: 1 } + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml index 7c816c2f08eca..459e02387f232 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml @@ -236,3 +236,66 @@ query: match_all: {} size: 0 + +--- +"Scroll with rest_total_as_int": + - skip: + version: " - 6.5.99" + reason: rest_total_hits_as_int was introduced in 6.6.0 + - do: + indices.create: + index: test_scroll + - do: + index: + index: test_scroll + type: test + id: 42 + body: { foo: 1 } + + - do: + index: + index: test_scroll + type: test + id: 43 + body: { foo: 2 } + + - do: + indices.refresh: {} + + - do: + search: + index: test_scroll + size: 1 + scroll: 1m + sort: foo + rest_total_hits_as_int: true + body: + query: + match_all: {} + + - set: {_scroll_id: scroll_id} + - match: {hits.total: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._id: "42" } + + - do: + scroll: + rest_total_hits_as_int: true + body: { "scroll_id": "$scroll_id", "scroll": "1m"} + + - match: {hits.total: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._id: "43" } + + - do: + scroll: + rest_total_hits_as_int: true + scroll_id: $scroll_id + scroll: 1m + + - match: {hits.total: 2 } + - length: {hits.hits: 0 } + + - do: + clear_scroll: + scroll_id: $scroll_id diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml index 52fbd19185335..b2b86640269b1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml @@ -75,3 +75,13 @@ setup: match: foo: bar +--- +"Search with rest_total_hits_as_int": + - skip: + version: " - 6.5.99" + reason: rest_total_hits_as_int was introduced in 6.6.0 + - do: + search: + rest_total_hits_as_int: true + + - match: {hits.total: 2} diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index e0ec14764f383..f1dde3cdf36d2 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -38,7 +38,9 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -47,7 +49,14 @@ public class RestMultiSearchAction extends BaseRestHandler { - private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM); + private static final Set RESPONSE_PARAMS; + + static { + final Set responseParams = new HashSet<>( + Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HIT_AS_INT_PARAM) + ); + RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); + } private final boolean allowExplicitIndex; diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 6f26ccce9b476..f0e81cb2f0cf7 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -43,6 +43,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.Set; import java.util.function.IntConsumer; @@ -54,7 +55,13 @@ public class RestSearchAction extends BaseRestHandler { public static final String TYPED_KEYS_PARAM = "typed_keys"; - private static final Set RESPONSE_PARAMS = Collections.singleton(TYPED_KEYS_PARAM); + public static final String TOTAL_HIT_AS_INT_PARAM = "rest_total_hits_as_int"; + private static final Set RESPONSE_PARAMS; + + static { + final Set responseParams = new HashSet<>(Arrays.asList(TYPED_KEYS_PARAM, TOTAL_HIT_AS_INT_PARAM)); + RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); + } public RestSearchAction(Settings settings, RestController controller) { super(settings); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java index bc3b0ccb56ac3..67f70b129c150 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java @@ -29,12 +29,16 @@ import org.elasticsearch.search.Scroll; import java.io.IOException; +import java.util.Collections; +import java.util.Set; import static org.elasticsearch.common.unit.TimeValue.parseTimeValue; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; public class RestSearchScrollAction extends BaseRestHandler { + Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TOTAL_HIT_AS_INT_PARAM); + public RestSearchScrollAction(Settings settings, RestController controller) { super(settings); @@ -70,4 +74,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC }}); return channel -> client.searchScroll(searchScrollRequest, new RestStatusToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return RESPONSE_PARAMS; + } } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java index 30638623ec937..e2bbfe9d8b679 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java @@ -16,9 +16,13 @@ import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction; import java.io.IOException; +import java.util.Collections; +import java.util.Set; public class RestRollupSearchAction extends BaseRestHandler { + private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TOTAL_HIT_AS_INT_PARAM); + public RestRollupSearchAction(Settings settings, RestController controller) { super(settings); controller.registerHandler(RestRequest.Method.GET, "_rollup_search", this); @@ -39,4 +43,9 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient public String getName() { return "rollup_search_action"; } + + @Override + protected Set responseParams() { + return RESPONSE_PARAMS; + } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml index 851d1cfa172e5..3399376b6ab69 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml @@ -154,6 +154,33 @@ setup: - match: { aggregations.histo.buckets.3.key_as_string: "2017-01-01T08:00:00.000Z" } - match: { aggregations.histo.buckets.3.doc_count: 20 } +--- +"Basic Search with rest_total_hits_as_int": + - skip: + version: " - 6.5.99" + reason: rest_total_hits_as_int was introduced in 6.6.0 + - do: + xpack.rollup.rollup_search: + index: "foo_rollup" + body: + size: 0 + aggs: + histo: + date_histogram: + field: "timestamp" + interval: "1h" + time_zone: "UTC" + + - length: { aggregations.histo.buckets: 4 } + - match: { aggregations.histo.buckets.0.key_as_string: "2017-01-01T05:00:00.000Z" } + - match: { aggregations.histo.buckets.0.doc_count: 1 } + - match: { aggregations.histo.buckets.1.key_as_string: "2017-01-01T06:00:00.000Z" } + - match: { aggregations.histo.buckets.1.doc_count: 2 } + - match: { aggregations.histo.buckets.2.key_as_string: "2017-01-01T07:00:00.000Z" } + - match: { aggregations.histo.buckets.2.doc_count: 10 } + - match: { aggregations.histo.buckets.3.key_as_string: "2017-01-01T08:00:00.000Z" } + - match: { aggregations.histo.buckets.3.doc_count: 20 } + --- "Formatted Date Histo":