Skip to content

Commit

Permalink
Add rest_total_hits_as_int in the search APIs (#35848)
Browse files Browse the repository at this point in the history
This change adds the support for a search query parameter named `rest_total_hits_as_int`.
This parameter will be used in the next major version (7.0) to restore the total hits as a number
in the response. This param is added in this version (6.x) to handle mixed cluster queries where nodes are in multiple versions (7.0 and 6.latest). The param does not change anything in 6x since total hits is always a number in 6x so it is just ignored.
  • Loading branch information
jimczi authored Nov 26, 2018
1 parent 44d6c74 commit bfbf134
Show file tree
Hide file tree
Showing 17 changed files with 277 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -29,15 +29,24 @@
import org.elasticsearch.rest.action.search.RestSearchAction;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;

import static org.elasticsearch.rest.RestRequest.Method.GET;
import static org.elasticsearch.rest.RestRequest.Method.POST;

public class RestMultiSearchTemplateAction extends BaseRestHandler {

private static final Set<String> RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM);
private static final Set<String> RESPONSE_PARAMS;

static {
final Set<String> responseParams = new HashSet<>(
Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HIT_AS_INT_PARAM)
);
RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams);
}

private final boolean allowExplicitIndex;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,24 @@
import org.elasticsearch.rest.action.search.RestSearchAction;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;

import static org.elasticsearch.rest.RestRequest.Method.GET;
import static org.elasticsearch.rest.RestRequest.Method.POST;

public class RestSearchTemplateAction extends BaseRestHandler {

private static final Set<String> RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM);
private static final Set<String> RESPONSE_PARAMS;

static {
final Set<String> responseParams = new HashSet<>(
Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HIT_AS_INT_PARAM)
);
RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams);
}

public RestSearchTemplateAction(Settings settings, RestController controller) {
super(settings);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,3 +124,30 @@
- match: { hits.total: 1 }
- length: { hits.hits: 1 }
- length: { profile: 1 }

---
"Test with rest_total_hits_as_int":
- skip:
version: " - 6.5.99"
reason: rest_total_hits_as_int was introduced in 6.6.0

- do:
index:
index: test
type: type
id: 1
body: {}

- do:
put_script:
id: "template_1"
body: { "script": { "lang": "mustache", "source": { "query": { "match_all": {} } } } }

- match: { acknowledged: true }

- do:
search_template:
rest_total_hits_as_int: true
body: { "id": "template_1", "params": {} }

- match: { hits.total: 0 }
Original file line number Diff line number Diff line change
Expand Up @@ -156,3 +156,47 @@ setup:
- match: { responses.0.hits.total: 2 }
- match: { responses.1.hits.total: 1 }
- match: { responses.2.hits.total: 1 }


- do:
put_script:
id: stored_template_1
body: { "script": { "lang" : "mustache", "source": { "query": {"match": {"{{field}}": "{{value}}" }}}}}
- match: { acknowledged: true }

---
"Test with rest_total_hits_as_int":
- skip:
version: " - 6.5.99"
reason: rest_total_hits_as_int was introduced in 6.6.0

- do:
put_script:
id: stored_template_1
body: { "script": { "lang": "mustache", "source": { "query": {"match": {"{{field}}": "{{value}}" }}}}}
- match: { acknowledged: true }

- do:
msearch_template:
rest_total_hits_as_int: true
body:
- index: index_*
- id: stored_template_1
params:
field: "foo"
value: "foo"
- index: _all
- id: stored_template_1
params:
field: "foo"
value: "bar"
- index: index_2
- id: stored_template_1
params:
field: "foo"
value: "foo"

- match: { responses.0.hits.total: 2 }
- match: { responses.1.hits.total: 1 }
- match: { responses.2.hits.total: 1 }

Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@
"type" : "number",
"description" : "The number of concurrent shard requests each sub search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests",
"default" : "The default grows with the number of nodes in the cluster but is at most 256."
},
"rest_total_hits_as_int" : {
"type" : "boolean",
"description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number",
"default" : false
}
}
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@
"max_concurrent_searches" : {
"type" : "number",
"description" : "Controls the maximum number of concurrent searches the multi search api will execute"
},
"rest_total_hits_as_int" : {
"type" : "boolean",
"description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number",
"default" : false
}
}
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,11 @@
"scroll_id": {
"type" : "string",
"description" : "The scroll ID for scrolled search"
},
"rest_total_hits_as_int" : {
"type" : "boolean",
"description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number",
"default" : false
}
}
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,11 @@
"type" : "number",
"description" : "A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.",
"default" : 128
},
"rest_total_hits_as_int" : {
"type" : "boolean",
"description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number",
"default" : false
}
}
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@
"typed_keys": {
"type" : "boolean",
"description" : "Specify whether aggregation and suggester names should be prefixed by their respective types in the response"
},
"rest_total_hits_as_int" : {
"type" : "boolean",
"description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number",
"default" : false
}
}
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,3 +96,28 @@ setup:
- match: { responses.3.error.root_cause.0.reason: "/no.such.index/" }
- match: { responses.3.error.root_cause.0.index: index_3 }
- match: { responses.4.hits.total: 4 }

---
"Search with rest_total_hits_as_int":
- skip:
version: " - 6.5.99"
reason: rest_total_hits_as_int was introduced in 6.6.0

- do:
msearch:
rest_total_hits_as_int: true
body:
- index: index_*
- query:
match: {foo: foo}
- index: index_2
- query:
match_all: {}
- index: index_1
- query:
match: {foo: foo}

- match: { responses.0.hits.total: 2 }
- match: { responses.1.hits.total: 1 }
- match: { responses.2.hits.total: 1 }

Original file line number Diff line number Diff line change
Expand Up @@ -236,3 +236,66 @@
query:
match_all: {}
size: 0

---
"Scroll with rest_total_as_int":
- skip:
version: " - 6.5.99"
reason: rest_total_hits_as_int was introduced in 6.6.0
- do:
indices.create:
index: test_scroll
- do:
index:
index: test_scroll
type: test
id: 42
body: { foo: 1 }

- do:
index:
index: test_scroll
type: test
id: 43
body: { foo: 2 }

- do:
indices.refresh: {}

- do:
search:
index: test_scroll
size: 1
scroll: 1m
sort: foo
rest_total_hits_as_int: true
body:
query:
match_all: {}

- set: {_scroll_id: scroll_id}
- match: {hits.total: 2 }
- length: {hits.hits: 1 }
- match: {hits.hits.0._id: "42" }

- do:
scroll:
rest_total_hits_as_int: true
body: { "scroll_id": "$scroll_id", "scroll": "1m"}

- match: {hits.total: 2 }
- length: {hits.hits: 1 }
- match: {hits.hits.0._id: "43" }

- do:
scroll:
rest_total_hits_as_int: true
scroll_id: $scroll_id
scroll: 1m

- match: {hits.total: 2 }
- length: {hits.hits: 0 }

- do:
clear_scroll:
scroll_id: $scroll_id
Original file line number Diff line number Diff line change
Expand Up @@ -75,3 +75,13 @@ setup:
match:
foo: bar

---
"Search with rest_total_hits_as_int":
- skip:
version: " - 6.5.99"
reason: rest_total_hits_as_int was introduced in 6.6.0
- do:
search:
rest_total_hits_as_int: true

- match: {hits.total: 2}
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,9 @@
import org.elasticsearch.search.builder.SearchSourceBuilder;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

Expand All @@ -47,7 +49,14 @@

public class RestMultiSearchAction extends BaseRestHandler {

private static final Set<String> RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM);
private static final Set<String> RESPONSE_PARAMS;

static {
final Set<String> responseParams = new HashSet<>(
Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HIT_AS_INT_PARAM)
);
RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams);
}

private final boolean allowExplicitIndex;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
import java.util.function.IntConsumer;

Expand All @@ -54,7 +55,13 @@
public class RestSearchAction extends BaseRestHandler {

public static final String TYPED_KEYS_PARAM = "typed_keys";
private static final Set<String> RESPONSE_PARAMS = Collections.singleton(TYPED_KEYS_PARAM);
public static final String TOTAL_HIT_AS_INT_PARAM = "rest_total_hits_as_int";
private static final Set<String> RESPONSE_PARAMS;

static {
final Set<String> responseParams = new HashSet<>(Arrays.asList(TYPED_KEYS_PARAM, TOTAL_HIT_AS_INT_PARAM));
RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams);
}

public RestSearchAction(Settings settings, RestController controller) {
super(settings);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,16 @@
import org.elasticsearch.search.Scroll;

import java.io.IOException;
import java.util.Collections;
import java.util.Set;

import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
import static org.elasticsearch.rest.RestRequest.Method.GET;
import static org.elasticsearch.rest.RestRequest.Method.POST;

public class RestSearchScrollAction extends BaseRestHandler {
Set<String> RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TOTAL_HIT_AS_INT_PARAM);

public RestSearchScrollAction(Settings settings, RestController controller) {
super(settings);

Expand Down Expand Up @@ -70,4 +74,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
}});
return channel -> client.searchScroll(searchScrollRequest, new RestStatusToXContentListener<>(channel));
}

@Override
protected Set<String> responseParams() {
return RESPONSE_PARAMS;
}
}
Loading

0 comments on commit bfbf134

Please sign in to comment.