Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support the 'fields' option in inner_hits and top_hits. (elastic#62259)
Browse files Browse the repository at this point in the history
This PR adds support for the 'fields' option in the following places:
* Anytime `inner_hits` is used, for both fetching nested/ child docs and field collapsing
* The `top_hits` aggregation

Addresses elastic#61949.
jtibshirani committed Sep 14, 2020
1 parent ec335c7 commit de100ba
Showing 15 changed files with 299 additions and 83 deletions.
Original file line number Diff line number Diff line change
@@ -24,6 +24,7 @@ The top_hits aggregation returns regular search hits, because of this many per h
* <<highlighting,Highlighting>>
* <<request-body-search-explain,Explain>>
* <<named-queries,Named queries>>
* <<search-fields-param,Search fields>>
* <<source-filtering,Source filtering>>
* <<stored-fields,Stored fields>>
* <<script-fields,Script fields>>
@@ -41,7 +42,7 @@ by aggregations, use a <<query-dsl-function-score-query,`function_score`>> or

==== Example

In the following example we group the sales by type and per type we show the last sale.
In the following example we group the sales by type and per type we show the last sale.
For each sale only the date and price fields are being included in the source.

[source,console]
Original file line number Diff line number Diff line change
@@ -74,6 +74,7 @@ Inner hits also supports the following per document features:

* <<highlighting,Highlighting>>
* <<request-body-search-explain,Explain>>
* <<search-fields-param,Search fields>>
* <<request-body-search-source-filtering,Source filtering>>
* <<script-fields,Script fields>>
* <<docvalue-fields,Doc value fields>>
Original file line number Diff line number Diff line change
@@ -176,7 +176,7 @@ public void testSimpleParentChild() throws Exception {
.setQuery(
hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(
new InnerHitBuilder()
.addDocValueField("message")
.addFetchField("message")
.setHighlightBuilder(new HighlightBuilder().field("message"))
.setExplain(true).setSize(1)
.addScriptField("script", new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5",
@@ -187,8 +187,18 @@ public void testSimpleParentChild() throws Exception {
assertThat(innerHits.getHits().length, equalTo(1));
assertThat(innerHits.getAt(0).getHighlightFields().get("message").getFragments()[0].string(), equalTo("<em>fox</em> eat quick"));
assertThat(innerHits.getAt(0).getExplanation().toString(), containsString("weight(message:fox"));
assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("eat"));
assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("fox eat quick"));
assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5"));

response = client().prepareSearch("articles")
.setQuery(
hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(
new InnerHitBuilder().addDocValueField("message").setSize(1)
)).get();
assertNoFailures(response);
innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
assertThat(innerHits.getHits().length, equalTo(1));
assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("eat"));
}

public void testRandomParentChild() throws Exception {
Original file line number Diff line number Diff line change
@@ -6,49 +6,50 @@ setup:
mappings:
properties:
numeric_group: { type: integer }
tag: { type: keyword }

- do:
index:
index: test
id: 1
version_type: external
version: 11
body: { numeric_group: 1, sort: 10 }
body: { numeric_group: 1, tag: A, sort: 10 }
- do:
index:
index: test
id: 2
version_type: external
version: 22
body: { numeric_group: 1, sort: 6 }
body: { numeric_group: 1, tag: B, sort: 6 }
- do:
index:
index: test
id: 3
version_type: external
version: 33
body: { numeric_group: 1, sort: 24 }
body: { numeric_group: 1, tag: A, sort: 24 }
- do:
index:
index: test
id: 4
version_type: external
version: 44
body: { numeric_group: 25, sort: 10 }
body: { numeric_group: 25, tag: B, sort: 10 }
- do:
index:
index: test
id: 5
version_type: external
version: 55
body: { numeric_group: 25, sort: 5 }
body: { numeric_group: 25, tag: A, sort: 5 }
- do:
index:
index: test
id: 6
version_type: external
version: 66
body: { numeric_group: 3, sort: 36 }
body: { numeric_group: 3, tag: B, sort: 36 }
- do:
indices.refresh:
index: test
@@ -161,6 +162,40 @@ setup:
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" }
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" }

---
"field collapsing, inner_hits, and fields":
- skip:
version: " - 7.99.99"
reason: the enhancement is not yet backported to 7.x
- do:
search:
rest_total_hits_as_int: true
index: test
body:
collapse:
field: numeric_group
inner_hits:
name: sub_hits
size: 2
sort: [{ sort: asc }]
fields: ["tag"]
sort: [{ sort: desc }]

- length: { hits.hits: 3 }
- length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 }
- match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" }
- match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0.fields.tag: ["B"]}
- length: { hits.hits.1.inner_hits.sub_hits.hits.hits: 2 }
- match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" }
- match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0.fields.tag: ["B"]}
- match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" }
- match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1.fields.tag: ["A"]}
- length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 }
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" }
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0.fields.tag: ["A"]}
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" }
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1.fields.tag: ["B"]}

---
"field collapsing, inner_hits and maxConcurrentGroupRequests":

Original file line number Diff line number Diff line change
@@ -166,6 +166,7 @@ public void setupSuiteScopeCluster() throws Exception {
.field(SORT_FIELD, i + 1)
.field("text", "some text to entertain")
.field("field1", 5)
.field("field2", 2.71)
.endObject()));
}

@@ -315,7 +316,7 @@ public void testBasics() throws Exception {
assertThat((Long) hits.getAt(1).getSortValues()[0], equalTo(higestSortValue - 1));
assertThat((Long) hits.getAt(2).getSortValues()[0], equalTo(higestSortValue - 2));

assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
}
}

@@ -402,7 +403,7 @@ public void testBreadthFirstWithScoreNeeded() throws Exception {
assertThat(hits.getTotalHits().value, equalTo(10L));
assertThat(hits.getHits().length, equalTo(3));

assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
}
}

@@ -433,7 +434,7 @@ public void testBreadthFirstWithAggOrderAndScoreNeeded() throws Exception {
assertThat(hits.getTotalHits().value, equalTo(10L));
assertThat(hits.getHits().length, equalTo(3));

assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
id--;
}
}
@@ -597,6 +598,7 @@ public void testFetchFeatures() {
.explain(true)
.storedField("text")
.docValueField("field1")
.fetchField("field2")
.scriptField("script",
new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap()))
.fetchSource("text", null)
@@ -639,13 +641,16 @@ public void testFetchFeatures() {

assertThat(hit.getMatchedQueries()[0], equalTo("test"));

DocumentField field = hit.field("field1");
assertThat(field.getValue().toString(), equalTo("5"));
DocumentField field1 = hit.field("field1");
assertThat(field1.getValue(), equalTo(5L));

DocumentField field2 = hit.field("field2");
assertThat(field2.getValue(), equalTo(2.71f));

assertThat(hit.getSourceAsMap().get("text").toString(), equalTo("some text to entertain"));

field = hit.field("script");
assertThat(field.getValue().toString(), equalTo("5"));
field2 = hit.field("script");
assertThat(field2.getValue().toString(), equalTo("5"));

assertThat(hit.getSourceAsMap().size(), equalTo(1));
assertThat(hit.getSourceAsMap().get("text").toString(), equalTo("some text to entertain"));
Original file line number Diff line number Diff line change
@@ -165,7 +165,7 @@ public void testSimpleNested() throws Exception {
.setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(
new InnerHitBuilder().setHighlightBuilder(new HighlightBuilder().field("comments.message"))
.setExplain(true)
.addDocValueField("comments.mes*")
.addFetchField("comments.mes*")
.addScriptField("script",
new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap()))
.setSize(1))).get();
@@ -176,8 +176,18 @@ public void testSimpleNested() throws Exception {
assertThat(innerHits.getAt(0).getHighlightFields().get("comments.message").getFragments()[0].string(),
equalTo("<em>fox</em> eat quick"));
assertThat(innerHits.getAt(0).getExplanation().toString(), containsString("weight(comments.message:fox in"));
assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("eat"));
assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("fox eat quick"));
assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5"));

response = client().prepareSearch("articles")
.setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(
new InnerHitBuilder()
.addDocValueField("comments.mes*")
.setSize(1))).get();
assertNoFailures(response);
innerHits = response.getHits().getAt(0).getInnerHits().get("comments");
assertThat(innerHits.getHits().length, equalTo(1));
assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("eat"));
}

public void testRandomNested() throws Exception {
Original file line number Diff line number Diff line change
@@ -136,6 +136,9 @@ private SearchSourceBuilder buildExpandSearchSourceBuilder(InnerHitBuilder optio
options.getFetchSourceContext().excludes());
}
}
if (options.getFetchFields() != null) {
options.getFetchFields().forEach(ff -> groupSource.fetchField(ff.field, ff.format));
}
if (options.getDocValueFields() != null) {
options.getDocValueFields().forEach(ff -> groupSource.docValueField(ff.field, ff.format));
}
Loading

0 comments on commit de100ba

Please sign in to comment.