Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow fields to be set to * #42301

Merged
merged 4 commits into from
May 23, 2019
Merged

Allow fields to be set to * #42301

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3fa53ce
Allow `fields` to be set to `*`
matriv May 21, 2019
5a72254
Address comments
matriv May 22, 2019
544a74d
Merge remote-tracking branch 'upstream/master' into fix-39577
matriv May 22, 2019
f16cd52
remove unused imports
matriv May 22, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions server/src/main/java/org/elasticsearch/index/query/MultiMatchQueryBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -783,18 +783,20 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
multiMatchQuery.setTranspositions(fuzzyTranspositions);

Map<String, Float> newFieldsBoosts;
boolean isAllField;
if (fieldsBoosts.isEmpty()) {
// no fields provided, defaults to index.query.default_field
List<String> defaultFields = context.defaultFields();
boolean isAllField = defaultFields.size() == 1 && Regex.isMatchAllPattern(defaultFields.get(0));
if (isAllField && lenient == null) {
// Sets leniency to true if not explicitly
// set in the request
multiMatchQuery.setLenient(true);
}
newFieldsBoosts = QueryParserHelper.resolveMappingFields(context, QueryParserHelper.parseFieldsAndWeights(defaultFields));
isAllField = defaultFields.size() == 1 && Regex.isMatchAllPattern(defaultFields.get(0));
} else {
newFieldsBoosts = QueryParserHelper.resolveMappingFields(context, fieldsBoosts);
isAllField = QueryParserHelper.hasAllFieldsWildcard(fieldsBoosts.keySet());
}
if (isAllField && lenient == null) {
// Sets leniency to true if not explicitly
// set in the request
multiMatchQuery.setLenient(true);
}
return multiMatchQuery.parse(type, newFieldsBoosts, value, minimumShouldMatch);
}
Expand Down
6 changes: 5 additions & 1 deletion server/src/main/java/org/elasticsearch/index/query/QueryStringQueryBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -847,7 +847,11 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
}
} else if (fieldsAndWeights.size() > 0) {
final Map<String, Float> resolvedFields = QueryParserHelper.resolveMappingFields(context, fieldsAndWeights);
queryParser = new QueryStringQueryParser(context, resolvedFields, isLenient);
if (QueryParserHelper.hasAllFieldsWildcard(fieldsAndWeights.keySet())) {
queryParser = new QueryStringQueryParser(context, resolvedFields, lenient == null ? true : lenient);
} else {
queryParser = new QueryStringQueryParser(context, resolvedFields, isLenient);
}
} else {
List<String> defaultFields = context.defaultFields();
boolean isAllField = defaultFields.size() == 1 && Regex.isMatchAllPattern(defaultFields.get(0));
Expand Down
11 changes: 7 additions & 4 deletions server/src/main/java/org/elasticsearch/index/query/SimpleQueryStringBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -399,18 +399,21 @@ public SimpleQueryStringBuilder fuzzyTranspositions(boolean fuzzyTranspositions)
protected Query doToQuery(QueryShardContext context) throws IOException {
Settings newSettings = new Settings(settings);
final Map<String, Float> resolvedFieldsAndWeights;
boolean isAllField;
if (fieldsAndWeights.isEmpty() == false) {
resolvedFieldsAndWeights = QueryParserHelper.resolveMappingFields(context, fieldsAndWeights);
isAllField = QueryParserHelper.hasAllFieldsWildcard(fieldsAndWeights.keySet());
} else {
List<String> defaultFields = context.defaultFields();
boolean isAllField = defaultFields.size() == 1 && Regex.isMatchAllPattern(defaultFields.get(0));
if (isAllField) {
newSettings.lenient(lenientSet ? settings.lenient() : true);
}
isAllField = defaultFields.size() == 1 && Regex.isMatchAllPattern(defaultFields.get(0));
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@jimczi Why do we check only the first element here, shouldn't we check all the defaultFields?
(same applies to all 3 types of queries)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree we should check all elements. However I don't think it's useful to set fields or the default fields to something like ["*", "field1", "field2"] but for consistency we should always set the leniency to true if * is part of the list of fields to query.

resolvedFieldsAndWeights = QueryParserHelper.resolveMappingFields(context,
QueryParserHelper.parseFieldsAndWeights(defaultFields));
}

if (isAllField) {
newSettings.lenient(lenientSet ? settings.lenient() : true);
}

final SimpleQueryStringQueryParser sqp;
if (analyzer == null) {
sqp = new SimpleQueryStringQueryParser(resolvedFieldsAndWeights, flags, newSettings, context);
Expand Down
8 changes: 8 additions & 0 deletions server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,4 +161,12 @@ private static void checkForTooManyFields(Map<String, Float> fields, QueryShardC
throw new IllegalArgumentException("field expansion matches too many fields, limit: " + limit + ", got: " + fields.size());
}
}

/**
* Returns true if any of the fields is the wildcard {@code *}, false otherwise.
* @param fields A collection of field names
*/
public static boolean hasAllFieldsWildcard(Collection<String> fields) {
return fields.stream().anyMatch(Regex::isMatchAllPattern);
}
}
94 changes: 59 additions & 35 deletions server/src/test/java/org/elasticsearch/index/query/MultiMatchQueryBuilderTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
import static org.hamcrest.CoreMatchers.anyOf;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.hasItems;
import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.collection.IsCollectionWithSize.hasSize;

Expand Down Expand Up @@ -419,42 +420,65 @@ public void testDefaultField() throws Exception {
.build())
);

MultiMatchQueryBuilder qb = new MultiMatchQueryBuilder("hello");
query = qb.toQuery(context);
DisjunctionMaxQuery expected = new DisjunctionMaxQuery(
Arrays.asList(
new TermQuery(new Term(STRING_FIELD_NAME, "hello")),
new BoostQuery(new TermQuery(new Term(STRING_FIELD_NAME_2, "hello")), 5.0f)
), 0.0f
);
assertEquals(expected, query);

context.getIndexSettings().updateIndexMetaData(
newIndexMeta("index", context.getIndexSettings().getSettings(),
Settings.builder().putList("index.query.default_field",
STRING_FIELD_NAME, STRING_FIELD_NAME_2 + "^5", INT_FIELD_NAME).build())
);
// should fail because lenient defaults to false
IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> qb.toQuery(context));
assertThat(exc, instanceOf(NumberFormatException.class));
assertThat(exc.getMessage(), equalTo("For input string: \"hello\""));

// explicitly sets lenient
qb.lenient(true);
query = qb.toQuery(context);
expected = new DisjunctionMaxQuery(
Arrays.asList(
new TermQuery(new Term(STRING_FIELD_NAME, "hello")),
new BoostQuery(new TermQuery(new Term(STRING_FIELD_NAME_2, "hello")), 5.0f),
new MatchNoDocsQuery("failed [mapped_int] query, caused by number_format_exception:[For input string: \"hello\"]")
), 0.0f
);
assertEquals(expected, query);
try {
MultiMatchQueryBuilder qb = new MultiMatchQueryBuilder("hello");
query = qb.toQuery(context);
DisjunctionMaxQuery expected = new DisjunctionMaxQuery(
Arrays.asList(
new TermQuery(new Term(STRING_FIELD_NAME, "hello")),
new BoostQuery(new TermQuery(new Term(STRING_FIELD_NAME_2, "hello")), 5.0f)
), 0.0f
);
assertEquals(expected, query);

context.getIndexSettings().updateIndexMetaData(
newIndexMeta("index", context.getIndexSettings().getSettings(),
Settings.builder().putList("index.query.default_field",
STRING_FIELD_NAME, STRING_FIELD_NAME_2 + "^5", INT_FIELD_NAME).build())
);
// should fail because lenient defaults to false
IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> qb.toQuery(context));
assertThat(exc, instanceOf(NumberFormatException.class));
assertThat(exc.getMessage(), equalTo("For input string: \"hello\""));

// explicitly sets lenient
qb.lenient(true);
query = qb.toQuery(context);
expected = new DisjunctionMaxQuery(
Arrays.asList(
new TermQuery(new Term(STRING_FIELD_NAME, "hello")),
new BoostQuery(new TermQuery(new Term(STRING_FIELD_NAME_2, "hello")), 5.0f),
new MatchNoDocsQuery("failed [mapped_int] query, caused by number_format_exception:[For input string: \"hello\"]")
), 0.0f
);
assertEquals(expected, query);

} finally {
// Reset to the default value
context.getIndexSettings().updateIndexMetaData(
newIndexMeta("index", context.getIndexSettings().getSettings(),
Settings.builder().putNull("index.query.default_field").build())
);
}
}

context.getIndexSettings().updateIndexMetaData(
newIndexMeta("index", context.getIndexSettings().getSettings(),
Settings.builder().putNull("index.query.default_field").build())
);
public void testAllFieldsWildcard() throws Exception {
Query query = new MultiMatchQueryBuilder("hello")
.field(STRING_FIELD_NAME)
.field("*")
.field(STRING_FIELD_NAME_2)
.toQuery(createShardContext());
assertEquals(DisjunctionMaxQuery.class, query.getClass());
DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query;
int noMatchNoDocsQueries = 0;
for (Query q : disjunctionMaxQuery.getDisjuncts()) {
if (q.getClass() == MatchNoDocsQuery.class) {
noMatchNoDocsQueries++;
}
}
assertEquals(11, noMatchNoDocsQueries);
assertThat(disjunctionMaxQuery.getDisjuncts(), hasItems(new TermQuery(new Term(STRING_FIELD_NAME, "hello")),
new TermQuery(new Term(STRING_FIELD_NAME_2, "hello"))));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you also add a test with a single field set on * ?

}

public void testWithStopWords() throws Exception {
Expand Down
20 changes: 20 additions & 0 deletions server/src/test/java/org/elasticsearch/index/query/QueryStringQueryBuilderTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertBooleanSubQuery;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertDisjunctionSubQuery;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.hamcrest.CoreMatchers.hasItems;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.instanceOf;

Expand Down Expand Up @@ -1278,6 +1279,25 @@ public void testDefaultField() throws Exception {
}
}

public void testAllFieldsWildcard() throws Exception {
Query query = new QueryStringQueryBuilder("hello")
.field(STRING_FIELD_NAME)
.field("*")
.field(STRING_FIELD_NAME_2)
.toQuery(createShardContext());
assertEquals(DisjunctionMaxQuery.class, query.getClass());
DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query;
int noMatchNoDocsQueries = 0;
for (Query q : disjunctionMaxQuery.getDisjuncts()) {
if (q.getClass() == MatchNoDocsQuery.class) {
noMatchNoDocsQueries++;
}
}
assertEquals(11, noMatchNoDocsQueries);
assertThat(disjunctionMaxQuery.getDisjuncts(), hasItems(new TermQuery(new Term(STRING_FIELD_NAME, "hello")),
new TermQuery(new Term(STRING_FIELD_NAME_2, "hello"))));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here

}

/**
* the quote analyzer should overwrite any other forced analyzer in quoted parts of the query
*/
Expand Down
49 changes: 36 additions & 13 deletions server/src/test/java/org/elasticsearch/index/query/SimpleQueryStringBuilderTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
import java.util.Set;

import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.hasItems;
import static org.hamcrest.Matchers.anyOf;
import static org.hamcrest.Matchers.either;
import static org.hamcrest.Matchers.equalTo;
Expand Down Expand Up @@ -580,20 +581,42 @@ public void testDefaultField() throws Exception {
newIndexMeta("index", context.getIndexSettings().getSettings(), Settings.builder().putList("index.query.default_field",
STRING_FIELD_NAME, STRING_FIELD_NAME_2 + "^5").build())
);
try {
Query query = new SimpleQueryStringBuilder("hello")
.toQuery(context);
Query expected = new DisjunctionMaxQuery(
Arrays.asList(
new TermQuery(new Term(STRING_FIELD_NAME, "hello")),
new BoostQuery(new TermQuery(new Term(STRING_FIELD_NAME_2, "hello")), 5.0f)
), 1.0f
);
assertEquals(expected, query);
} finally {
// Reset to the default value
context.getIndexSettings().updateIndexMetaData(
newIndexMeta("index",
context.getIndexSettings().getSettings(), Settings.builder().putList("index.query.default_field", "*").build())
);
}
}

public void testAllFieldsWildcard() throws Exception {
Query query = new SimpleQueryStringBuilder("hello")
.toQuery(context);
Query expected = new DisjunctionMaxQuery(
Arrays.asList(
new TermQuery(new Term(STRING_FIELD_NAME, "hello")),
new BoostQuery(new TermQuery(new Term(STRING_FIELD_NAME_2, "hello")), 5.0f)
), 1.0f
);
assertEquals(expected, query);
// Reset the default value
context.getIndexSettings().updateIndexMetaData(
newIndexMeta("index",
context.getIndexSettings().getSettings(), Settings.builder().putList("index.query.default_field", "*").build())
);
.field(STRING_FIELD_NAME)
.field("*")
.field(STRING_FIELD_NAME_2)
.toQuery(createShardContext());
assertEquals(DisjunctionMaxQuery.class, query.getClass());
DisjunctionMaxQuery disjunctionMaxQuery = (DisjunctionMaxQuery) query;
int noMatchNoDocsQueries = 0;
for (Query q : disjunctionMaxQuery.getDisjuncts()) {
if (q.getClass() == MatchNoDocsQuery.class) {
noMatchNoDocsQueries++;
}
}
assertEquals(11, noMatchNoDocsQueries);
assertThat(disjunctionMaxQuery.getDisjuncts(), hasItems(new TermQuery(new Term(STRING_FIELD_NAME, "hello")),
new TermQuery(new Term(STRING_FIELD_NAME_2, "hello"))));
}

public void testToFuzzyQuery() throws Exception {
Expand Down