Skip to content

Commit

Permalink
Limit the number of expanded fields it query_string and simple_query_…
Browse files Browse the repository at this point in the history
…string (#26541)

* Limit the number of expanded fields it query_string and simple_query_string

This limits the number of automatically expanded fields for the "all fields"
mode (`"default_field": "*"`) for the `query_string` and `simple_query_string`
queries to 1024 fields.

Resolves #25105

* Add blurb about limit to the docs
  • Loading branch information
dakrone authored Sep 8, 2017
1 parent dd90cf1 commit 2702918
Show file tree
Hide file tree
Showing 5 changed files with 87 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ public static Map<String, Float> resolveMappingFields(QueryShardContext context,
!multiField, !allField, fieldSuffix);
resolvedFields.putAll(fieldMap);
}
checkForTooManyFields(resolvedFields);
return resolvedFields;
}

Expand Down Expand Up @@ -184,6 +185,13 @@ public static Map<String, Float> resolveMappingField(QueryShardContext context,
}
fields.put(fieldName, weight);
}
checkForTooManyFields(fields);
return fields;
}

private static void checkForTooManyFields(Map<String, Float> fields) {
if (fields.size() > 1024) {
throw new IllegalArgumentException("field expansion matches too many fields, limit: 1024, got: " + fields.size());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.Operator;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.QueryStringQueryBuilder;
Expand All @@ -46,6 +47,7 @@
import java.util.List;
import java.util.Set;

import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery;
import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
Expand Down Expand Up @@ -351,6 +353,37 @@ public void testGraphQueries() throws Exception {
assertSearchHits(searchResponse, "1", "2", "3");
}

public void testLimitOnExpandedFields() throws Exception {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject("type1");
builder.startObject("properties");
for (int i = 0; i < 1025; i++) {
builder.startObject("field" + i).field("type", "text").endObject();
}
builder.endObject(); // properties
builder.endObject(); // type1
builder.endObject();

assertAcked(prepareCreate("toomanyfields")
.setSettings(Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1200))
.addMapping("type1", builder));

client().prepareIndex("toomanyfields", "type1", "1").setSource("field171", "foo bar baz").get();
refresh();

Exception e = expectThrows(Exception.class, () -> {
QueryStringQueryBuilder qb = queryStringQuery("bar");
if (randomBoolean()) {
qb.useAllFields(true);
}
logger.info("--> using {}", qb);
client().prepareSearch("toomanyfields").setQuery(qb).get();
});
assertThat(ExceptionsHelper.detailedMessage(e),
containsString("field expansion matches too many fields, limit: 1024, got: 1025"));
}

private void assertHits(SearchHits hits, String... ids) {
assertThat(hits.getTotalHits(), equalTo((long) ids.length));
Set<String> hitIds = new HashSet<>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,15 @@
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.Operator;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.SimpleQueryStringBuilder;
import org.elasticsearch.index.query.SimpleQueryStringFlag;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.SearchHit;
Expand Down Expand Up @@ -540,6 +544,38 @@ public void testAllFieldsWithSpecifiedLeniency() throws IOException {
containsString("NumberFormatException[For input string: \"foo123\"]"));
}


public void testLimitOnExpandedFields() throws Exception {
XContentBuilder builder = jsonBuilder();
builder.startObject();
builder.startObject("type1");
builder.startObject("properties");
for (int i = 0; i < 1025; i++) {
builder.startObject("field" + i).field("type", "text").endObject();
}
builder.endObject(); // properties
builder.endObject(); // type1
builder.endObject();

assertAcked(prepareCreate("toomanyfields")
.setSettings(Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1200))
.addMapping("type1", builder));

client().prepareIndex("toomanyfields", "type1", "1").setSource("field171", "foo bar baz").get();
refresh();

Exception e = expectThrows(Exception.class, () -> {
SimpleQueryStringBuilder qb = simpleQueryStringQuery("bar");
if (randomBoolean()) {
qb.useAllFields(true);
}
logger.info("--> using {}", qb);
client().prepareSearch("toomanyfields").setQuery(qb).get();
});
assertThat(ExceptionsHelper.detailedMessage(e),
containsString("field expansion matches too many fields, limit: 1024, got: 1025"));
}

private void assertHits(SearchHits hits, String... ids) {
assertThat(hits.getTotalHits(), equalTo((long) ids.length));
Set<String> hitIds = new HashSet<>();
Expand Down
12 changes: 6 additions & 6 deletions docs/reference/query-dsl/query-string-query.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,12 @@ The `query_string` top level parameters include:
|Parameter |Description
|`query` |The actual query to be parsed. See <<query-string-syntax>>.

|`default_field` |The default field for query terms if no prefix field
is specified. Defaults to the `index.query.default_field` index
settings, which in turn defaults to `*`.
`*` extracts all fields in the mapping that are eligible to term queries
and filters the metadata fields. All extracted fields are then combined
to build a query when no prefix field is provided.
|`default_field` |The default field for query terms if no prefix field is
specified. Defaults to the `index.query.default_field` index settings, which in
turn defaults to `*`. `*` extracts all fields in the mapping that are eligible
to term queries and filters the metadata fields. All extracted fields are then
combined to build a query when no prefix field is provided. There is a limit of
no more than 1024 fields being queried at once.

|`default_operator` |The default operator used if no explicit operator
is specified. For example, with a default operator of `OR`, the query
Expand Down
7 changes: 4 additions & 3 deletions docs/reference/query-dsl/simple-query-string-query.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,10 @@ The `simple_query_string` top level parameters include:
|`query` |The actual query to be parsed. See below for syntax.

|`fields` |The fields to perform the parsed query against. Defaults to the
`index.query.default_field` index settings, which in turn defaults to `*`.
`*` extracts all fields in the mapping that are eligible to term queries
and filters the metadata fields.
`index.query.default_field` index settings, which in turn defaults to `*`. `*`
extracts all fields in the mapping that are eligible to term queries and filters
the metadata fields. There is a limit of no more than 1024 fields being queried
at once.

|`default_operator` |The default operator used if no explicit operator
is specified. For example, with a default operator of `OR`, the query
Expand Down

0 comments on commit 2702918

Please sign in to comment.