From 8d12de9ee5b6c9e413e6b806d955966f8b5c8ac6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 31 Mar 2021 19:24:21 +0200 Subject: [PATCH] Add _size and _doc_count to fields output (#70575) Currently metadata fields like `_size` or `_doc_count` cannot be retrieved using the fields API. With this change, we allow this if the field is explicitely queried for using its name, but won't include metadata fields when e.g. requesting all fields via "*". With this change, not all metadata fields will be retrievable by using its name, but support for "_size" and "_doc_count" (which is fetched from source) is added. Support for other metadata field types will need to be decided case by case and an appropriate ValueFetcher needs to be supplied. Relates to #63569 --- docs/plugins/mapper-size.asciidoc | 22 ++++++------------- .../index/mapper/size/SizeMappingIT.java | 16 +++++++++++++- .../index/mapper/size/SizeFieldMapper.java | 21 ++++++++++++++++-- .../search/fetch/subphase/FieldFetcher.java | 7 +++++- .../fetch/subphase/FieldFetcherTests.java | 11 ++++++++-- 5 files changed, 56 insertions(+), 21 deletions(-) diff --git a/docs/plugins/mapper-size.asciidoc b/docs/plugins/mapper-size.asciidoc index c7140d865b832..dc44b3bc11ddc 100644 --- a/docs/plugins/mapper-size.asciidoc +++ b/docs/plugins/mapper-size.asciidoc @@ -26,7 +26,7 @@ PUT my-index-000001 -------------------------- The value of the `_size` field is accessible in queries, aggregations, scripts, -and when sorting: +and when sorting. It can be retrieved using the {ref}/search-fields.html#search-fields-param[fields API]: [source,console] -------------------------- @@ -65,16 +65,12 @@ GET my-index-000001/_search } } ], + "fields": ["_size"], <4> "script_fields": { "size": { - "script": "doc['_size']" <4> + "script": "doc['_size']" <5> } - }, - "docvalue_fields": [ - { - "field": "_size" <5> - } - ] + } } -------------------------- // TEST[continued] @@ -82,12 +78,8 @@ GET my-index-000001/_search <1> Querying on the `_size` field <2> Aggregating on the `_size` field <3> Sorting on the `_size` field -<4> Uses a +<4> Use the `fields` parameter to return the `_size` in the search response. +<5> Uses a {ref}/search-fields.html#script-fields[script field] to return the `_size` field in the search response. -<5> Uses a -{ref}/search-fields.html#docvalue-fields[doc value -field] to return the `_size` field in the search response. Doc value fields are -useful if -{ref}/modules-scripting-security.html#allowed-script-types-setting[inline -scripts are disabled]. + diff --git a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java index d678dff4d8195..72501aeafcf81 100644 --- a/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java +++ b/plugins/mapper-size/src/internalClusterTest/java/org/elasticsearch/index/mapper/size/SizeMappingIT.java @@ -9,6 +9,7 @@ import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse; import org.elasticsearch.action.get.GetResponse; +import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentType; @@ -95,11 +96,24 @@ private void assertSizeMappingEnabled(String index, String type, boolean enabled public void testBasic() throws Exception { assertAcked(prepareCreate("test").addMapping("type", "_size", "enabled=true")); - final String source = "{\"f\":10}"; + final String source = "{\"f\":\"" + randomAlphaOfLengthBetween(1, 100)+ "\"}"; indexRandom(true, client().prepareIndex("test", "type", "1").setSource(source, XContentType.JSON)); GetResponse getResponse = client().prepareGet("test", "type", "1").setStoredFields("_size").get(); assertNotNull(getResponse.getField("_size")); assertEquals(source.length(), (int) getResponse.getField("_size").getValue()); } + + public void testGetWithFields() throws Exception { + assertAcked(prepareCreate("test").addMapping("type", "_size", "enabled=true")); + final String source = "{\"f\":\"" + randomAlphaOfLengthBetween(1, 100)+ "\"}"; + indexRandom(true, + client().prepareIndex("test", "type").setId("1").setSource(source, XContentType.JSON)); + SearchResponse searchResponse = client().prepareSearch("test").addFetchField("_size").get(); + assertEquals(source.length(), ((Long) searchResponse.getHits().getHits()[0].getFields().get("_size").getValue()).intValue()); + + // this should not work when requesting fields via wildcard expression + searchResponse = client().prepareSearch("test").addFetchField("*").get(); + assertNull(searchResponse.getHits().getHits()[0].getFields().get("_size")); + } } diff --git a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java index f59aa820c086d..a37e3bb16fbb7 100644 --- a/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java +++ b/plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java @@ -9,12 +9,15 @@ package org.elasticsearch.index.mapper.size; import org.elasticsearch.common.Explicit; +import org.elasticsearch.index.mapper.DocValueFetcher; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.ValueFetcher; +import org.elasticsearch.index.query.SearchExecutionContext; import java.io.IOException; import java.util.Collections; @@ -43,12 +46,26 @@ protected List> getParameters() { @Override public SizeFieldMapper build() { - return new SizeFieldMapper(enabled.getValue(), new NumberFieldType(NAME, NumberType.INTEGER)); + return new SizeFieldMapper(enabled.getValue(), new SizeFieldType()); + } + } + + private static class SizeFieldType extends NumberFieldType { + SizeFieldType() { + super(NAME, NumberType.INTEGER); + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + if (hasDocValues() == false) { + return lookup -> Collections.emptyList(); + } + return new DocValueFetcher(docValueFormat(format, null), context.getForField(this)); } } public static final TypeParser PARSER = new ConfigurableTypeParser( - c -> new SizeFieldMapper(new Explicit<>(false, false), new NumberFieldType(NAME, NumberType.INTEGER)), + c -> new SizeFieldMapper(new Explicit<>(false, false), new SizeFieldType()), c -> new Builder() ); diff --git a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java index fe69da746fb2d..17cc7aab001e6 100644 --- a/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java +++ b/server/src/main/java/org/elasticsearch/search/fetch/subphase/FieldFetcher.java @@ -65,6 +65,7 @@ private static FieldFetcher create(SearchExecutionContext context, for (FieldAndFormat fieldAndFormat : fieldAndFormats) { String fieldPattern = fieldAndFormat.field; + boolean isWildcardPattern = Regex.isSimpleMatchPattern(fieldPattern); if (fieldAndFormat.includeUnmapped != null && fieldAndFormat.includeUnmapped) { unmappedFetchPattern.add(fieldAndFormat.field); } @@ -72,7 +73,11 @@ private static FieldFetcher create(SearchExecutionContext context, Collection concreteFields = context.simpleMatchToIndexNames(fieldPattern); for (String field : concreteFields) { MappedFieldType ft = context.getFieldType(field); - if (ft == null || context.isMetadataField(field)) { + if (ft == null) { + continue; + } + // we want to skip metadata fields if we have a wildcard pattern + if (context.isMetadataField(field) && isWildcardPattern) { continue; } if (field.startsWith(nestedScopePath) == false) { diff --git a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java index d4185c81d6c64..cd2f819df9410 100644 --- a/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java +++ b/server/src/test/java/org/elasticsearch/search/fetch/subphase/FieldFetcherTests.java @@ -182,10 +182,17 @@ public void testMetadataFields() throws IOException { MapperService mapperService = createMapperService(); XContentBuilder source = XContentFactory.jsonBuilder().startObject() .field("field", "value") + .field("_doc_count", 100) .endObject(); - Map fields = fetchFields(mapperService, source, "_routing"); - assertTrue(fields.isEmpty()); + Map fields = fetchFields(mapperService, source, "_doc_count"); + assertNotNull(fields.get("_doc_count")); + assertEquals(100, ((Integer) fields.get("_doc_count").getValue()).intValue()); + + // several other metadata fields throw exceptions via their value fetchers when trying to get them + for (String fieldname : new String[] { "_id", "_index", "_seq_no", "_routing", "_ignored" }) { + expectThrows(UnsupportedOperationException.class, () -> fetchFields(mapperService, source, fieldname)); + } } public void testFetchAllFields() throws IOException {