From 96e3428d5604a0849f422e50b6c5fe3789aad7f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 11 Feb 2021 17:41:46 +0100 Subject: [PATCH] Add runtime field section to Field Capabilities API (#68904) (#68915) Currently runtime fields from search requests don't appear in the output of the field capabilities API, but some consumer of runtime fields would like to see runtime section just like they are defined in search requests reflected and merged into the field capabilities output. This change adds parsing of a "runtime_mappings" section equivallent to the one on search requests to the `_field_caps` endpoint, passes this section down to the shard level where any runtime fields defined here overwrite the mapping of the targetet indices. Closes #68117 --- docs/reference/search/field-caps.asciidoc | 7 ++ .../FieldCapabilitiesIndexRequest.java | 15 +++- .../fieldcaps/FieldCapabilitiesRequest.java | 27 +++++- .../TransportFieldCapabilitiesAction.java | 2 +- ...TransportFieldCapabilitiesIndexAction.java | 2 +- .../action/RestFieldCapabilitiesAction.java | 15 +++- .../FieldCapabilitiesRequestTests.java | 58 +++++++++++++ .../runtime_fields/40_runtime_mappings.yml | 87 +++++++++++++++++++ 8 files changed, 207 insertions(+), 6 deletions(-) create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml diff --git a/docs/reference/search/field-caps.asciidoc b/docs/reference/search/field-caps.asciidoc index 1825903277ce8..d2ce46fedc746 100644 --- a/docs/reference/search/field-caps.asciidoc +++ b/docs/reference/search/field-caps.asciidoc @@ -88,6 +88,13 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailab (Optional, <> Allows to filter indices if the provided query rewrites to `match_none` on every shard. +`runtime_mappings`:: +(Optional, object) +Defines ad-hoc <> in the request similar +to the way it is done in <>. These fields +exist only as part of the query and take precedence over fields defined with the +same name in the index mappings. + [[search-field-caps-api-response-body]] ==== {api-response-body-title} diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java index bf4f5ae000464..39550b64e0f46 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java @@ -20,6 +20,8 @@ import org.elasticsearch.index.shard.ShardId; import java.io.IOException; +import java.util.Collections; +import java.util.Map; import java.util.Objects; public class FieldCapabilitiesIndexRequest extends ActionRequest implements IndicesRequest { @@ -31,6 +33,7 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi private final OriginalIndices originalIndices; private final QueryBuilder indexFilter; private final long nowInMillis; + private Map runtimeFields; private ShardId shardId; @@ -47,13 +50,15 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi } indexFilter = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalNamedWriteable(QueryBuilder.class) : null; nowInMillis = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readLong() : 0L; + runtimeFields = in.getVersion().onOrAfter(Version.V_7_12_0) ? in.readMap() : Collections.emptyMap(); } FieldCapabilitiesIndexRequest(String[] fields, String index, OriginalIndices originalIndices, QueryBuilder indexFilter, - long nowInMillis) { + long nowInMillis, + Map runtimeFields) { if (fields == null || fields.length == 0) { throw new IllegalArgumentException("specified fields can't be null or empty"); } @@ -62,6 +67,7 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi this.originalIndices = originalIndices; this.indexFilter = indexFilter; this.nowInMillis = nowInMillis; + this.runtimeFields = runtimeFields; } public String[] fields() { @@ -86,6 +92,10 @@ public QueryBuilder indexFilter() { return indexFilter; } + public Map runtimeFields() { + return runtimeFields; + } + public ShardId shardId() { return shardId; } @@ -112,6 +122,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalNamedWriteable(indexFilter); out.writeLong(nowInMillis); } + if (out.getVersion().onOrAfter(Version.V_7_12_0)) { + out.writeMap(runtimeFields); + } } @Override diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java index 7026fe2e3d999..161f6ae879415 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java @@ -23,7 +23,9 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collections; import java.util.HashSet; +import java.util.Map; import java.util.Objects; import java.util.Set; @@ -37,6 +39,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind // pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged private boolean mergeResults = true; private QueryBuilder indexFilter; + private Map runtimeFields = Collections.emptyMap(); private Long nowInMillis; public FieldCapabilitiesRequest(StreamInput in) throws IOException { @@ -52,6 +55,7 @@ public FieldCapabilitiesRequest(StreamInput in) throws IOException { } indexFilter = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalNamedWriteable(QueryBuilder.class) : null; nowInMillis = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalLong() : null; + runtimeFields = in.getVersion().onOrAfter(Version.V_7_12_0) ? in.readMap() : Collections.emptyMap(); } public FieldCapabilitiesRequest() { @@ -90,6 +94,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalNamedWriteable(indexFilter); out.writeOptionalLong(nowInMillis); } + if (out.getVersion().onOrAfter(Version.V_7_12_0)) { + out.writeMap(runtimeFields); + } } @Override @@ -98,6 +105,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (indexFilter != null) { builder.field("index_filter", indexFilter); } + if (runtimeFields.isEmpty() == false) { + builder.field("runtime_mappings", runtimeFields); + } builder.endObject(); return builder; } @@ -121,6 +131,7 @@ public String[] fields() { /** * The list of indices to lookup */ + @Override public FieldCapabilitiesRequest indices(String... indices) { this.indices = Objects.requireNonNull(indices, "indices must not be null"); return this; @@ -166,6 +177,17 @@ public FieldCapabilitiesRequest indexFilter(QueryBuilder indexFilter) { public QueryBuilder indexFilter() { return indexFilter; } + /** + * Allows adding search runtime fields if provided. + */ + public FieldCapabilitiesRequest runtimeFields(Map runtimeFieldsSection) { + this.runtimeFields = runtimeFieldsSection; + return this; + } + + public Map runtimeFields() { + return this.runtimeFields; + } Long nowInMillis() { return nowInMillis; @@ -195,12 +217,13 @@ public boolean equals(Object o) { indicesOptions.equals(that.indicesOptions) && Arrays.equals(fields, that.fields) && Objects.equals(indexFilter, that.indexFilter) && - Objects.equals(nowInMillis, that.nowInMillis); + Objects.equals(nowInMillis, that.nowInMillis) && + Objects.equals(runtimeFields, that.runtimeFields); } @Override public int hashCode() { - int result = Objects.hash(indicesOptions, includeUnmapped, mergeResults, indexFilter, nowInMillis); + int result = Objects.hash(indicesOptions, includeUnmapped, mergeResults, indexFilter, nowInMillis, runtimeFields); result = 31 * result + Arrays.hashCode(indices); result = 31 * result + Arrays.hashCode(fields); return result; diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java index 00c704b08a321..0241313976e25 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java @@ -101,7 +101,7 @@ public void onFailure(Exception e) { }; for (String index : concreteIndices) { shardAction.execute(new FieldCapabilitiesIndexRequest(request.fields(), index, localIndices, - request.indexFilter(), nowInMillis), innerListener); + request.indexFilter(), nowInMillis, request.runtimeFields()), innerListener); } // this is the cross cluster part of this API - we force the other cluster to not merge the results but instead diff --git a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java index 1b459591f431a..93560c2ca02f6 100644 --- a/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java @@ -107,7 +107,7 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd try (Engine.Searcher searcher = indexShard.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE)) { final SearchExecutionContext searchExecutionContext = indexService.newSearchExecutionContext(shardId.id(), 0, - searcher, request::nowInMillis, null, Collections.emptyMap()); + searcher, request::nowInMillis, null, request.runtimeFields()); if (canMatchShard(request, searchExecutionContext) == false) { return new FieldCapabilitiesIndexResponse(request.index(), Collections.emptyMap(), false); diff --git a/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java b/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java index a2e9eb11a51eb..3cdce9e676797 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java @@ -11,7 +11,9 @@ import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; @@ -20,6 +22,7 @@ import static java.util.Arrays.asList; import static java.util.Collections.unmodifiableList; +import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; @@ -52,9 +55,19 @@ public RestChannelConsumer prepareRequest(final RestRequest request, fieldRequest.includeUnmapped(request.paramAsBoolean("include_unmapped", false)); request.withContentOrSourceParamParserOrNull(parser -> { if (parser != null) { - fieldRequest.indexFilter(RestActions.getQueryContent("index_filter", parser)); + PARSER.parse(parser, fieldRequest, null); } }); return channel -> client.fieldCaps(fieldRequest, new RestToXContentListener<>(channel)); } + + private static ParseField INDEX_FILTER_FIELD = new ParseField("index_filter"); + private static ParseField RUNTIME_MAPPINGS_FIELD = new ParseField("runtime_mappings"); + + private static final ObjectParser PARSER = new ObjectParser<>("field_caps_request"); + + static { + PARSER.declareObject(FieldCapabilitiesRequest::indexFilter, (p, c) -> parseInnerQueryBuilder(p), INDEX_FILTER_FIELD); + PARSER.declareObject(FieldCapabilitiesRequest::runtimeFields, (p, c) -> p.map(), RUNTIME_MAPPINGS_FIELD); + } } diff --git a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java index e7b7aafe14155..7fd34579d71fb 100644 --- a/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java @@ -10,15 +10,27 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.ArrayUtils; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.AbstractWireSerializingTestCase; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.function.Consumer; +import static java.util.Collections.singletonMap; + public class FieldCapabilitiesRequestTests extends AbstractWireSerializingTestCase { @Override @@ -41,9 +53,24 @@ protected FieldCapabilitiesRequest createTestInstance() { request.indicesOptions(randomBoolean() ? IndicesOptions.strictExpand() : IndicesOptions.lenientExpandOpen()); } request.includeUnmapped(randomBoolean()); + if (randomBoolean()) { + request.nowInMillis(randomLong()); + } + if (randomBoolean()) { + request.indexFilter(QueryBuilders.termQuery("field", randomAlphaOfLength(5))); + } + if (randomBoolean()) { + request.runtimeFields(Collections.singletonMap(randomAlphaOfLength(5), randomAlphaOfLength(5))); + } return request; } + @Override + protected NamedWriteableRegistry getNamedWriteableRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList()); + return new NamedWriteableRegistry(searchModule.getNamedWriteables()); + } + @Override protected Writeable.Reader instanceReader() { return FieldCapabilitiesRequest::new; @@ -67,6 +94,11 @@ protected FieldCapabilitiesRequest mutateInstance(FieldCapabilitiesRequest insta }); mutators.add(request -> request.setMergeResults(request.isMergeResults() == false)); mutators.add(request -> request.includeUnmapped(request.includeUnmapped() == false)); + mutators.add(request -> request.nowInMillis(request.nowInMillis() != null ? request.nowInMillis() + 1 : 1L)); + mutators.add( + request -> request.indexFilter(request.indexFilter() != null ? request.indexFilter().boost(2) : QueryBuilders.matchAllQuery()) + ); + mutators.add(request -> request.runtimeFields(Collections.singletonMap("other_key", "other_value"))); FieldCapabilitiesRequest mutatedInstance = copyInstance(instance); Consumer mutator = randomFrom(mutators); @@ -74,6 +106,32 @@ protected FieldCapabilitiesRequest mutateInstance(FieldCapabilitiesRequest insta return mutatedInstance; } + public void testToXContent() throws IOException { + FieldCapabilitiesRequest request = new FieldCapabilitiesRequest(); + request.indexFilter(QueryBuilders.termQuery("field", "value")); + request.runtimeFields(singletonMap("day_of_week", singletonMap("type", "keyword"))); + XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON); + String xContent = BytesReference.bytes(request.toXContent(builder, ToXContent.EMPTY_PARAMS)).utf8ToString(); + assertEquals( + ("{" + + " \"index_filter\": {\n" + + " \"term\": {\n" + + " \"field\": {\n" + + " \"value\": \"value\",\n" + + " \"boost\": 1.0\n" + + " }\n" + + " }\n" + + " },\n" + + " \"runtime_mappings\": {\n" + + " \"day_of_week\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + "}").replaceAll("\\s+", ""), + xContent + ); + } + public void testValidation() { FieldCapabilitiesRequest request = new FieldCapabilitiesRequest() .indices("index2"); diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml new file mode 100644 index 0000000000000..5bf4c8923ef79 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml @@ -0,0 +1,87 @@ +--- +setup: + - do: + indices.create: + index: test-1 + body: + mappings: + properties: + timestamp: + type: date + + - do: + index: + index: test-1 + body: { timestamp: "2015-01-02" } + + - do: + indices.refresh: + index: [test-1] + +--- +"Field caps with runtime mappings section": + + - skip: + version: " - 7.11.99" + reason: Runtime mappings support was added in 7.12 + + - do: + field_caps: + index: test-* + fields: "*" + body: + runtime_mappings: + day_of_week: + type: keyword + script: + source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + + - match: {indices: ["test-1"]} + - length: {fields.timestamp: 1} + - match: {fields.timestamp.date.type: date} + - match: {fields.timestamp.date.searchable: true} + - match: {fields.timestamp.date.aggregatable: true} + - length: {fields.day_of_week: 1} + - match: {fields.day_of_week.keyword.type: keyword} + - match: {fields.day_of_week.keyword.searchable: true} + - match: {fields.day_of_week.keyword.aggregatable: true} + +--- +"Field caps with runtime mappings section overwriting existing mapping": + + - skip: + version: " - 7.99.99" + reason: Runtime mappings support was added in 8.0 + + - do: + index: + index: test-2 + body: { day_of_week: 123 } + + - do: + field_caps: + index: test-* + fields: "day*" + + - match: {indices: ["test-1", "test-2"]} + - length: {fields.day_of_week: 1} + - match: {fields.day_of_week.long.type: long} + - match: {fields.day_of_week.long.searchable: true} + - match: {fields.day_of_week.long.aggregatable: true} + + - do: + field_caps: + index: test-* + fields: "day*" + body: + runtime_mappings: + day_of_week: + type: keyword + script: + source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" + + - match: {indices: ["test-1", "test-2"]} + - length: {fields.day_of_week: 1} + - match: {fields.day_of_week.keyword.type: keyword} + - match: {fields.day_of_week.keyword.searchable: true} + - match: {fields.day_of_week.keyword.aggregatable: true}