Add runtime field section to Field Capabilities API (#68904)

docs/reference/search/field-caps.asciidoc

@@ -88,6 +88,13 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=index-ignore-unavailab
 (Optional,  <<query-dsl,query object>> Allows to filter indices if the provided
 query rewrites to `match_none` on every shard.
 
+`runtime_mappings`::
+(Optional, object)
+Defines ad-hoc <<runtime-search-request,runtime fields>> in the request similar
+to the way it is done in <<search-api-body-runtime, search requests>>. These fields
+exist only as part of the query and take precedence over fields defined with the
+same name in the index mappings.
+
 [[search-field-caps-api-response-body]]
 ==== {api-response-body-title}
 

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesIndexRequest.java

@@ -20,6 +20,8 @@
 import org.elasticsearch.index.shard.ShardId;
 
 import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
 import java.util.Objects;
 
 public class FieldCapabilitiesIndexRequest extends ActionRequest implements IndicesRequest {
@@ -31,6 +33,7 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi
     private final OriginalIndices originalIndices;
     private final QueryBuilder indexFilter;
     private final long nowInMillis;
+    private Map<String, Object> runtimeFields;
 
     private ShardId shardId;
 
@@ -47,13 +50,15 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi
         }
         indexFilter = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalNamedWriteable(QueryBuilder.class) : null;
         nowInMillis =  in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readLong() : 0L;
+        runtimeFields = in.getVersion().onOrAfter(Version.V_7_12_0) ? in.readMap() : Collections.emptyMap();
     }
 
     FieldCapabilitiesIndexRequest(String[] fields,
                                   String index,
                                   OriginalIndices originalIndices,
                                   QueryBuilder indexFilter,
-                                  long nowInMillis) {
+                                  long nowInMillis,
+                                  Map<String, Object> runtimeFields) {
         if (fields == null || fields.length == 0) {
             throw new IllegalArgumentException("specified fields can't be null or empty");
         }
@@ -62,6 +67,7 @@ public class FieldCapabilitiesIndexRequest extends ActionRequest implements Indi
         this.originalIndices = originalIndices;
         this.indexFilter = indexFilter;
         this.nowInMillis = nowInMillis;
+        this.runtimeFields = runtimeFields;
     }
 
     public String[] fields() {
@@ -86,6 +92,10 @@ public QueryBuilder indexFilter() {
         return indexFilter;
     }
 
+    public Map<String, Object> runtimeFields() {
+        return runtimeFields;
+    }
+
     public ShardId shardId() {
         return shardId;
     }
@@ -112,6 +122,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalNamedWriteable(indexFilter);
             out.writeLong(nowInMillis);
         }
+        if (out.getVersion().onOrAfter(Version.V_7_12_0)) {
+            out.writeMap(runtimeFields);
+        }
     }
 
     @Override

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

@@ -23,7 +23,9 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 
@@ -37,6 +39,7 @@ public final class FieldCapabilitiesRequest extends ActionRequest implements Ind
     // pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged
     private boolean mergeResults = true;
     private QueryBuilder indexFilter;
+    private Map<String, Object> runtimeFields = Collections.emptyMap();
     private Long nowInMillis;
 
     public FieldCapabilitiesRequest(StreamInput in) throws IOException {
@@ -52,6 +55,7 @@ public FieldCapabilitiesRequest(StreamInput in) throws IOException {
         }
         indexFilter = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalNamedWriteable(QueryBuilder.class) : null;
         nowInMillis = in.getVersion().onOrAfter(Version.V_7_9_0) ? in.readOptionalLong() : null;
+        runtimeFields = in.getVersion().onOrAfter(Version.V_7_12_0) ? in.readMap() : Collections.emptyMap();
     }
 
     public FieldCapabilitiesRequest() {
@@ -90,6 +94,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalNamedWriteable(indexFilter);
             out.writeOptionalLong(nowInMillis);
         }
+        if (out.getVersion().onOrAfter(Version.V_7_12_0)) {
+            out.writeMap(runtimeFields);
+        }
     }
 
     @Override
@@ -98,6 +105,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (indexFilter != null) {
             builder.field("index_filter", indexFilter);
         }
+        if (runtimeFields.isEmpty() == false) {
+            builder.field("runtime_mappings", runtimeFields);
+        }
         builder.endObject();
         return builder;
     }
@@ -121,6 +131,7 @@ public String[] fields() {
     /**
      * The list of indices to lookup
      */
+    @Override
     public FieldCapabilitiesRequest indices(String... indices) {
         this.indices = Objects.requireNonNull(indices, "indices must not be null");
         return this;
@@ -166,6 +177,17 @@ public FieldCapabilitiesRequest indexFilter(QueryBuilder indexFilter) {
     public QueryBuilder indexFilter() {
         return indexFilter;
     }
+    /**
+     * Allows adding search runtime fields if provided.
+     */
+    public FieldCapabilitiesRequest runtimeFields(Map<String, Object> runtimeFieldsSection) {
+        this.runtimeFields = runtimeFieldsSection;
+        return this;
+    }
+
+    public Map<String, Object> runtimeFields() {
+        return this.runtimeFields;
+    }
 
     Long nowInMillis() {
         return nowInMillis;
@@ -195,12 +217,13 @@ public boolean equals(Object o) {
             indicesOptions.equals(that.indicesOptions) &&
             Arrays.equals(fields, that.fields) &&
             Objects.equals(indexFilter, that.indexFilter) &&
-            Objects.equals(nowInMillis, that.nowInMillis);
+            Objects.equals(nowInMillis, that.nowInMillis) &&
+            Objects.equals(runtimeFields, that.runtimeFields);
     }
 
     @Override
     public int hashCode() {
-        int result = Objects.hash(indicesOptions, includeUnmapped, mergeResults, indexFilter, nowInMillis);
+        int result = Objects.hash(indicesOptions, includeUnmapped, mergeResults, indexFilter, nowInMillis, runtimeFields);
         result = 31 * result + Arrays.hashCode(indices);
         result = 31 * result + Arrays.hashCode(fields);
         return result;

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java

@@ -101,7 +101,7 @@ public void onFailure(Exception e) {
             };
             for (String index : concreteIndices) {
                 shardAction.execute(new FieldCapabilitiesIndexRequest(request.fields(), index, localIndices,
-                    request.indexFilter(), nowInMillis), innerListener);
+                    request.indexFilter(), nowInMillis, request.runtimeFields()), innerListener);
             }
 
             // this is the cross cluster part of this API - we force the other cluster to not merge the results but instead

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesIndexAction.java

@@ -107,7 +107,7 @@ private FieldCapabilitiesIndexResponse shardOperation(final FieldCapabilitiesInd
         try (Engine.Searcher searcher = indexShard.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE)) {
 
             final SearchExecutionContext searchExecutionContext = indexService.newSearchExecutionContext(shardId.id(), 0,
-                searcher, request::nowInMillis, null, Collections.emptyMap());
+                searcher, request::nowInMillis, null, request.runtimeFields());
 
             if (canMatchShard(request, searchExecutionContext) == false) {
                 return new FieldCapabilitiesIndexResponse(request.index(), Collections.emptyMap(), false);

server/src/main/java/org/elasticsearch/rest/action/RestFieldCapabilitiesAction.java

@@ -11,7 +11,9 @@
 import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestRequest;
 
@@ -20,6 +22,7 @@
 
 import static java.util.Arrays.asList;
 import static java.util.Collections.unmodifiableList;
+import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
 
@@ -52,9 +55,19 @@ public RestChannelConsumer prepareRequest(final RestRequest request,
         fieldRequest.includeUnmapped(request.paramAsBoolean("include_unmapped", false));
         request.withContentOrSourceParamParserOrNull(parser -> {
             if (parser != null) {
-                fieldRequest.indexFilter(RestActions.getQueryContent("index_filter", parser));
+                PARSER.parse(parser, fieldRequest, null);
             }
         });
         return channel -> client.fieldCaps(fieldRequest, new RestToXContentListener<>(channel));
     }
+
+    private static ParseField INDEX_FILTER_FIELD = new ParseField("index_filter");
+    private static ParseField RUNTIME_MAPPINGS_FIELD = new ParseField("runtime_mappings");
+
+    private static final ObjectParser<FieldCapabilitiesRequest, Void> PARSER = new ObjectParser<>("field_caps_request");
+
+    static {
+        PARSER.declareObject(FieldCapabilitiesRequest::indexFilter, (p, c) -> parseInnerQueryBuilder(p), INDEX_FILTER_FIELD);
+        PARSER.declareObject(FieldCapabilitiesRequest::runtimeFields, (p, c) -> p.map(), RUNTIME_MAPPINGS_FIELD);
+    }
 }

server/src/test/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestTests.java

@@ -10,15 +10,27 @@
 
 import org.elasticsearch.action.ActionRequestValidationException;
 import org.elasticsearch.action.support.IndicesOptions;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.ArrayUtils;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.SearchModule;
 import org.elasticsearch.test.AbstractWireSerializingTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.function.Consumer;
 
+import static java.util.Collections.singletonMap;
+
 public class FieldCapabilitiesRequestTests extends AbstractWireSerializingTestCase<FieldCapabilitiesRequest> {
 
     @Override
@@ -41,9 +53,24 @@ protected FieldCapabilitiesRequest createTestInstance() {
             request.indicesOptions(randomBoolean() ? IndicesOptions.strictExpand() : IndicesOptions.lenientExpandOpen());
         }
         request.includeUnmapped(randomBoolean());
+        if (randomBoolean()) {
+            request.nowInMillis(randomLong());
+        }
+        if (randomBoolean()) {
+            request.indexFilter(QueryBuilders.termQuery("field", randomAlphaOfLength(5)));
+        }
+        if (randomBoolean()) {
+            request.runtimeFields(Collections.singletonMap(randomAlphaOfLength(5), randomAlphaOfLength(5)));
+        }
         return request;
     }
 
+    @Override
+    protected NamedWriteableRegistry getNamedWriteableRegistry() {
+        SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
+        return new NamedWriteableRegistry(searchModule.getNamedWriteables());
+    }
+
     @Override
     protected Writeable.Reader<FieldCapabilitiesRequest> instanceReader() {
         return FieldCapabilitiesRequest::new;
@@ -67,13 +94,44 @@ protected FieldCapabilitiesRequest mutateInstance(FieldCapabilitiesRequest insta
         });
         mutators.add(request -> request.setMergeResults(request.isMergeResults() == false));
         mutators.add(request -> request.includeUnmapped(request.includeUnmapped() == false));
+        mutators.add(request -> request.nowInMillis(request.nowInMillis() != null ? request.nowInMillis() + 1 : 1L));
+        mutators.add(
+            request -> request.indexFilter(request.indexFilter() != null ? request.indexFilter().boost(2) : QueryBuilders.matchAllQuery())
+        );
+        mutators.add(request -> request.runtimeFields(Collections.singletonMap("other_key", "other_value")));
 
         FieldCapabilitiesRequest mutatedInstance = copyInstance(instance);
         Consumer<FieldCapabilitiesRequest> mutator = randomFrom(mutators);
         mutator.accept(mutatedInstance);
         return mutatedInstance;
     }
 
+    public void testToXContent() throws IOException {
+        FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
+        request.indexFilter(QueryBuilders.termQuery("field", "value"));
+        request.runtimeFields(singletonMap("day_of_week", singletonMap("type", "keyword")));
+        XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
+        String xContent = BytesReference.bytes(request.toXContent(builder, ToXContent.EMPTY_PARAMS)).utf8ToString();
+        assertEquals(
+            ("{"
+                + "  \"index_filter\": {\n"
+                + "    \"term\": {\n"
+                + "      \"field\": {\n"
+                + "        \"value\": \"value\",\n"
+                + "        \"boost\": 1.0\n"
+                + "      }\n"
+                + "    }\n"
+                + "  },\n"
+                + "  \"runtime_mappings\": {\n"
+                + "    \"day_of_week\": {\n"
+                + "      \"type\": \"keyword\"\n"
+                + "    }\n"
+                + "  }\n"
+                + "}").replaceAll("\\s+", ""),
+            xContent
+        );
+    }
+
     public void testValidation() {
         FieldCapabilitiesRequest request = new FieldCapabilitiesRequest()
             .indices("index2");