Teach _search to read search time mappings

server/src/main/java/org/elasticsearch/index/IndexService.java

@@ -584,10 +584,27 @@ public IndexSettings getIndexSettings() {
      * {@link IndexReader}-specific optimizations, such as rewriting containing range queries.
      */
     public QueryShardContext newQueryShardContext(int shardId, IndexSearcher searcher, LongSupplier nowInMillis, String clusterAlias) {
+        return newQueryShardContext(shardId, searcher, nowInMillis, clusterAlias, null);
+    }
+
+    /**
+     * Creates a new QueryShardContext.
+     *
+     * Passing a {@code null} {@link IndexSearcher} will return a valid context, however it won't be able to make
+     * {@link IndexReader}-specific optimizations, such as rewriting containing range queries.
+     */
+    public QueryShardContext newQueryShardContext(
+        int shardId,
+        IndexSearcher searcher,
+        LongSupplier nowInMillis,
+        String clusterAlias,
+        Map<String, Object> runtimeMappings
+    ) {
         final SearchIndexNameMatcher indexNameMatcher =
             new SearchIndexNameMatcher(index().getName(), clusterAlias, clusterService, expressionResolver);
         return new QueryShardContext(
-            shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField, mapperService(),
+            shardId, indexSettings, bigArrays, indexCache.bitsetFilterCache(), indexFieldData::getForField,
+            mapperService().forSearch(runtimeMappings),
             similarityService(), scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, clusterAlias,
             indexNameMatcher, allowExpensiveQueries, valuesSourceRegistry);
     }

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -448,6 +448,14 @@ protected final void doXContentAnalyzers(XContentBuilder builder, boolean includ
         }
     }
 
+    /**
+     * Called when this {@linkplain Mapper} is parsed on the {@code _search}
+     * request to check if this field can be a runtime field. 
+     */
+    public boolean isRuntimeField() {
+        return false;
+    }
+
     protected static String indexOptionToString(IndexOptions indexOption) {
         switch (indexOption) {
             case DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS:

server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java

@@ -35,8 +35,8 @@
  */
 class FieldTypeLookup implements Iterable<MappedFieldType> {
 
-    private final Map<String, MappedFieldType> fullNameToFieldType = new HashMap<>();
-    private final Map<String, String> aliasToConcreteName = new HashMap<>();
+    private final Map<String, MappedFieldType> fullNameToFieldType;
+    private final Map<String, String> aliasToConcreteName;
     private final DynamicKeyFieldTypeLookup dynamicKeyLookup;
 
     FieldTypeLookup() {
@@ -48,6 +48,7 @@ class FieldTypeLookup implements Iterable<MappedFieldType> {
 
         Map<String, DynamicKeyFieldMapper> dynamicKeyMappers = new HashMap<>();
 
+        fullNameToFieldType = new HashMap<>(fieldMappers.size());
         for (FieldMapper fieldMapper : fieldMappers) {
             String fieldName = fieldMapper.name();
             MappedFieldType fieldType = fieldMapper.fieldType();
@@ -57,6 +58,7 @@ class FieldTypeLookup implements Iterable<MappedFieldType> {
             }
         }
 
+        aliasToConcreteName = new HashMap<>(fieldAliasMappers.size());
         for (FieldAliasMapper fieldAliasMapper : fieldAliasMappers) {
             String aliasName = fieldAliasMapper.name();
             String path = fieldAliasMapper.path();
@@ -66,6 +68,16 @@ class FieldTypeLookup implements Iterable<MappedFieldType> {
         this.dynamicKeyLookup = new DynamicKeyFieldTypeLookup(dynamicKeyMappers, aliasToConcreteName);
     }
 
+    private FieldTypeLookup(
+        Map<String, MappedFieldType> fullNameToFieldType,
+        Map<String, String> aliasToConcreteName,
+        DynamicKeyFieldTypeLookup dynamicKeyLookup
+    ) {
+        this.fullNameToFieldType = fullNameToFieldType;
+        this.aliasToConcreteName = aliasToConcreteName;
+        this.dynamicKeyLookup = dynamicKeyLookup;
+    }
+
     /**
      * Returns the mapped field type for the given field name.
      */
@@ -105,4 +117,27 @@ public Iterator<MappedFieldType> iterator() {
         Iterator<MappedFieldType> keyedFieldTypes = dynamicKeyLookup.fieldTypes();
         return Iterators.concat(concreteFieldTypes, keyedFieldTypes);
     }
+
+    /**
+     * Returns a copy of this lookup with runtime mappings merged into it.
+     */
+    public FieldTypeLookup withRuntimeMappings(Collection<FieldMapper> runtimeMappings) {
+        Map<String, MappedFieldType> mappers = new HashMap<>(fullNameToFieldType.size() + runtimeMappings.size());
+        mappers.putAll(fullNameToFieldType);
+        for (FieldMapper fm : runtimeMappings) {
+            if (false == fm.isRuntimeField()) {
+                throw new IllegalArgumentException(
+                    "[" + fm.typeName() + "] are not supported in runtime mappings"
+                );
+            }
+            MappedFieldType fromIndexMapping = fullNameToFieldType.get(fm.name());
+            if (fromIndexMapping != null) {
+                throw new IllegalArgumentException(
+                    "[" + fm.name() + "] can't be defined in the search's runtime mappings and the index's mappings"
+                );
+            }
+            mappers.put(fm.name(), fm.fieldType());
+        }
+        return new FieldTypeLookup(mappers, aliasToConcreteName, dynamicKeyLookup);
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -150,6 +150,38 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers,
         this.idFieldDataEnabled = idFieldDataEnabled;
     }
 
+    private MapperService(
+        IndexSettings indexSettings,
+        IndexAnalyzers indexAnalyzers,
+        DocumentMapper mapper,
+        FieldTypeLookup fieldTypes,
+        Map<String, ObjectMapper> fullPathObjectMappers,
+        boolean hasNested,
+        DocumentMapperParser documentParser,
+        Version indexVersionCreated,
+        MapperAnalyzerWrapper indexAnalyzer,
+        MapperAnalyzerWrapper searchAnalyzer,
+        MapperAnalyzerWrapper searchQuoteAnalyzer,
+        Map<String, MappedFieldType> unmappedFieldTypes,
+        MapperRegistry mapperRegistry,
+        BooleanSupplier idFieldDataEnabled
+    ) {
+        super(indexSettings);
+        this.indexAnalyzers = indexAnalyzers;
+        this.mapper = mapper;
+        this.fieldTypes = fieldTypes;
+        this.fullPathObjectMappers = fullPathObjectMappers;
+        this.hasNested = hasNested;
+        this.documentParser = documentParser;
+        this.indexVersionCreated = indexVersionCreated;
+        this.indexAnalyzer = indexAnalyzer;
+        this.searchAnalyzer = searchAnalyzer;
+        this.searchQuoteAnalyzer = searchQuoteAnalyzer;
+        this.unmappedFieldTypes = unmappedFieldTypes;
+        this.mapperRegistry = mapperRegistry;
+        this.idFieldDataEnabled = idFieldDataEnabled;
+    }
+
     public boolean hasNested() {
         return this.hasNested;
     }
@@ -618,6 +650,59 @@ public Analyzer searchQuoteAnalyzer() {
         return this.searchQuoteAnalyzer;
     }
 
+    /**
+     * Builds a {@linkplain Function} to lookup mappers for a request, adding
+     * any {@code runtimeMappings} provided.
+     * @param runtimeMappings extra mappings parse and to add to the request
+     *        lookup or {@code null} if there aren't any extra mappings  
+     */
+    public MapperService forSearch(Map<String, Object> runtimeMappings) {
+        if (runtimeMappings == null || runtimeMappings.size() == 0) {
+            return this;
+        }
+        Mapper.BuilderContext builderContext = new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath(0));
+        Collection<ObjectMapper> objectMappers = new ArrayList<>();
+        Collection<FieldMapper> fieldMappers = new ArrayList<>();
+        Collection<FieldAliasMapper> fieldAliasMappers = new ArrayList<>();
+        for (Map.Entry<String, Object> runtimeEntry : runtimeMappings.entrySet()) {
+            @SuppressWarnings("unchecked") // Safe because that is how we deserialized it
+            Map<String, Object> definition = (Map<String, Object>) runtimeEntry.getValue();
+            String type = (String) definition.remove("type");
+            if (type == null) {
+                throw new IllegalArgumentException("[type] is required for runtime mapping [" + runtimeEntry.getKey() + "]");
+            }
+            Mapper.TypeParser parser = documentMapperParser().parserContext().typeParser(type);
+            if (parser == null) {
+                throw new IllegalArgumentException("[" + type + "] is unknown type for runtime mapping [" + runtimeEntry.getKey() + "]");
+            }
+            Mapper.Builder<?> builder = parser.parse(runtimeEntry.getKey(), definition, documentMapperParser().parserContext());
+            Mapper mapper = builder.build(builderContext);
+
+            // MapperUtils.collect will find the mappers declared in objectss
+            MapperUtils.collect(mapper, objectMappers, fieldMappers, fieldAliasMappers);
+
+        }
+        if (false == fieldAliasMappers.isEmpty()) {
+            throw new IllegalArgumentException("aliases are not supported in runtime mappings");
+        }
+        return new MapperService(
+            indexSettings,
+            indexAnalyzers,
+            mapper,
+            fieldTypes.withRuntimeMappings(fieldMappers),
+            fullPathObjectMappers,
+            hasNested,
+            documentParser,
+            indexVersionCreated,
+            indexAnalyzer,
+            searchAnalyzer,
+            searchQuoteAnalyzer,
+            unmappedFieldTypes,
+            mapperRegistry,
+            idFieldDataEnabled
+        );
+    }
+
     /**
      * Returns <code>true</code> if fielddata is enabled for the {@link IdFieldMapper} field, <code>false</code> otherwise.
      */

server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java

@@ -121,9 +121,9 @@ public QueryShardContext(int shardId,
                              BooleanSupplier allowExpensiveQueries,
                              ValuesSourceRegistry valuesSourceRegistry) {
         this(shardId, indexSettings, bigArrays, bitsetFilterCache, indexFieldDataLookup, mapperService, similarityService,
-                scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, indexNameMatcher,
-                new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()),
-                        indexSettings.getIndex().getUUID()), allowExpensiveQueries, valuesSourceRegistry);
+            scriptService, xContentRegistry, namedWriteableRegistry, client, searcher, nowInMillis, indexNameMatcher,
+            new Index(RemoteClusterAware.buildRemoteIndexName(clusterAlias, indexSettings.getIndex().getName()),
+                indexSettings.getIndex().getUUID()), allowExpensiveQueries, valuesSourceRegistry);
     }
 
     public QueryShardContext(QueryShardContext source) {

server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java

@@ -175,8 +175,9 @@ final class DefaultSearchContext extends SearchContext {
             engineSearcher.getQueryCache(), engineSearcher.getQueryCachingPolicy(), lowLevelCancellation);
         this.relativeTimeSupplier = relativeTimeSupplier;
         this.timeout = timeout;
+        Map<String, Object> runtimeMappings = request.source() == null ? null : request.source().runtimeMappings();
         queryShardContext = indexService.newQueryShardContext(request.shardId().id(), searcher,
-            request::nowInMillis, shardTarget.getClusterAlias());
+            request::nowInMillis, shardTarget.getClusterAlias(), runtimeMappings);
         queryBoost = request.indexBoost();
         this.lowLevelCancellation = lowLevelCancellation;
     }
@@ -466,7 +467,7 @@ public IndexShard indexShard() {
 
     @Override
     public MapperService mapperService() {
-        return indexService.mapperService();
+        return queryShardContext.getMapperService();
     }
 
     @Override
@@ -775,7 +776,7 @@ public FetchSearchResult fetchResult() {
 
     @Override
     public MappedFieldType fieldType(String name) {
-        return mapperService().fieldType(name);
+        return queryShardContext.fieldMapper(name);
     }
 
     @Override

server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.search.builder;
 
 import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Booleans;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
@@ -62,6 +63,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
@@ -107,6 +109,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R
     public static final ParseField SEARCH_AFTER = new ParseField("search_after");
     public static final ParseField COLLAPSE = new ParseField("collapse");
     public static final ParseField SLICE = new ParseField("slice");
+    public static final ParseField RUNTIME_MAPPINGS = new ParseField("runtime_mappings");
 
     public static SearchSourceBuilder fromXContent(XContentParser parser) throws IOException {
         return fromXContent(parser, true);
@@ -185,6 +188,8 @@ public static HighlightBuilder highlight() {
 
     private CollapseBuilder collapse = null;
 
+    private Map<String, Object> runtimeMappings; 
+
     /**
      * Constructs a new search source builder.
      */
@@ -239,6 +244,10 @@ public SearchSourceBuilder(StreamInput in) throws IOException {
         sliceBuilder = in.readOptionalWriteable(SliceBuilder::new);
         collapse = in.readOptionalWriteable(CollapseBuilder::new);
         trackTotalHitsUpTo = in.readOptionalInt();
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) {
+            // TODO update version after backporting runtime fields
+            runtimeMappings = in.readMap();
+        }
     }
 
     @Override
@@ -293,6 +302,16 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeOptionalWriteable(sliceBuilder);
         out.writeOptionalWriteable(collapse);
         out.writeOptionalInt(trackTotalHitsUpTo);
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) {
+            // TODO update version after backporting runtime fields
+            out.writeMap(runtimeMappings);
+        } else {
+            if (runtimeMappings != null && false == runtimeMappings.isEmpty()) {
+                throw new IllegalArgumentException(
+                    "[" + RUNTIME_MAPPINGS.getPreferredName() + "] are not supported on nodes older than 8.0.0"
+                );
+            }
+        }
     }
 
     /**
@@ -895,6 +914,21 @@ public List<String> stats() {
         return stats;
     }
 
+    /**
+     * Extra runtime field mappings.
+     */
+    public SearchSourceBuilder runtimeMappings(Map<String, Object> runtimeMappings) {
+        this.runtimeMappings = runtimeMappings;
+        return this;
+    }
+
+    /**
+     * Extra runtime field mappings.
+     */
+    public Map<String, Object> runtimeMappings() {
+        return runtimeMappings;
+    }
+
     public SearchSourceBuilder ext(List<SearchExtBuilder> searchExtBuilders) {
         this.extBuilders = Objects.requireNonNull(searchExtBuilders, "searchExtBuilders must not be null");
         return this;
@@ -996,6 +1030,7 @@ private SearchSourceBuilder shallowCopy(QueryBuilder queryBuilder, QueryBuilder
         rewrittenBuilder.version = version;
         rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm;
         rewrittenBuilder.collapse = collapse;
+        rewrittenBuilder.runtimeMappings = runtimeMappings;
         return rewrittenBuilder;
     }
 
@@ -1104,6 +1139,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th
                     sliceBuilder = SliceBuilder.fromXContent(parser);
                 } else if (COLLAPSE.match(currentFieldName, parser.getDeprecationHandler())) {
                     collapse = CollapseBuilder.fromXContent(parser);
+                } else if (RUNTIME_MAPPINGS.match(currentFieldName, parser.getDeprecationHandler())) {
+                    runtimeMappings = parser.map();
                 } else {
                     throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + currentFieldName + "].",
                             parser.getTokenLocation());
@@ -1300,6 +1337,10 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
         if (collapse != null) {
             builder.field(COLLAPSE.getPreferredName(), collapse);
         }
+
+        if (runtimeMappings != null && false == runtimeMappings.isEmpty()) {
+            builder.field(RUNTIME_MAPPINGS.getPreferredName(), runtimeMappings);
+        }
         return builder;
     }
 
@@ -1551,7 +1592,8 @@ public boolean equals(Object obj) {
                 && Objects.equals(profile, other.profile)
                 && Objects.equals(extBuilders, other.extBuilders)
                 && Objects.equals(collapse, other.collapse)
-                && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo);
+                && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo)
+                && Objects.equals(runtimeMappings, other.runtimeMappings);
     }
 
     @Override