Introduce runtime section in mappings

server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java

@@ -155,6 +155,8 @@ private static Map<String, FieldMappingMetadata> findFieldMappings(Predicate<Str
         if (documentMapper == null) {
             return Collections.emptyMap();
         }
+        //TODO the logic here needs to be reworked to also include runtime fields. Though matching is against mappers rather
+        // than field types, and runtime fields are mixed with ordinary fields in FieldTypeLookup
         Map<String, FieldMappingMetadata> fieldMappings = new HashMap<>();
         final MappingLookup mappingLookup = documentMapper.mappers();
         for (String field : request.fields()) {

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -278,15 +278,7 @@ public final FieldMapper merge(Mapper mergeWith) {
         Conflicts conflicts = new Conflicts(name());
         builder.merge((FieldMapper) mergeWith, conflicts);
         conflicts.check();
-        return builder.build(parentPath(name()));
-    }
-
-    private static ContentPath parentPath(String name) {
-        int endPos = name.lastIndexOf(".");
-        if (endPos == -1) {
-            return new ContentPath(0);
-        }
-        return new ContentPath(name.substring(0, endPos));
+        return builder.build(Builder.parentPath(name()));
     }
 
     protected void checkIncomingMergeType(FieldMapper mergeWith) {
@@ -482,7 +474,7 @@ public List<String> copyToFields() {
     /**
      * Serializes a parameter
      */
-    protected interface Serializer<T> {
+    public interface Serializer<T> {
         void serialize(XContentBuilder builder, String name, T value) throws IOException;
     }
 
@@ -931,7 +923,7 @@ protected String buildFullName(ContentPath contentPath) {
         /**
          * Writes the current builder parameter values as XContent
          */
-        protected final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException {
+        public final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException {
             for (Parameter<?> parameter : getParameters()) {
                 parameter.toXContent(builder, includeDefaults);
             }
@@ -1011,6 +1003,14 @@ public final void parse(String name, ParserContext parserContext, Map<String, Ob
             validate();
         }
 
+        protected static ContentPath parentPath(String name) {
+            int endPos = name.lastIndexOf(".");
+            if (endPos == -1) {
+                return new ContentPath(0);
+            }
+            return new ContentPath(name.substring(0, endPos));
+        }
+
         // These parameters were previously *always* parsed by TypeParsers#parseField(), even if they
         // made no sense; if we've got here, that means that they're not declared on a current mapper,
         // and so we emit a deprecation warning rather than failing a previously working mapping.

server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java

@@ -32,6 +32,7 @@
 /**
  * An immutable container for looking up {@link MappedFieldType}s by their name.
  */
+//TODO Does this need to implement Iterable? It's used in two places
 final class FieldTypeLookup implements Iterable<MappedFieldType> {
 
     private final Map<String, MappedFieldType> fullNameToFieldType = new HashMap<>();
@@ -46,8 +47,10 @@ final class FieldTypeLookup implements Iterable<MappedFieldType> {
     private final Map<String, Set<String>> fieldToCopiedFields = new HashMap<>();
     private final DynamicKeyFieldTypeLookup dynamicKeyLookup;
 
+    //TODO ideally the constructor would not take mappers, but MappedFieldTypes and an external method does the conversion
     FieldTypeLookup(Collection<FieldMapper> fieldMappers,
-                    Collection<FieldAliasMapper> fieldAliasMappers) {
+                    Collection<FieldAliasMapper> fieldAliasMappers,
+                    Collection<RuntimeFieldType> runtimeFieldTypes) {
         Map<String, DynamicKeyFieldMapper> dynamicKeyMappers = new HashMap<>();
 
         for (FieldMapper fieldMapper : fieldMappers) {
@@ -77,6 +80,11 @@ final class FieldTypeLookup implements Iterable<MappedFieldType> {
             fullNameToFieldType.put(aliasName, fullNameToFieldType.get(path));
         }
 
+        for (RuntimeFieldType runtimeFieldType : runtimeFieldTypes) {
+            //this will override concrete fields with runtime fields that have the same name
+            fullNameToFieldType.put(runtimeFieldType.name(), runtimeFieldType);
+        }
+
         this.dynamicKeyLookup = new DynamicKeyFieldTypeLookup(dynamicKeyMappers, aliasToConcreteName);
     }
 

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -59,6 +59,7 @@ class ParserContext {
 
             private final Function<String, SimilarityProvider> similarityLookupService;
             private final Function<String, TypeParser> typeParsers;
+            private final Function<String, RuntimeFieldType.Parser> runtimeTypeParsers;
             private final Version indexVersionCreated;
             private final Supplier<QueryShardContext> queryShardContextSupplier;
             private final DateFormatter dateFormatter;
@@ -69,6 +70,7 @@ class ParserContext {
 
             public ParserContext(Function<String, SimilarityProvider> similarityLookupService,
                                  Function<String, TypeParser> typeParsers,
+                                 Function<String, RuntimeFieldType.Parser> runtimeTypeParsers,
                                  Version indexVersionCreated,
                                  Supplier<QueryShardContext> queryShardContextSupplier,
                                  DateFormatter dateFormatter,
@@ -78,6 +80,7 @@ public ParserContext(Function<String, SimilarityProvider> similarityLookupServic
                                  BooleanSupplier idFieldDataEnabled) {
                 this.similarityLookupService = similarityLookupService;
                 this.typeParsers = typeParsers;
+                this.runtimeTypeParsers = runtimeTypeParsers;
                 this.indexVersionCreated = indexVersionCreated;
                 this.queryShardContextSupplier = queryShardContextSupplier;
                 this.dateFormatter = dateFormatter;
@@ -132,6 +135,8 @@ public DateFormatter getDateFormatter() {
 
             protected Function<String, TypeParser> typeParsers() { return typeParsers; }
 
+            protected Function<String, RuntimeFieldType.Parser> runtimeTypeParsers() { return runtimeTypeParsers; }
+
             protected Function<String, SimilarityProvider> similarityLookupService() { return similarityLookupService; }
 
             /**
@@ -147,8 +152,9 @@ public ParserContext createMultiFieldContext(ParserContext in) {
 
             static class MultiFieldParserContext extends ParserContext {
                 MultiFieldParserContext(ParserContext in) {
-                    super(in.similarityLookupService, in.typeParsers, in.indexVersionCreated, in.queryShardContextSupplier,
-                        in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings, in.idFieldDataEnabled);
+                    super(in.similarityLookupService, in.typeParsers, in.runtimeTypeParsers, in.indexVersionCreated,
+                        in.queryShardContextSupplier, in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings,
+                        in.idFieldDataEnabled);
                 }
 
                 @Override

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -125,8 +125,8 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers,
         this.mapperRegistry = mapperRegistry;
         Function<DateFormatter, Mapper.TypeParser.ParserContext> parserContextFunction =
             dateFormatter -> new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperRegistry.getMapperParsers()::get,
-                indexVersionCreated, queryShardContextSupplier, dateFormatter, scriptService, indexAnalyzers, indexSettings,
-                idFieldDataEnabled);
+                mapperRegistry.getRuntimeFieldTypeParsers()::get, indexVersionCreated, queryShardContextSupplier, dateFormatter,
+                scriptService, indexAnalyzers, indexSettings, idFieldDataEnabled);
         this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction);
         Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers =
             mapperRegistry.getMetadataMapperParsers(indexSettings.getIndexVersionCreated());

server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java

@@ -32,7 +32,6 @@
 import java.util.stream.Stream;
 
 public final class MappingLookup {
-
     /** Full field name to mapper */
     private final Map<String, Mapper> fieldMappers;
     private final Map<String, ObjectMapper> objectMappers;
@@ -50,24 +49,24 @@ public static MappingLookup fromMapping(Mapping mapping) {
                 newFieldMappers.add(metadataMapper);
             }
         }
-        collect(mapping.root, newObjectMappers, newFieldMappers, newFieldAliasMappers);
-        return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length);
+        for (Mapper child : mapping.root) {
+            collect(child, newObjectMappers, newFieldMappers, newFieldAliasMappers);
+        }
+        return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers,
+            mapping.root.runtimeFieldTypes(), mapping.metadataMappers.length);
     }
 
     private static void collect(Mapper mapper, Collection<ObjectMapper> objectMappers,
                                Collection<FieldMapper> fieldMappers,
                                Collection<FieldAliasMapper> fieldAliasMappers) {
-        if (mapper instanceof RootObjectMapper) {
-            // root mapper isn't really an object mapper
-        } else if (mapper instanceof ObjectMapper) {
+        if (mapper instanceof ObjectMapper) {
             objectMappers.add((ObjectMapper)mapper);
         } else if (mapper instanceof FieldMapper) {
             fieldMappers.add((FieldMapper)mapper);
         } else if (mapper instanceof FieldAliasMapper) {
             fieldAliasMappers.add((FieldAliasMapper) mapper);
         } else {
-            throw new IllegalStateException("Unrecognized mapper type [" +
-                mapper.getClass().getSimpleName() + "].");
+            throw new IllegalStateException("Unrecognized mapper type [" + mapper.getClass().getSimpleName() + "].");
         }
 
         for (Mapper child : mapper) {
@@ -78,6 +77,7 @@ private static void collect(Mapper mapper, Collection<ObjectMapper> objectMapper
     public MappingLookup(Collection<FieldMapper> mappers,
                          Collection<ObjectMapper> objectMappers,
                          Collection<FieldAliasMapper> aliasMappers,
+                         Collection<RuntimeFieldType> runtimeFieldTypes,
                          int metadataFieldCount) {
         Map<String, Mapper> fieldMappers = new HashMap<>();
         Map<String, Analyzer> indexAnalyzers = new HashMap<>();
@@ -114,7 +114,7 @@ public MappingLookup(Collection<FieldMapper> mappers,
             }
         }
 
-        this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers);
+        this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers, runtimeFieldTypes);
 
         this.fieldMappers = Collections.unmodifiableMap(fieldMappers);
         this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers);

server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java

@@ -19,6 +19,7 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.Strings;
@@ -33,11 +34,14 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
 import static org.elasticsearch.index.mapper.TypeParsers.parseDateTimeFormatter;
@@ -61,6 +65,7 @@ public static class Builder extends ObjectMapper.Builder {
         protected Explicit<DateFormatter[]> dynamicDateTimeFormatters = new Explicit<>(Defaults.DYNAMIC_DATE_TIME_FORMATTERS, false);
         protected Explicit<Boolean> dateDetection = new Explicit<>(Defaults.DATE_DETECTION, false);
         protected Explicit<Boolean> numericDetection = new Explicit<>(Defaults.NUMERIC_DETECTION, false);
+        protected final Map<String, RuntimeFieldType> runtimeFieldTypes = new HashMap<>();
 
         public Builder(String name, Version indexCreatedVersion) {
             super(name, indexCreatedVersion);
@@ -82,6 +87,11 @@ public RootObjectMapper.Builder add(Mapper.Builder builder) {
             return this;
         }
 
+        public RootObjectMapper.Builder addRuntime(RuntimeFieldType runtimeFieldType) {
+            this.runtimeFieldTypes.put(runtimeFieldType.name(), runtimeFieldType);
+            return this;
+        }
+
         @Override
         public RootObjectMapper build(ContentPath contentPath) {
             return (RootObjectMapper) super.build(contentPath);
@@ -91,7 +101,7 @@ public RootObjectMapper build(ContentPath contentPath) {
         protected ObjectMapper createMapper(String name, String fullPath, Explicit<Boolean> enabled, Nested nested, Dynamic dynamic,
                 Map<String, Mapper> mappers, Version indexCreatedVersion) {
             assert !nested.isNested();
-            return new RootObjectMapper(name, enabled, dynamic, mappers,
+            return new RootObjectMapper(name, enabled, dynamic, mappers, runtimeFieldTypes,
                     dynamicDateTimeFormatters,
                     dynamicTemplates,
                     dateDetection, numericDetection, indexCreatedVersion);
@@ -126,7 +136,7 @@ private static void fixRedundantIncludes(ObjectMapper objectMapper, boolean pare
         }
     }
 
-    public static class TypeParser extends ObjectMapper.TypeParser {
+    static final class TypeParser extends ObjectMapper.TypeParser {
 
         @Override
         public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
@@ -145,8 +155,7 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
         }
 
         @SuppressWarnings("unchecked")
-        protected boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode,
-                                       ParserContext parserContext) {
+        private boolean processField(RootObjectMapper.Builder builder, String fieldName, Object fieldNode, ParserContext parserContext) {
             if (fieldName.equals("date_formats") || fieldName.equals("dynamic_date_formats")) {
                 if (fieldNode instanceof List) {
                     List<DateFormatter> formatters = new ArrayList<>();
@@ -189,10 +198,8 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam
                     String templateName = entry.getKey();
                     Map<String, Object> templateParams = (Map<String, Object>) entry.getValue();
                     DynamicTemplate template = DynamicTemplate.parse(templateName, templateParams);
-                    if (template != null) {
-                        validateDynamicTemplate(parserContext, template);
-                        templates.add(template);
-                    }
+                    validateDynamicTemplate(parserContext, template);
+                    templates.add(template);
                 }
                 builder.dynamicTemplates(templates);
                 return true;
@@ -202,6 +209,13 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam
             } else if (fieldName.equals("numeric_detection")) {
                 builder.numericDetection = new Explicit<>(nodeBooleanValue(fieldNode, "numeric_detection"), true);
                 return true;
+            } else if (fieldName.equals("runtime")) {
+                if (fieldNode instanceof Map) {
+                    RuntimeFieldType.parseRuntimeFields((Map<String, Object>) fieldNode, parserContext, builder::addRuntime);
+                    return true;
+                } else {
+                    throw new ElasticsearchParseException("runtime must be a map type");
+                }
             }
             return false;
         }
@@ -211,11 +225,14 @@ protected boolean processField(RootObjectMapper.Builder builder, String fieldNam
     private Explicit<Boolean> dateDetection;
     private Explicit<Boolean> numericDetection;
     private Explicit<DynamicTemplate[]> dynamicTemplates;
+    private final Map<String, RuntimeFieldType> runtimeFieldTypes;
 
     RootObjectMapper(String name, Explicit<Boolean> enabled, Dynamic dynamic, Map<String, Mapper> mappers,
+                     Map<String, RuntimeFieldType> runtimeFieldTypes,
                      Explicit<DateFormatter[]> dynamicDateTimeFormatters, Explicit<DynamicTemplate[]> dynamicTemplates,
                      Explicit<Boolean> dateDetection, Explicit<Boolean> numericDetection, Version indexCreatedVersion) {
         super(name, name, enabled, Nested.NO, dynamic, mappers, indexCreatedVersion);
+        this.runtimeFieldTypes = runtimeFieldTypes;
         this.dynamicTemplates = dynamicTemplates;
         this.dynamicDateTimeFormatters = dynamicDateTimeFormatters;
         this.dateDetection = dateDetection;
@@ -235,23 +252,26 @@ public ObjectMapper mappingUpdate(Mapper mapper) {
         return update;
     }
 
-    public boolean dateDetection() {
+    boolean dateDetection() {
         return this.dateDetection.value();
     }
 
-    public boolean numericDetection() {
+    boolean numericDetection() {
         return this.numericDetection.value();
     }
 
-    public DateFormatter[] dynamicDateTimeFormatters() {
+    DateFormatter[] dynamicDateTimeFormatters() {
         return dynamicDateTimeFormatters.value();
     }
 
-    public DynamicTemplate[] dynamicTemplates() {
+    DynamicTemplate[] dynamicTemplates() {
         return dynamicTemplates.value();
     }
 
-    @SuppressWarnings("rawtypes")
+    Collection<RuntimeFieldType> runtimeFieldTypes() {
+        return runtimeFieldTypes.values();
+    }
+
     public Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType) {
         return findTemplateBuilder(context, name, matchType, null);
     }
@@ -267,7 +287,6 @@ public Mapper.Builder findTemplateBuilder(ParseContext context, String name, Dat
      * @param dateFormat  a dateformatter to use if the type is a date, null if not a date or is using the default format
      * @return a mapper builder, or null if there is no template for such a field
      */
-    @SuppressWarnings("rawtypes")
     private Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType, DateFormatter dateFormat) {
         DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType);
         if (dynamicTemplate == null) {
@@ -330,6 +349,8 @@ protected void doMerge(ObjectMapper mergeWith, MergeReason reason) {
                 this.dynamicTemplates = mergeWithObject.dynamicTemplates;
             }
         }
+
+        this.runtimeFieldTypes.putAll(mergeWithObject.runtimeFieldTypes);
     }
 
     @Override
@@ -360,6 +381,16 @@ protected void doXContent(XContentBuilder builder, ToXContent.Params params) thr
         if (numericDetection.explicit() || includeDefaults) {
             builder.field("numeric_detection", numericDetection.value());
         }
+
+        if (runtimeFieldTypes.size() > 0) {
+            builder.startObject("runtime");
+            List<RuntimeFieldType> sortedRuntimeFieldTypes = runtimeFieldTypes.values().stream().sorted(
+                Comparator.comparing(RuntimeFieldType::name)).collect(Collectors.toList());
+            for (RuntimeFieldType fieldType : sortedRuntimeFieldTypes) {
+                fieldType.toXContent(builder, params);
+            }
+            builder.endObject();
+        }
     }
 
     private static void validateDynamicTemplate(Mapper.TypeParser.ParserContext parserContext,