Introduce runtime section in mappings

server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java

@@ -39,9 +39,9 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -155,6 +155,8 @@ private static Map<String, FieldMappingMetadata> findFieldMappings(Predicate<Str
         if (documentMapper == null) {
             return Collections.emptyMap();
         }
+        //TODO the logic here needs to be reworked to also include runtime fields. Though matching is against mappers rather
+        // than field types, and runtime fields are mixed with ordinary fields in FieldTypeLookup
         Map<String, FieldMappingMetadata> fieldMappings = new HashMap<>();
         final MappingLookup allFieldMappers = documentMapper.mappers();
         for (String field : request.fields()) {

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -278,15 +278,7 @@ public final FieldMapper merge(Mapper mergeWith) {
         Conflicts conflicts = new Conflicts(name());
         builder.merge((FieldMapper) mergeWith, conflicts);
         conflicts.check();
-        return builder.build(parentPath(name()));
-    }
-
-    private static ContentPath parentPath(String name) {
-        int endPos = name.lastIndexOf(".");
-        if (endPos == -1) {
-            return new ContentPath(0);
-        }
-        return new ContentPath(name.substring(0, endPos));
+        return builder.build(Builder.parentPath(name()));
     }
 
     protected void checkIncomingMergeType(FieldMapper mergeWith) {
@@ -482,7 +474,7 @@ public List<String> copyToFields() {
     /**
      * Serializes a parameter
      */
-    protected interface Serializer<T> {
+    public interface Serializer<T> {
         void serialize(XContentBuilder builder, String name, T value) throws IOException;
     }
 
@@ -931,7 +923,7 @@ protected String buildFullName(ContentPath contentPath) {
         /**
          * Writes the current builder parameter values as XContent
          */
-        protected final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException {
+        public final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException {
             for (Parameter<?> parameter : getParameters()) {
                 parameter.toXContent(builder, includeDefaults);
             }
@@ -1011,6 +1003,14 @@ public final void parse(String name, ParserContext parserContext, Map<String, Ob
             validate();
         }
 
+        protected static ContentPath parentPath(String name) {
+            int endPos = name.lastIndexOf(".");
+            if (endPos == -1) {
+                return new ContentPath(0);
+            }
+            return new ContentPath(name.substring(0, endPos));
+        }
+
         // These parameters were previously *always* parsed by TypeParsers#parseField(), even if they
         // made no sense; if we've got here, that means that they're not declared on a current mapper,
         // and so we emit a deprecation warning rather than failing a previously working mapping.

server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java

@@ -32,6 +32,7 @@
 /**
  * An immutable container for looking up {@link MappedFieldType}s by their name.
  */
+//TODO Does this need to implement Iterable? It's used in two places
 final class FieldTypeLookup implements Iterable<MappedFieldType> {
 
     private final Map<String, MappedFieldType> fullNameToFieldType = new HashMap<>();
@@ -46,8 +47,10 @@ final class FieldTypeLookup implements Iterable<MappedFieldType> {
     private final Map<String, Set<String>> fieldToCopiedFields = new HashMap<>();
     private final DynamicKeyFieldTypeLookup dynamicKeyLookup;
 
+    //TODO ideally the constructor would not take mappers, but MappedFieldTypes and an external method does the conversion
     FieldTypeLookup(Collection<FieldMapper> fieldMappers,
-                    Collection<FieldAliasMapper> fieldAliasMappers) {
+                    Collection<FieldAliasMapper> fieldAliasMappers,
+                    Collection<RuntimeFieldType> runtimeFieldTypes) {
         Map<String, DynamicKeyFieldMapper> dynamicKeyMappers = new HashMap<>();
 
         for (FieldMapper fieldMapper : fieldMappers) {
@@ -77,6 +80,11 @@ final class FieldTypeLookup implements Iterable<MappedFieldType> {
             fullNameToFieldType.put(aliasName, fullNameToFieldType.get(path));
         }
 
+        for (RuntimeFieldType runtimeFieldType : runtimeFieldTypes) {
+            //this will override concrete fields with runtime fields that have the same name
+            fullNameToFieldType.put(runtimeFieldType.name(), runtimeFieldType);
+        }
+
         this.dynamicKeyLookup = new DynamicKeyFieldTypeLookup(dynamicKeyMappers, aliasToConcreteName);
     }
 

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -59,6 +59,7 @@ class ParserContext {
 
             private final Function<String, SimilarityProvider> similarityLookupService;
             private final Function<String, TypeParser> typeParsers;
+            private final Function<String, RuntimeFieldType.Parser> runtimeTypeParsers;
             private final Version indexVersionCreated;
             private final Supplier<QueryShardContext> queryShardContextSupplier;
             private final DateFormatter dateFormatter;
@@ -69,6 +70,7 @@ class ParserContext {
 
             public ParserContext(Function<String, SimilarityProvider> similarityLookupService,
                                  Function<String, TypeParser> typeParsers,
+                                 Function<String, RuntimeFieldType.Parser> runtimeTypeParsers,
                                  Version indexVersionCreated,
                                  Supplier<QueryShardContext> queryShardContextSupplier,
                                  DateFormatter dateFormatter,
@@ -78,6 +80,7 @@ public ParserContext(Function<String, SimilarityProvider> similarityLookupServic
                                  BooleanSupplier idFieldDataEnabled) {
                 this.similarityLookupService = similarityLookupService;
                 this.typeParsers = typeParsers;
+                this.runtimeTypeParsers = runtimeTypeParsers;
                 this.indexVersionCreated = indexVersionCreated;
                 this.queryShardContextSupplier = queryShardContextSupplier;
                 this.dateFormatter = dateFormatter;
@@ -132,6 +135,8 @@ public DateFormatter getDateFormatter() {
 
             protected Function<String, TypeParser> typeParsers() { return typeParsers; }
 
+            protected Function<String, RuntimeFieldType.Parser> runtimeTypeParsers() { return runtimeTypeParsers; }
+
             protected Function<String, SimilarityProvider> similarityLookupService() { return similarityLookupService; }
 
             /**
@@ -147,8 +152,9 @@ public ParserContext createMultiFieldContext(ParserContext in) {
 
             static class MultiFieldParserContext extends ParserContext {
                 MultiFieldParserContext(ParserContext in) {
-                    super(in.similarityLookupService, in.typeParsers, in.indexVersionCreated, in.queryShardContextSupplier,
-                        in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings, in.idFieldDataEnabled);
+                    super(in.similarityLookupService, in.typeParsers, in.runtimeTypeParsers, in.indexVersionCreated,
+                        in.queryShardContextSupplier, in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings,
+                        in.idFieldDataEnabled);
                 }
 
                 @Override

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -125,8 +125,8 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers,
         this.mapperRegistry = mapperRegistry;
         Function<DateFormatter, Mapper.TypeParser.ParserContext> parserContextFunction =
             dateFormatter -> new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperRegistry.getMapperParsers()::get,
-                indexVersionCreated, queryShardContextSupplier, dateFormatter, scriptService, indexAnalyzers, indexSettings,
-                idFieldDataEnabled);
+                mapperRegistry.getRuntimeFieldTypeParsers()::get, indexVersionCreated, queryShardContextSupplier, dateFormatter,
+                scriptService, indexAnalyzers, indexSettings, idFieldDataEnabled);
         this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction);
         Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers =
             mapperRegistry.getMetadataMapperParsers(indexSettings.getIndexVersionCreated());

server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java

@@ -32,6 +32,7 @@
 import java.util.Map;
 import java.util.stream.Stream;
 
+//TODO does this need to be iterable? It is not easy to track who's relying on that
 public final class MappingLookup implements Iterable<Mapper> {
 
     /** Full field name to mapper */
@@ -51,24 +52,24 @@ public static MappingLookup fromMapping(Mapping mapping) {
                 newFieldMappers.add(metadataMapper);
             }
         }
-        collect(mapping.root, newObjectMappers, newFieldMappers, newFieldAliasMappers);
-        return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length);
+        for (Mapper child : mapping.root) {
+            collect(child, newObjectMappers, newFieldMappers, newFieldAliasMappers);
+        }
+        return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers,
+            mapping.root.runtimeFieldTypes(), mapping.metadataMappers.length);
     }
 
     private static void collect(Mapper mapper, Collection<ObjectMapper> objectMappers,
                                Collection<FieldMapper> fieldMappers,
                                Collection<FieldAliasMapper> fieldAliasMappers) {
-        if (mapper instanceof RootObjectMapper) {
-            // root mapper isn't really an object mapper
-        } else if (mapper instanceof ObjectMapper) {
+        if (mapper instanceof ObjectMapper) {
             objectMappers.add((ObjectMapper)mapper);
         } else if (mapper instanceof FieldMapper) {
             fieldMappers.add((FieldMapper)mapper);
         } else if (mapper instanceof FieldAliasMapper) {
             fieldAliasMappers.add((FieldAliasMapper) mapper);
         } else {
-            throw new IllegalStateException("Unrecognized mapper type [" +
-                mapper.getClass().getSimpleName() + "].");
+            throw new IllegalStateException("Unrecognized mapper type [" + mapper.getClass().getSimpleName() + "].");
         }
 
         for (Mapper child : mapper) {
@@ -79,6 +80,7 @@ private static void collect(Mapper mapper, Collection<ObjectMapper> objectMapper
     public MappingLookup(Collection<FieldMapper> mappers,
                          Collection<ObjectMapper> objectMappers,
                          Collection<FieldAliasMapper> aliasMappers,
+                         Collection<RuntimeFieldType> runtimeFieldTypes,
                          int metadataFieldCount) {
         Map<String, Mapper> fieldMappers = new HashMap<>();
         Map<String, Analyzer> indexAnalyzers = new HashMap<>();
@@ -115,7 +117,7 @@ public MappingLookup(Collection<FieldMapper> mappers,
             }
         }
 
-        this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers);
+        this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers, runtimeFieldTypes);
 
         this.fieldMappers = Collections.unmodifiableMap(fieldMappers);
         this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers);