Introduce dynamic runtime setting

modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java

@@ -1113,7 +1113,7 @@ private void duelRun(PercolateQuery.QueryStore queryStore, MemoryIndex memoryInd
     }
 
     private void addQuery(Query query, List<ParseContext.Document> docs) {
-        ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null);
+        ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
         fieldMapper.processQuery(query, parseContext);
         ParseContext.Document queryDocument = parseContext.doc();
         // Add to string representation of the query to make debugging easier:

modules/percolator/src/test/java/org/elasticsearch/percolator/PercolatorFieldMapperTests.java

@@ -181,7 +181,7 @@ public void testExtractTerms() throws Exception {
 
         DocumentMapper documentMapper = mapperService.documentMapper();
         PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
-        ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null);
+        ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
         fieldMapper.processQuery(bq.build(), parseContext);
         ParseContext.Document document = parseContext.doc();
 
@@ -202,7 +202,7 @@ public void testExtractTerms() throws Exception {
         bq.add(termQuery1, Occur.MUST);
         bq.add(termQuery2, Occur.MUST);
 
-        parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null);
+        parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
         fieldMapper.processQuery(bq.build(), parseContext);
         document = parseContext.doc();
 
@@ -231,7 +231,7 @@ public void testExtractRanges() throws Exception {
 
         DocumentMapper documentMapper = mapperService.documentMapper();
         PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
-        ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null);
+        ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
         fieldMapper.processQuery(bq.build(), parseContext);
         ParseContext.Document document = parseContext.doc();
 
@@ -256,7 +256,7 @@ public void testExtractRanges() throws Exception {
             .rangeQuery(15, 20, true, true, null, null, null, context);
         bq.add(rangeQuery2, Occur.MUST);
 
-        parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null);
+        parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
         fieldMapper.processQuery(bq.build(), parseContext);
         document = parseContext.doc();
 
@@ -279,7 +279,7 @@ public void testExtractTermsAndRanges_failed() throws Exception {
         TermRangeQuery query = new TermRangeQuery("field1", new BytesRef("a"), new BytesRef("z"), true, true);
         DocumentMapper documentMapper = mapperService.documentMapper();
         PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
-        ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null);
+        ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
         fieldMapper.processQuery(query, parseContext);
         ParseContext.Document document = parseContext.doc();
 
@@ -293,7 +293,7 @@ public void testExtractTermsAndRanges_partial() throws Exception {
         PhraseQuery phraseQuery = new PhraseQuery("field", "term");
         DocumentMapper documentMapper = mapperService.documentMapper();
         PercolatorFieldMapper fieldMapper = (PercolatorFieldMapper) documentMapper.mappers().getMapper(fieldName);
-        ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null);
+        ParseContext.InternalParseContext parseContext = new ParseContext.InternalParseContext(documentMapper, null, null, null, null);
         fieldMapper.processQuery(phraseQuery, parseContext);
         ParseContext.Document document = parseContext.doc();
 

server/src/main/java/org/elasticsearch/index/mapper/DynamicRuntimeFieldsBuilder.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.mapper;
+
+import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.plugins.MapperPlugin;
+
+import static org.elasticsearch.index.mapper.ObjectMapper.Dynamic;
+
+/**
+ * Defines how runtime fields are dynamically created. Used when objects are mapped with dynamic:runtime.
+ * Plugins that provide runtime field implementations can also plug in their implementation of this interface
+ * to define how leaf fields of each supported type can be dynamically created in dynamic runtime mode.
+ *
+ * @see MapperPlugin#getDynamicRuntimeFieldsBuilder()
+ * @see Dynamic
+ */
+public interface DynamicRuntimeFieldsBuilder {
+    /**
+     * Dynamically creates a runtime field from a parsed string value
+     */
+    RuntimeFieldType newDynamicStringField(String name);
+    /**
+     * Dynamically creates a runtime field from a parsed long value
+     */
+    RuntimeFieldType newDynamicLongField(String name);
+    /**
+     * Dynamically creates a runtime field from a parsed double value
+     */
+    RuntimeFieldType newDynamicDoubleField(String name);
+    /**
+     * Dynamically creates a runtime field from a parsed boolean value
+     */
+    RuntimeFieldType newDynamicBooleanField(String name);
+    /**
+     * Dynamically creates a runtime field from a parsed date value
+     */
+    RuntimeFieldType newDynamicDateField(String name, DateFormatter dateFormatter);
+}

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -60,6 +60,7 @@ class ParserContext {
             private final Function<String, SimilarityProvider> similarityLookupService;
             private final Function<String, TypeParser> typeParsers;
             private final Function<String, RuntimeFieldType.Parser> runtimeTypeParsers;
+            private final boolean supportsDynamicRuntimeMappings;
             private final Version indexVersionCreated;
             private final Supplier<QueryShardContext> queryShardContextSupplier;
             private final DateFormatter dateFormatter;
@@ -77,7 +78,8 @@ public ParserContext(Function<String, SimilarityProvider> similarityLookupServic
                                  ScriptService scriptService,
                                  IndexAnalyzers indexAnalyzers,
                                  IndexSettings indexSettings,
-                                 BooleanSupplier idFieldDataEnabled) {
+                                 BooleanSupplier idFieldDataEnabled,
+                                 boolean supportsDynamicRuntimeMappings) {
                 this.similarityLookupService = similarityLookupService;
                 this.typeParsers = typeParsers;
                 this.runtimeTypeParsers = runtimeTypeParsers;
@@ -88,6 +90,7 @@ public ParserContext(Function<String, SimilarityProvider> similarityLookupServic
                 this.indexAnalyzers = indexAnalyzers;
                 this.indexSettings = indexSettings;
                 this.idFieldDataEnabled = idFieldDataEnabled;
+                this.supportsDynamicRuntimeMappings = supportsDynamicRuntimeMappings;
             }
 
             public IndexAnalyzers getIndexAnalyzers() {
@@ -118,6 +121,10 @@ public RuntimeFieldType.Parser runtimeFieldTypeParser(String type) {
                 return runtimeTypeParsers.apply(type);
             }
 
+            public boolean supportsDynamicRuntimeMappings() {
+                return supportsDynamicRuntimeMappings;
+            }
+
             public Version indexVersionCreated() {
                 return indexVersionCreated;
             }
@@ -154,7 +161,7 @@ static class MultiFieldParserContext extends ParserContext {
                 MultiFieldParserContext(ParserContext in) {
                     super(in.similarityLookupService, in.typeParsers, in.runtimeTypeParsers, in.indexVersionCreated,
                         in.queryShardContextSupplier, in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings,
-                        in.idFieldDataEnabled);
+                        in.idFieldDataEnabled, in.supportsDynamicRuntimeMappings);
                 }
 
                 @Override

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -122,8 +122,8 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers,
         Function<DateFormatter, Mapper.TypeParser.ParserContext> parserContextFunction =
             dateFormatter -> new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperRegistry.getMapperParsers()::get,
                 mapperRegistry.getRuntimeFieldTypeParsers()::get, indexVersionCreated, queryShardContextSupplier, dateFormatter,
-                scriptService, indexAnalyzers, indexSettings, idFieldDataEnabled);
-        this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction);
+                scriptService, indexAnalyzers, indexSettings, idFieldDataEnabled, mapperRegistry.getDynamicRuntimeFieldsBuilder() != null);
+        this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction, mapperRegistry.getDynamicRuntimeFieldsBuilder());
         Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers =
             mapperRegistry.getMetadataMapperParsers(indexSettings.getIndexVersionCreated());
         this.parserContextSupplier = () -> parserContextFunction.apply(null);

server/src/main/java/org/elasticsearch/index/mapper/Mapping.java

@@ -85,8 +85,8 @@ public void validate(MappingLookup mappers) {
     /**
      * Generate a mapping update for the given root object mapper.
      */
-    public Mapping mappingUpdate(Mapper rootObjectMapper) {
-        return new Mapping((RootObjectMapper) rootObjectMapper, metadataMappers, meta);
+    public Mapping mappingUpdate(RootObjectMapper rootObjectMapper) {
+        return new Mapping(rootObjectMapper, metadataMappers, meta);
     }
 
     /** Get the root mapper with the given class. */

server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java

@@ -50,13 +50,27 @@ public class ObjectMapper extends Mapper implements Cloneable {
     public static class Defaults {
         public static final boolean ENABLED = true;
         public static final Nested NESTED = Nested.NO;
-        public static final Dynamic DYNAMIC = null; // not set, inherited from root
     }
 
     public enum Dynamic {
-        TRUE,
+        TRUE {
+            @Override
+            DynamicFieldsBuilder getDynamicFieldsBuilder() {
+                return DynamicFieldsBuilder.DYNAMIC_TRUE;
+            }
+        },
         FALSE,
-        STRICT
+        STRICT,
+        RUNTIME {
+            @Override
+            DynamicFieldsBuilder getDynamicFieldsBuilder() {
+                return DynamicFieldsBuilder.DYNAMIC_RUNTIME;
+            }
+        };
+
+        DynamicFieldsBuilder getDynamicFieldsBuilder() {
+            throw new UnsupportedOperationException("Cannot create dynamic fields when dynamic is set to [" + this + "]");
+        };
     }
 
     public static class Nested {
@@ -136,7 +150,7 @@ public static class Builder extends Mapper.Builder {
 
         protected Nested nested = Defaults.NESTED;
 
-        protected Dynamic dynamic = Defaults.DYNAMIC;
+        protected Dynamic dynamic;
 
         protected final List<Mapper.Builder> mappersBuilders = new ArrayList<>();
         protected final Version indexCreatedVersion;
@@ -193,7 +207,6 @@ protected ObjectMapper createMapper(String name, String fullPath, Explicit<Boole
 
     public static class TypeParser implements Mapper.TypeParser {
         @Override
-        @SuppressWarnings("rawtypes")
         public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
             ObjectMapper.Builder builder = new Builder(name, parserContext.indexVersionCreated());
             parseNested(name, node, builder);
@@ -215,6 +228,12 @@ protected static boolean parseObjectOrDocumentTypeProperties(String fieldName, O
                 String value = fieldNode.toString();
                 if (value.equalsIgnoreCase("strict")) {
                     builder.dynamic(Dynamic.STRICT);
+                }  else if (value.equalsIgnoreCase("runtime")) {
+                    if (parserContext.supportsDynamicRuntimeMappings() == false) {
+                        throw new IllegalArgumentException("unable to set dynamic:runtime as there is " +
+                            "no registered dynamic runtime fields builder");
+                    }
+                    builder.dynamic(Dynamic.RUNTIME);
                 } else {
                     boolean dynamic = XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".dynamic");
                     builder.dynamic(dynamic ? Dynamic.TRUE : Dynamic.FALSE);
@@ -240,7 +259,6 @@ protected static boolean parseObjectOrDocumentTypeProperties(String fieldName, O
             return false;
         }
 
-        @SuppressWarnings("rawtypes")
         protected static void parseNested(String name, Map<String, Object> node, ObjectMapper.Builder builder) {
             boolean nested = false;
             Explicit<Boolean> nestedIncludeInParent = new Explicit<>(false, false);
@@ -274,7 +292,6 @@ protected static void parseNested(String name, Map<String, Object> node, ObjectM
             }
         }
 
-        @SuppressWarnings("rawtypes")
         protected static void parseProperties(ObjectMapper.Builder objBuilder, Map<String, Object> propsNode, ParserContext parserContext) {
             Iterator<Map.Entry<String, Object>> iterator = propsNode.entrySet().iterator();
             while (iterator.hasNext()) {
@@ -385,13 +402,18 @@ protected ObjectMapper clone() {
         return clone;
     }
 
+    ObjectMapper copyAndReset() {
+        ObjectMapper copy = clone();
+        // reset the sub mappers
+        copy.mappers = new CopyOnWriteHashMap<>();
+        return copy;
+    }
+
     /**
      * Build a mapping update with the provided sub mapping update.
      */
-    public ObjectMapper mappingUpdate(Mapper mapper) {
-        ObjectMapper mappingUpdate = clone();
-        // reset the sub mappers
-        mappingUpdate.mappers = new CopyOnWriteHashMap<>();
+    final ObjectMapper mappingUpdate(Mapper mapper) {
+        ObjectMapper mappingUpdate = copyAndReset();
         mappingUpdate.putMapper(mapper);
         return mappingUpdate;
     }
@@ -568,5 +590,4 @@ public int compare(Mapper o1, Mapper o2) {
     protected void doXContent(XContentBuilder builder, Params params) throws IOException {
 
     }
-
 }