Make document parsing aware of runtime fields

Runtime fields are defined in a separate runtime section in the mappings. Since the runtime section was introduced, runtime fields are not taken into account when parsing documents. That means that if a document gets indexed that holds a field that's already defined as a runtime field, the field gets dynamically mapped as a concrete field although it will always be shadowed by the runtime field defined with the same name.

A more sensible default would be to instead consider runtime fields like ordinary mapped fields, so a dynamic update is not necessary whenever a field is defined as part of the runtime section. As a consequence, the field does not get indexed. If users prefer to keep indexing the field although it is shadowed, we consider this an exception, and they can do so by mapping the field under properties explicitly.

Relates to elastic#62906

server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -21,6 +21,7 @@
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Strings;
@@ -29,15 +30,19 @@
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.mapper.DynamicTemplate.XContentFieldType;
+import org.elasticsearch.index.query.QueryShardContext;
+import org.elasticsearch.search.lookup.SearchLookup;
 
 import java.io.IOException;
 import java.time.format.DateTimeParseException;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.function.Function;
@@ -209,7 +214,7 @@ static Mapping createDynamicUpdate(Mapping mapping, DocumentMapper docMapper, Li
         // We build a mapping by first sorting the mappers, so that all mappers containing a common prefix
         // will be processed in a contiguous block. When the prefix is no longer seen, we pop the extra elements
         // off the stack, merging them upwards into the existing mappers.
-        Collections.sort(dynamicMappers, (Mapper o1, Mapper o2) -> o1.name().compareTo(o2.name()));
+        dynamicMappers.sort(Comparator.comparing(Mapper::name));
         Iterator<Mapper> dynamicMapperItr = dynamicMappers.iterator();
         List<ObjectMapper> parentMappers = new ArrayList<>();
         Mapper firstUpdate = dynamicMapperItr.next();
@@ -814,14 +819,14 @@ private static Tuple<Integer, ObjectMapper> getDynamicParentMapper(ParseContext
         int pathsAdded = 0;
         ObjectMapper parent = mapper;
         for (int i = 0; i < paths.length-1; i++) {
-        String currentPath = context.path().pathAsText(paths[i]);
-        Mapper existingFieldMapper = context.docMapper().mappers().getMapper(currentPath);
-        if (existingFieldMapper != null) {
-            throw new MapperParsingException(
+            String currentPath = context.path().pathAsText(paths[i]);
+            Mapper existingFieldMapper = context.docMapper().mappers().getMapper(currentPath);
+            if (existingFieldMapper != null) {
+                throw new MapperParsingException(
                     "Could not dynamically add mapping for field [{}]. Existing mapping for [{}] must be of type object but found [{}].",
                     null, String.join(".", paths), currentPath, existingFieldMapper.typeName());
-        }
-        mapper = context.docMapper().mappers().objectMappers().get(currentPath);
+            }
+            mapper = context.docMapper().mappers().objectMappers().get(currentPath);
             if (mapper == null) {
                 // One mapping is missing, check if we are allowed to create a dynamic one.
                 ObjectMapper.Dynamic dynamic = dynamicOrDefault(parent, context);
@@ -890,7 +895,7 @@ private static Mapper getMapper(final ParseContext context, ObjectMapper objectM
 
         for (int i = 0; i < subfields.length - 1; ++i) {
             mapper = objectMapper.getMapper(subfields[i]);
-            if (mapper == null || (mapper instanceof ObjectMapper) == false) {
+            if (mapper instanceof ObjectMapper == false) {
                 return null;
             }
             objectMapper = (ObjectMapper)mapper;
@@ -900,6 +905,93 @@ private static Mapper getMapper(final ParseContext context, ObjectMapper objectM
                         + mapper.name() + "]");
             }
         }
-        return objectMapper.getMapper(subfields[subfields.length - 1]);
+        String leafName = subfields[subfields.length - 1];
+        mapper = objectMapper.getMapper(leafName);
+        if (mapper != null) {
+            return mapper;
+        }
+        //concrete fields take the precedence over runtime fields when parsing documents, though when a field is defined as runtime field
+        //only, and not under properties, it is ignored when it is sent as part of _source
+        RuntimeFieldType runtimeFieldType = context.docMapper().mapping().root.getRuntimeFieldType(fieldPath);
+        if (runtimeFieldType != null) {
+            return new NoOpFieldMapper(leafName, runtimeFieldType);
+        }
+        return null;
+    }
+
+    private static class NoOpFieldMapper extends FieldMapper {
+        NoOpFieldMapper(String simpleName, RuntimeFieldType runtimeField) {
+            super(simpleName, new MappedFieldType(runtimeField.name(), false, false, false, TextSearchInfo.NONE, Collections.emptyMap()) {
+                @Override
+                public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public String typeName() {
+                    throw new UnsupportedOperationException();
+                }
+
+                @Override
+                public Query termQuery(Object value, QueryShardContext context) {
+                    throw new UnsupportedOperationException();
+                }
+            }, MultiFields.empty(), CopyTo.empty());
+        }
+
+        @Override
+        protected void parseCreateField(ParseContext context) throws IOException {
+            //field defined as runtime field, don't index anything
+        }
+
+        @Override
+        public String name() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public String typeName() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public MappedFieldType fieldType() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public MultiFields multiFields() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Iterator<Mapper> iterator() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        protected void doValidate(MappingLookup mappers) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        protected void checkIncomingMergeType(FieldMapper mergeWith) {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public Builder getMergeBuilder() {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        protected String contentType() {
+            throw new UnsupportedOperationException();
+        }
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java

@@ -274,6 +274,10 @@ Collection<RuntimeFieldType> runtimeFieldTypes() {
         return runtimeFieldTypes.values();
     }
 
+    RuntimeFieldType getRuntimeFieldType(String name) {
+        return runtimeFieldTypes.get(name);
+    }
+
     public Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType) {
         return findTemplateBuilder(context, name, matchType, null);
     }

server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -59,20 +60,50 @@ protected Collection<? extends Plugin> getPlugins() {
         return List.of(new DocumentParserTestsPlugin(), new TestRuntimeField.Plugin());
     }
 
-    public void testDynamicUpdateWithRuntimeField() throws Exception {
+    public void testParseWithRuntimeField() throws Exception {
         DocumentMapper mapper = createDocumentMapper(runtimeFieldMapping(b -> b.field("type", "test")));
-        ParsedDocument doc = mapper.parse(source(b -> b.field("test", "value")));
-        RootObjectMapper root = doc.dynamicMappingsUpdate().root;
-        assertEquals(0, root.runtimeFieldTypes().size());
-        assertNotNull(root.getMapper("test"));
+        ParsedDocument doc = mapper.parse(source(b -> b.field("field", "value")));
+        //field defined as runtime field but not under properties: no dynamic updates, the field does not get indexed
+        assertNull(doc.dynamicMappingsUpdate());
+        assertNull(doc.rootDoc().getField("field"));
     }
 
-    public void testDynamicUpdateWithRuntimeFieldSameName() throws Exception {
-        DocumentMapper mapper = createDocumentMapper(runtimeFieldMapping(b -> b.field("type", "test")));
+    public void testParseWithShadowedField() throws Exception {
+        XContentBuilder builder = XContentFactory.jsonBuilder()
+            .startObject().startObject("_doc")
+                .startObject("runtime")
+                  .startObject("field").field("type", "test").endObject()
+                .endObject()
+                .startObject("properties")
+                  .startObject("field").field("type", "keyword").endObject()
+                .endObject()
+            .endObject().endObject();
+
+        DocumentMapper mapper = createDocumentMapper(builder);
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", "value")));
-        RootObjectMapper root = doc.dynamicMappingsUpdate().root;
-        assertEquals(0, root.runtimeFieldTypes().size());
-        assertNotNull(root.getMapper("field"));
+        //field defined as runtime field as well as under properties: no dynamic updates, the field gets indexed
+        assertNull(doc.dynamicMappingsUpdate());
+        assertNotNull(doc.rootDoc().getField("field"));
+    }
+
+    public void testParseWithRuntimeFieldDottedNameDisabledObject() throws Exception {
+        XContentBuilder builder = XContentFactory.jsonBuilder()
+            .startObject().startObject("_doc")
+                .startObject("runtime")
+                    .startObject("path1.path2.path3.field").field("type", "test").endObject()
+                .endObject()
+                .startObject("properties")
+                    .startObject("path1").field("type", "object").field("enabled", false).endObject()
+                .endObject()
+            .endObject().endObject();
+        MapperService mapperService = createMapperService(builder);
+        ParsedDocument doc = mapperService.documentMapper().parse(source(b -> {
+            b.startObject("path1").startObject("path2").startObject("path3");
+            b.field("field", "value");
+            b.endObject().endObject().endObject();
+        }));
+        assertNull(doc.dynamicMappingsUpdate());
+        assertNull(doc.rootDoc().getField("path1.path2.path3.field"));
     }
 
     public void testFieldDisabled() throws Exception {

server/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java

@@ -192,20 +192,43 @@ public void testDynamicUpdateWithRuntimeField() throws Exception {
         Mapping merged = mapperService.documentMapper().mapping();
         assertNotNull(merged.root.getMapper("test"));
         assertEquals(1, merged.root.runtimeFieldTypes().size());
-        assertEquals("field", merged.root.runtimeFieldTypes().iterator().next().name());
+        assertNotNull(merged.root.getRuntimeFieldType("field"));
     }
 
-    public void testDynamicUpdateWithRuntimeFieldSameName() throws Exception {
-        MapperService mapperService = createMapperService(runtimeFieldMapping(b -> b.field("type", "test")));
-        ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("field", "value")));
-        assertEquals("{\"_doc\":{\"properties\":{" +
-            "\"field\":{\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\",\"ignore_above\":256}}}}}}",
-            Strings.toString(doc.dynamicMappingsUpdate().root));
+    public void testDynamicUpdateWithRuntimeFieldDottedName() throws Exception {
+        MapperService mapperService = createMapperService(runtimeMapping(
+            b -> b.startObject("path1.path2.path3.field").field("type", "test").endObject()));
+        ParsedDocument doc = mapperService.documentMapper().parse(source(b -> {
+            b.startObject("path1").startObject("path2").startObject("path3");
+            b.field("field", "value");
+            b.endObject().endObject().endObject();
+        }));
+        RootObjectMapper root = doc.dynamicMappingsUpdate().root;
+        assertEquals(0, root.runtimeFieldTypes().size());
+        {
+            //the runtime field is defined but the object structure is not, hence it is defined under properties
+            Mapper path1 = root.getMapper("path1");
+            assertThat(path1, instanceOf(ObjectMapper.class));
+            Mapper path2 = ((ObjectMapper) path1).getMapper("path2");
+            assertThat(path2, instanceOf(ObjectMapper.class));
+            Mapper path3 = ((ObjectMapper) path2).getMapper("path3");
+            assertThat(path3, instanceOf(ObjectMapper.class));
+            assertFalse(path3.iterator().hasNext());
+        }
+        assertNull(doc.rootDoc().getField("path1.path2.path3.field"));
         merge(mapperService, dynamicMapping(doc.dynamicMappingsUpdate()));
         Mapping merged = mapperService.documentMapper().mapping();
-        assertNotNull(merged.root.getMapper("field"));
+        {
+            Mapper path1 = merged.root.getMapper("path1");
+            assertThat(path1, instanceOf(ObjectMapper.class));
+            Mapper path2 = ((ObjectMapper) path1).getMapper("path2");
+            assertThat(path2, instanceOf(ObjectMapper.class));
+            Mapper path3 = ((ObjectMapper) path2).getMapper("path3");
+            assertThat(path3, instanceOf(ObjectMapper.class));
+            assertFalse(path3.iterator().hasNext());
+        }
         assertEquals(1, merged.root.runtimeFieldTypes().size());
-        assertEquals("field", merged.root.runtimeFieldTypes().iterator().next().name());
+        assertNotNull(merged.root.getRuntimeFieldType("path1.path2.path3.field"));
     }
 
     public void testIncremental() throws Exception {