Create aliases for PassThroughObjectMapper subfields in FieldTypeLookup

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java

@@ -17,6 +17,8 @@
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -197,31 +199,20 @@ public void testIndexingGettingAndSearching() throws Exception {
         assertMap(attributes.get("pod.ip"), matchesMap().entry("type", "ip").entry("time_series_dimension", true));
         assertMap(attributes.get("pod.uid"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true));
         assertMap(attributes.get("pod.name"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true));
-        // alias field mappers:
-        assertMap(
-            ObjectPath.eval("properties.metricset", mapping),
-            matchesMap().entry("type", "alias").entry("path", "attributes.metricset")
-        );
-        assertMap(
-            ObjectPath.eval("properties.number.properties.long", mapping),
-            matchesMap().entry("type", "alias").entry("path", "attributes.number.long")
-        );
-        assertMap(
-            ObjectPath.eval("properties.number.properties.double", mapping),
-            matchesMap().entry("type", "alias").entry("path", "attributes.number.double")
-        );
-        assertMap(
-            ObjectPath.eval("properties.pod.properties", mapping),
-            matchesMap().extraOk().entry("name", matchesMap().entry("type", "alias").entry("path", "attributes.pod.name"))
-        );
-        assertMap(
-            ObjectPath.eval("properties.pod.properties", mapping),
-            matchesMap().extraOk().entry("uid", matchesMap().entry("type", "alias").entry("path", "attributes.pod.uid"))
-        );
-        assertMap(
-            ObjectPath.eval("properties.pod.properties", mapping),
-            matchesMap().extraOk().entry("ip", matchesMap().entry("type", "alias").entry("path", "attributes.pod.ip"))
-        );
+
+        FieldCapabilitiesResponse fieldCaps = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("k8s")).actionGet();
+        assertTrue(fieldCaps.getField("attributes.metricset").get("keyword").isDimension());
+        assertTrue(fieldCaps.getField("metricset").get("keyword").isDimension());
+        assertTrue(fieldCaps.getField("attributes.number.long").get("long").isDimension());
+        assertTrue(fieldCaps.getField("number.long").get("long").isDimension());
+        assertTrue(fieldCaps.getField("attributes.number.double").get("float").isDimension());
+        assertTrue(fieldCaps.getField("number.double").get("float").isDimension());
+        assertTrue(fieldCaps.getField("attributes.pod.ip").get("ip").isDimension());
+        assertTrue(fieldCaps.getField("pod.ip").get("ip").isDimension());
+        assertTrue(fieldCaps.getField("attributes.pod.uid").get("keyword").isDimension());
+        assertTrue(fieldCaps.getField("pod.uid").get("keyword").isDimension());
+        assertTrue(fieldCaps.getField("attributes.pod.name").get("keyword").isDimension());
+        assertTrue(fieldCaps.getField("pod.name").get("keyword").isDimension());
     }
 
     public void testIndexingGettingAndSearchingShrunkIndex() throws Exception {

modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java

@@ -239,7 +239,7 @@ public void setup() throws Exception {
                 new MetadataFieldMapper[] { dtfm },
                 Collections.emptyMap()
             );
-            MappingLookup mappingLookup = MappingLookup.fromMappers(mapping, List.of(dtfm, dateFieldMapper), List.of(), List.of());
+            MappingLookup mappingLookup = MappingLookup.fromMappers(mapping, List.of(dtfm, dateFieldMapper), List.of());
             indicesService = DataStreamTestHelper.mockIndicesServices(mappingLookup);
         }
 

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml

@@ -354,6 +354,7 @@ dynamic templates - conflicting aliases:
                   type: passthrough
                   dynamic: true
                   time_series_dimension: true
+                  superseded_by: attributes
               dynamic_templates:
                 - counter_metric:
                     mapping:
@@ -391,7 +392,7 @@ dynamic templates - conflicting aliases:
             filterA:
               filter:
                 term:
-                  dim: "C"
+                  dim: A
               aggs:
                 tsids:
                   terms:
@@ -410,7 +411,7 @@ dynamic templates - conflicting aliases:
             filterA:
               filter:
                 term:
-                  attributes.dim: A
+                  resource_attributes.dim: C
               aggs:
                 tsids:
                   terms:
@@ -833,11 +834,39 @@ enable subobjects in passthrough object:
               index:
                 number_of_shards: 1
                 mode: time_series
-                time_series:
-                  start_time: 2023-08-31T13:03:08.138Z
 
             mappings:
               properties:
                 attributes:
                   type: passthrough
                   subobjects: true
+
+---
+passthrough objects with circular deps:
+  - skip:
+      version: " - 8.12.99"
+      reason: "Support for passthrough fields was added in 8.13"
+  - do:
+      catch: /Circular dependency between pass-through objects/
+      indices.put_index_template:
+        name: my-dynamic-template
+        body:
+          index_patterns: [k9s*]
+          data_stream: {}
+          template:
+            settings:
+              index:
+                number_of_shards: 1
+                mode: time_series
+
+            mappings:
+              properties:
+                attributes:
+                  type: passthrough
+                  superseded_by: system.attributes
+                resource.attributes:
+                  type: passthrough
+                  superseded_by: attributes
+                system.attributes:
+                  type: passthrough
+                  superseded_by: resource.attributes

server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java

@@ -14,6 +14,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -38,9 +39,14 @@ final class FieldTypeLookup {
 
     private final int maxParentPathDots;
 
+    FieldTypeLookup(Collection<FieldMapper> fieldMappers, Collection<FieldAliasMapper> fieldAliasMappers) {
+        this(fieldMappers, fieldAliasMappers, List.of(), List.of());
+    }
+
     FieldTypeLookup(
         Collection<FieldMapper> fieldMappers,
         Collection<FieldAliasMapper> fieldAliasMappers,
+        Collection<PassThroughObjectMapper> passThroughMappers,
         Collection<RuntimeField> runtimeFields
     ) {
 
@@ -86,6 +92,35 @@ final class FieldTypeLookup {
             }
         }
 
+        // Pass-though subfields can be referenced without the prefix corresponding to the
+        // PassThroughObjectMapper name. This is achieved by adding a second reference to their
+        // MappedFieldType using the remaining suffix.
+        Map<String, PassThroughObjectMapper> passThroughFieldAliases = new HashMap<>();
+        for (PassThroughObjectMapper passThroughMapper : passThroughMappers) {
+            for (Mapper subfield : passThroughMapper.mappers.values()) {
+                if (subfield instanceof FieldMapper fieldMapper) {
+                    String name = fieldMapper.simpleName();
+                    // Check for conflict between PassThroughObjectMapper subfields.
+                    PassThroughObjectMapper conflict = passThroughFieldAliases.put(name, passThroughMapper);
+                    if (conflict != null) {
+                        // Check the priorities of the conflicting objects.
+                        if (conflict.priority() > passThroughMapper.priority()) {
+                            passThroughFieldAliases.put(name, conflict);
+                            continue;
+                        }
+                    } else if (fullNameToFieldType.containsKey(name)) {
+                        // There's an existing field or alias for the same field.
+                        continue;
+                    }
+                    MappedFieldType fieldType = fieldMapper.fieldType();
+                    fullNameToFieldType.put(name, fieldType);
+                    if (fieldType instanceof DynamicFieldType) {
+                        dynamicFieldTypes.put(name, (DynamicFieldType) fieldType);
+                    }
+                }
+            }
+        }
+
         for (MappedFieldType fieldType : RuntimeField.collectFieldTypes(runtimeFields).values()) {
             // this will override concrete fields with runtime fields that have the same name
             fullNameToFieldType.put(fieldType.name(), fieldType);

server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java

@@ -40,7 +40,7 @@ private CacheKey() {}
      * A lookup representing an empty mapping. It can be used to look up fields, although it won't hold any, but it does not
      * hold a valid {@link DocumentParser}, {@link IndexSettings} or {@link IndexAnalyzers}.
      */
-    public static final MappingLookup EMPTY = fromMappers(Mapping.EMPTY, List.of(), List.of(), List.of());
+    public static final MappingLookup EMPTY = fromMappers(Mapping.EMPTY, List.of(), List.of());
 
     private final CacheKey cacheKey = new CacheKey();
 
@@ -67,35 +67,40 @@ public static MappingLookup fromMapping(Mapping mapping) {
         List<ObjectMapper> newObjectMappers = new ArrayList<>();
         List<FieldMapper> newFieldMappers = new ArrayList<>();
         List<FieldAliasMapper> newFieldAliasMappers = new ArrayList<>();
+        List<PassThroughObjectMapper> newPassThroughMappers = new ArrayList<>();
         for (MetadataFieldMapper metadataMapper : mapping.getSortedMetadataMappers()) {
             if (metadataMapper != null) {
                 newFieldMappers.add(metadataMapper);
             }
         }
         for (Mapper child : mapping.getRoot()) {
-            collect(child, newObjectMappers, newFieldMappers, newFieldAliasMappers);
+            collect(child, newObjectMappers, newFieldMappers, newFieldAliasMappers, newPassThroughMappers);
         }
-        return new MappingLookup(mapping, newFieldMappers, newObjectMappers, newFieldAliasMappers);
+        return new MappingLookup(mapping, newFieldMappers, newObjectMappers, newFieldAliasMappers, newPassThroughMappers);
     }
 
     private static void collect(
         Mapper mapper,
         Collection<ObjectMapper> objectMappers,
         Collection<FieldMapper> fieldMappers,
-        Collection<FieldAliasMapper> fieldAliasMappers
+        Collection<FieldAliasMapper> fieldAliasMappers,
+        Collection<PassThroughObjectMapper> passThroughMappers
     ) {
-        if (mapper instanceof ObjectMapper) {
-            objectMappers.add((ObjectMapper) mapper);
-        } else if (mapper instanceof FieldMapper) {
-            fieldMappers.add((FieldMapper) mapper);
-        } else if (mapper instanceof FieldAliasMapper) {
-            fieldAliasMappers.add((FieldAliasMapper) mapper);
+        if (mapper instanceof PassThroughObjectMapper passThroughObjectMapper) {
+            passThroughMappers.add(passThroughObjectMapper);
+            objectMappers.add(passThroughObjectMapper);
+        } else if (mapper instanceof ObjectMapper objectMapper) {
+            objectMappers.add(objectMapper);
+        } else if (mapper instanceof FieldMapper fieldMapper) {
+            fieldMappers.add(fieldMapper);
+        } else if (mapper instanceof FieldAliasMapper fieldAliasMapper) {
+            fieldAliasMappers.add(fieldAliasMapper);
         } else {
             throw new IllegalStateException("Unrecognized mapper type [" + mapper.getClass().getSimpleName() + "].");
         }
 
         for (Mapper child : mapper) {
-            collect(child, objectMappers, fieldMappers, fieldAliasMappers);
+            collect(child, objectMappers, fieldMappers, fieldAliasMappers, passThroughMappers);
         }
     }
 
@@ -111,22 +116,29 @@ private static void collect(
      * @param mappers the field mappers
      * @param objectMappers the object mappers
      * @param aliasMappers the field alias mappers
+     * @param passThroughMappers the pass-through mappers
      * @return the newly created lookup instance
      */
     public static MappingLookup fromMappers(
         Mapping mapping,
         Collection<FieldMapper> mappers,
         Collection<ObjectMapper> objectMappers,
-        Collection<FieldAliasMapper> aliasMappers
+        Collection<FieldAliasMapper> aliasMappers,
+        Collection<PassThroughObjectMapper> passThroughMappers
     ) {
-        return new MappingLookup(mapping, mappers, objectMappers, aliasMappers);
+        return new MappingLookup(mapping, mappers, objectMappers, aliasMappers, passThroughMappers);
+    }
+
+    public static MappingLookup fromMappers(Mapping mapping, Collection<FieldMapper> mappers, Collection<ObjectMapper> objectMappers) {
+        return new MappingLookup(mapping, mappers, objectMappers, List.of(), List.of());
     }
 
     private MappingLookup(
         Mapping mapping,
         Collection<FieldMapper> mappers,
         Collection<ObjectMapper> objectMappers,
-        Collection<FieldAliasMapper> aliasMappers
+        Collection<FieldAliasMapper> aliasMappers,
+        Collection<PassThroughObjectMapper> passThroughMappers
     ) {
         this.totalFieldsCount = mapping.getRoot().getTotalFieldsCount();
         this.mapping = mapping;
@@ -172,13 +184,14 @@ private MappingLookup(
             }
         }
 
+        PassThroughObjectMapper.checkForInvalidPriorities(passThroughMappers);
         final Collection<RuntimeField> runtimeFields = mapping.getRoot().runtimeFields();
-        this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers, runtimeFields);
+        this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers, passThroughMappers, runtimeFields);
         if (runtimeFields.isEmpty()) {
             // without runtime fields this is the same as the field type lookup
             this.indexTimeLookup = fieldTypeLookup;
         } else {
-            this.indexTimeLookup = new FieldTypeLookup(mappers, aliasMappers, Collections.emptyList());
+            this.indexTimeLookup = new FieldTypeLookup(mappers, aliasMappers, passThroughMappers, Collections.emptyList());
         }
         // make all fields into compact+fast immutable maps
         this.fieldMappers = Map.copyOf(fieldMappers);