Returning nested data in fields API

rest-api-spec/src/main/resources/rest-api-spec/test/search/330_fetch_fields.yml

@@ -402,3 +402,275 @@ Test unmapped fields inside disabled objects:
   -  match:
         hits.hits.0.fields.f1\.a:
         - b
+---
+Test nested fields:
+  -  skip:
+        version: ' - 7.99.99'
+        reason: support was introduced in 8.0.0 and not backported yet
+  -  do:
+        indices.create:
+           index: test
+           body:
+              mappings:
+                 properties:
+                   products:
+                     type: nested
+                     properties:
+                       manufacturer:
+                         type: keyword
+                       base_price:
+                         type: double
+                       product_id:
+                         type: integer
+
+  -  do:
+        index:
+           index: test
+           id: 1
+           refresh: true
+           body:
+              products:
+                 - { "manufacturer" : "Supersoft", "base_price" : 1.55, "product_id" : 12345}
+                 - { "manufacturer" : "HyperSmart", "base_price" : 20.20, "product_id" : 54321}
+
+  -  do:
+        search:
+           index: test
+           body:
+              _source: false
+              fields: [ "*" ]
+  - length: { hits.hits.0.fields : 1 }
+  - match:
+        hits.hits.0.fields.products.0: { "manufacturer" : ["Supersoft"], "base_price" : [1.55], "product_id" : [12345]}
+  - match:
+        hits.hits.0.fields.products.1: { "manufacturer" : ["HyperSmart"], "base_price" : [20.20], "product_id" : [54321]}
+
+  -  do:
+        search:
+           index: test
+           body:
+              _source: false
+              fields: ["products.manufacturer", "products.base_price"]
+  - length: { hits.hits.0.fields : 1 }
+  - match:
+        hits.hits.0.fields.products.0: { "manufacturer" : ["Supersoft"], "base_price" : [1.55]}
+  - match:
+        hits.hits.0.fields.products.1: { "manufacturer" : ["HyperSmart"], "base_price" : [20.20]}
+---
+Test nested field inside object structure:
+  -  skip:
+        version: ' - 7.99.99'
+        reason: support was introduced in 8.0.0 and not backported yet
+  -  do:
+        indices.create:
+           index: test
+           body:
+              mappings:
+                 properties:
+                   obj:
+                      type: object
+                      properties:
+                        products:
+                          type: nested
+                          properties:
+                            manufacturer:
+                              type: keyword
+                            base_price:
+                              type: double
+                            product_id:
+                              type: integer
+                        other_obj_field:
+                          type: keyword
+
+  -  do:
+        index:
+           index: test
+           id: 1
+           refresh: true
+           body:
+              obj:
+                 products:
+                    - { "manufacturer" : "Supersoft", "base_price" : 1.55, "product_id" : 12345}
+                    - { "manufacturer" : "HyperSmart", "base_price" : 20.20, "product_id" : 54321}
+                 other_obj_field: other_value
+
+  -  do:
+        search:
+           index: test
+           body:
+              _source: false
+              fields: [ "*" ]
+  - length: { hits.hits.0.fields : 2 }
+  - match:
+        hits.hits.0.fields.obj\.products.0: { "manufacturer" : ["Supersoft"], "base_price" : [1.55], "product_id" : [12345]}
+  - match:
+        hits.hits.0.fields.obj\.products.1: { "manufacturer" : ["HyperSmart"], "base_price" : [20.20], "product_id" : [54321]}
+  - match:
+        hits.hits.0.fields.obj\.other_obj_field.0: other_value
+
+  -  do:
+        search:
+           index: test
+           body:
+              _source: false
+              fields: ["obj.other_obj_field"]
+  - match:
+        hits.hits.0.fields.obj\.other_obj_field.0: other_value
+---
+Test doubly nested fields:
+  -  skip:
+        version: ' - 7.99.99'
+        reason: support was introduced in 8.0.0 and not backported yet
+  -  do:
+        indices.create:
+           index: test
+           body:
+              mappings:
+                 properties:
+                   id:
+                     type: keyword
+                   user:
+                     type: nested
+                     properties:
+                       first:
+                         type: keyword
+                       last:
+                         type: keyword
+                         fields:
+                           keyword:
+                             type : keyword
+                       address:
+                         type: nested
+  -  do:
+        index:
+           index: test
+           id: 1
+           refresh: true
+           body:
+              id: abcd1234
+              user:
+                 - { "first" : "John", "address" : { "city" : "Berlin" }, "account" : { "size" : 1213 }}
+                 - { "first" : "Alice", "last" : "White", "address" : [{ "city" : "Toronto", "zip" : "1111" },{ "city" : "Ottawa", "zip" : "2222" }]}
+                 - { "first" : "John", "last" : "Snow" }
+
+  -  do:
+        search:
+           index: test
+           body:
+              _source: false
+              fields: [ { "field" : "*" } ]
+  -  match:
+        hits.hits.0.fields.id:
+           - abcd1234
+  - is_false: hits.hits.0.fields.user\.first
+  - is_false: hits.hits.0.fields.user\.last
+  - is_false: hits.hits.0.fields.user\.account\.size
+  -  match:
+        hits.hits.0.fields.user.0: { "first" : [ "John" ], "address" : [{ "city" : ["Berlin"], "city.keyword" : ["Berlin"]}],  "account.size" : [1213] }
+  -  match:
+        hits.hits.0.fields.user.1: { "first" : [ "Alice" ], "last" : [ "White" ], "last.keyword" : [ "White" ], "address" : [ { "zip" : [ "1111" ], "zip.keyword" : [ "1111" ], "city" : [ "Toronto" ], "city.keyword" : [ "Toronto" ] }, { "zip" : ["2222"], "zip.keyword" : ["2222"], "city" : ["Ottawa"], "city.keyword" : [ "Ottawa" ]}] }
+  -  match:
+        hits.hits.0.fields.user.2: { "first" : [ "John" ], "last" : [ "Snow" ], "last.keyword" : [ "Snow" ] }
+
+  -  do:
+        search:
+           index: test
+           body:
+              _source: false
+              fields: [ { "field" : "user.address*" } ]
+
+  -  match:
+        hits.hits.0.fields.user.0: { "address" : [{ "city" : ["Berlin"], "city.keyword" : ["Berlin"]}] }
+  -  match:
+        hits.hits.0.fields.user.1: { "address" : [ { "zip" : [ "1111" ], "zip.keyword" : [ "1111" ], "city" : [ "Toronto" ], "city.keyword" : [ "Toronto" ] }, { "zip" : ["2222"], "zip.keyword" : ["2222"], "city" : ["Ottawa"], "city.keyword" : [ "Ottawa" ]}] }
+  - length: { hits.hits.0.fields.user : 2 }
+
+---
+Test nested fields with unmapped subfields:
+  -  skip:
+        version: ' - 7.99.99'
+        reason: support was introduced in 8.0.0 and not backported yet
+  -  do:
+        indices.create:
+           index: test
+           body:
+              mappings:
+                 properties:
+                   id:
+                     type: keyword
+                   user:
+                     type: nested
+                     properties:
+                       first:
+                         type: keyword
+                       address:
+                         type: object
+                         enabled: false
+  -  do:
+        index:
+           index: test
+           id: 1
+           refresh: true
+           body:
+              id: abcd1234
+              user:
+                 - { "first" : "John", "address" : { "city" : "Berlin" }}
+
+  -  do:
+        search:
+           index: test
+           body:
+              _source: false
+              fields:
+              - { "field" : "*", "include_unmapped" : true }
+  -  match:
+        hits.hits.0.fields.id:
+           - abcd1234
+  - is_false: hits.hits.0.fields.user\.first
+  - is_false: hits.hits.0.fields.user\.last
+  - is_false: hits.hits.0.fields.user\.account\.size
+  -  match:
+        hits.hits.0.fields.user.0: { "first" : [ "John" ], "address.city" : ["Berlin"]}
+---
+Test nested fields with ignored subfields:
+  -  skip:
+        version: ' - 7.99.99'
+        reason: support was introduced in 8.0.0 and not backported yet
+  -  do:
+        indices.create:
+           index: test
+           body:
+              mappings:
+                 properties:
+                   malformed_outside:
+                     type: integer
+                     ignore_malformed: true
+                   user:
+                     type: nested
+                     properties:
+                       malformed_inside:
+                         type: integer
+                         ignore_malformed: true
+                       first:
+                         type: keyword
+  -  do:
+        index:
+           index: test
+           id: 1
+           refresh: true
+           body:
+              malformed_outside : "bad_value_1"
+              user:
+                 - { "first" : "John", "malformed_inside" : "bad_value_2"}
+
+  -  do:
+        search:
+           index: test
+           body:
+              _source: false
+              fields:
+              - { "field" : "*", "include_unmapped" : true }
+  - is_false: hits.hits.0.fields.malformed_outside
+  -  match:
+        hits.hits.0.fields.user:
+           - { "first" : [ "John" ] }

server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java

@@ -176,7 +176,10 @@ public void testSimpleNested() throws Exception {
         assertThat(innerHits.getAt(0).getHighlightFields().get("comments.message").getFragments()[0].string(),
                 equalTo("<em>fox</em> eat quick"));
         assertThat(innerHits.getAt(0).getExplanation().toString(), containsString("weight(comments.message:fox in"));
-        assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("fox eat quick"));
+        assertThat(
+            innerHits.getAt(0).getFields().get("comments").getValue(),
+            equalTo(Collections.singletonMap("message", Collections.singletonList("fox eat quick")))
+        );
         assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5"));
 
         response = client().prepareSearch("articles")

server/src/main/java/org/elasticsearch/common/xcontent/support/XContentMapValues.java

@@ -32,6 +32,7 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -121,6 +122,32 @@ public static Object extractValue(Map<?, ?> map, String... pathElements) {
         return XContentMapValues.extractValue(pathElements, 0, map, null);
     }
 
+    /**
+     * For the provided nested path, return its value in the xContent map.
+     *
+     * @param nestedPath the nested field value's path in the map.
+     *
+     * @return the list associated with the path in the map or {@code null} if the path does not exits.
+     */
+    @SuppressWarnings("unchecked")
+    public static List<?> extractNestedValue(String nestedPath, Map<?, ?> map) {
+        Object extractedValue = XContentMapValues.extractValue(nestedPath, map);
+        List<?> nestedParsedSource = null;
+        if (extractedValue != null) {
+            if (extractedValue instanceof List) {
+                // nested field has an array value in the _source
+                nestedParsedSource = (List<?>) extractedValue;
+            } else if (extractedValue instanceof Map) {
+                // nested field has an object value in the _source. This just means the nested field has just one inner object,
+                // which is valid, but uncommon.
+                nestedParsedSource = Collections.singletonList(extractedValue);
+            } else {
+                throw new IllegalStateException("extracted source isn't an object or an array");
+            }
+        }
+        return nestedParsedSource;
+    }
+
     /**
      * For the provided path, return its value in the xContent map.
      *