Add some more tests for source-only fields

server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java

@@ -146,6 +146,43 @@ public void testParseWithShadowedMultiField() throws Exception {
         assertNotNull(doc.rootDoc().getField("field.keyword"));
     }
 
+    public void testParseWithShadowedNestedField() throws Exception {
+        DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
+            b.field("dynamic", "true");
+            b.startObject("runtime");
+            {
+                b.startObject("child.name").field("type", "keyword").endObject();
+            }
+            b.endObject();
+            b.startObject("properties");
+            {
+                b.startObject("child");
+                {
+                    b.field("type", "nested");
+                }
+                b.endObject();
+            }
+            b.endObject();
+        }));
+
+        ParsedDocument doc = mapper.parse(source(b -> {
+            b.field("name", "alice");
+            b.field("age", 42);
+            b.startArray("child");
+            {
+                b.startObject().field("name", "bob").field("age", 12).endObject();
+                b.startObject().field("name", "charlie").field("age", 10).endObject();
+            }
+            b.endArray();
+        }));
+        assertEquals(3, doc.docs().size());
+        assertEquals("alice", doc.rootDoc().getField("name").stringValue());
+        assertNull(doc.docs().get(0).getField("child.name"));   // shadowed by the runtime field
+        assertEquals(12L, doc.docs().get(0).getField("child.age").numericValue());
+        assertNull(doc.docs().get(1).getField("child.name"));
+        assertEquals(10L, doc.docs().get(1).getField("child.age").numericValue());
+    }
+
     public void testRuntimeFieldAndArrayChildren() throws IOException {
         DocumentMapper mapper = createDocumentMapper(topMapping(b -> {
             b.field("dynamic", "true");

x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/101_geo_point_from_source.yml

@@ -8,15 +8,13 @@ setup:
             number_of_shards: 1
             number_of_replicas: 0
           mappings:
+            dynamic: false
             runtime:
               location:
                 type: geo_point
             properties:
               timestamp:
                 type: date
-              location:
-                type: object
-                enabled: false
 
   - do:
       bulk:

x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/102_geo_point_source_in_query.yml

@@ -0,0 +1,160 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: locations
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            dynamic: false
+            properties:
+              timestamp:
+                type: date
+
+  - do:
+      bulk:
+        index: locations
+        refresh: true
+        body: |
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:30:17-05:00", "location" : {"lat": 13.5, "lon" : 34.89}}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:30:53-05:00", "location" : {"lat": -7.9, "lon" : 120.78}}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:31:12-05:00", "location" : {"lat": 45.78, "lon" : -173.45}}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:31:19-05:00", "location" : {"lat": 32.45, "lon" : 45.6}}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:31:22-05:00", "location" : {"lat": -63.24, "lon" : 31.0}}
+          {"index":{}}
+          {"timestamp": "1998-04-30T14:31:27-05:00", "location" : {"lat": 0.0, "lon" : 0.0}}
+
+
+---
+"fetch fields from source":
+  - do:
+      search:
+        index: locations
+        body:
+          runtime_mappings:
+            location:
+              type: geo_point
+          sort: timestamp
+          fields: [location]
+  - match: {hits.total.value: 6}
+  - match: {hits.hits.0.fields.location: ["13.499999991618097, 34.889999935403466"] }
+
+---
+"exists query":
+  - do:
+      search:
+        index: locations
+        body:
+          runtime_mappings:
+            location:
+              type: geo_point
+          query:
+            exists:
+              field: location
+  - match: {hits.total.value: 6}
+
+---
+"geo bounding box query":
+  - do:
+      search:
+        index: locations
+        body:
+          runtime_mappings:
+            location:
+              type: geo_point
+          query:
+            geo_bounding_box:
+              location:
+                top_left:
+                  lat: 10
+                  lon: -10
+                bottom_right:
+                  lat: -10
+                  lon: 10
+  - match: {hits.total.value: 1}
+
+---
+"geo shape query":
+  - do:
+      search:
+        index: locations
+        body:
+          runtime_mappings:
+            location:
+              type: geo_point
+          query:
+            geo_shape:
+              location:
+                shape:
+                  type: "envelope"
+                  coordinates: [ [ -10, 10 ], [ 10, -10 ] ]
+  - match: {hits.total.value: 1}
+
+---
+"geo distance query":
+  - do:
+      search:
+        index: locations
+        body:
+          runtime_mappings:
+            location:
+              type: geo_point
+          query:
+            geo_distance:
+              distance: "2000km"
+              location:
+                lat: 0
+                lon: 0
+  - match: {hits.total.value: 1}
+
+---
+"bounds agg":
+  - do:
+      search:
+        index: locations
+        body:
+          runtime_mappings:
+            location:
+              type: geo_point
+          aggs:
+            bounds:
+              geo_bounds:
+                field: "location"
+                wrap_longitude: false
+  - match: {hits.total.value: 6}
+  - match: {aggregations.bounds.bounds.top_left.lat: 45.7799999602139 }
+  - match: {aggregations.bounds.bounds.top_left.lon: -173.4500000718981 }
+  - match: {aggregations.bounds.bounds.bottom_right.lat: -63.240000014193356 }
+  - match: {aggregations.bounds.bounds.bottom_right.lon: 120.77999993227422 }
+
+---
+"geo_distance sort":
+  - do:
+      search:
+        index: locations
+        body:
+          runtime_mappings:
+            location:
+              type: geo_point
+          sort:
+            _geo_distance:
+              location:
+                lat: 0.0
+                lon: 0.0
+  - match: {hits.total.value: 6}
+  - match: {hits.hits.0._source.location.lat: 0.0 }
+  - match: {hits.hits.0._source.location.lon: 0.0 }
+  - match: {hits.hits.1._source.location.lat: 13.5 }
+  - match: {hits.hits.1._source.location.lon: 34.89 }
+  - match: {hits.hits.2._source.location.lat: 32.45 }
+  - match: {hits.hits.2._source.location.lon: 45.6 }
+  - match: {hits.hits.3._source.location.lat: -63.24 }
+  - match: {hits.hits.3._source.location.lon: 31.0 }
+

x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/11_keyword_from_source.yml

@@ -8,6 +8,7 @@ setup:
             number_of_shards: 1
             number_of_replicas: 0
           mappings:
+            dynamic: false
             runtime:
               status.active:
                 type: keyword
@@ -20,9 +21,6 @@ setup:
                 type: double
               node:
                 type: keyword
-              status:
-                type: object
-                enabled: false
 
 
   - do:

x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/12_keyword_source_in_query.yml

@@ -0,0 +1,64 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: sensor
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            dynamic: false
+            properties:
+              timestamp:
+                type: date
+              temperature:
+                type: long
+              voltage:
+                type: double
+              node:
+                type: keyword
+
+
+  - do:
+      bulk:
+        index: sensor
+        refresh: true
+        body: |
+          {"index":{}}
+          {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a", "status" : { "active" : "on" } }
+          {"index":{}}
+          {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b", "status" : [ { "active" : "on" } ] }
+          {"index":{}}
+          {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a", "status" : { "active" : [ "active", "on", true ] } }
+          {"index":{}}
+          {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b", "status" : 14 }
+          {"index":{}}
+          {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c", "status" : [ 9, null, { "active" : "off" } ] }
+          {"index":{}}
+          {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c", "status" : { "active" : "off" } }
+
+---
+"test matching":
+  - do:
+      search:
+        index: sensor
+        body:
+          runtime_mappings:
+            status.active:
+              type: keyword
+          query:
+            term:
+              status.active: "on"
+  - match: { hits.total.value: 3 }
+  - do:
+      search:
+        index: sensor
+        body:
+          runtime_mappings:
+            status.active:
+              type: keyword
+          query:
+            term:
+              status.active: "off"
+  - match: { hits.total.value: 2 }

x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/150_nested_runtime_fields.yml

@@ -0,0 +1,55 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: products
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              name:
+                type: keyword
+              resellers:
+                type: nested
+                properties:
+                  reseller:
+                    type: keyword
+
+  - do:
+      bulk:
+        index: products
+        refresh: true
+        body: |
+          {"index":{}}
+          {"name":"LED TV","resellers":[{"reseller":"companyA","price":350},{"reseller":"companyB","price":500}]}
+
+---
+"test docvalue loading":
+  - do:
+      search:
+        index: products
+        body:
+          runtime_mappings:
+            "resellers.price_with_vat":
+              type: double
+              script: |
+                for (double v : doc['resellers.price']) {
+                  emit(v * 1.2)
+                }
+          query:
+            match:
+              name: "LED TV"
+          aggs:
+            resellers:
+              nested:
+                path: resellers
+              aggs:
+                min_price:
+                  min:
+                    field: resellers.price_with_vat
+
+  - match: { aggregations.resellers.doc_count: 2 }
+  - match: { aggregations.resellers.min_price.value: 420 }
+

x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/21_long_from_source.yml

@@ -8,6 +8,7 @@ setup:
             number_of_shards: 1
             number_of_replicas: 0
           mappings:
+            dynamic: false
             runtime:
               history.count:
                 type: long
@@ -20,9 +21,6 @@ setup:
                 type: double
               node:
                 type: keyword
-              history:
-                type: object
-                enabled: false
 
   - do:
       bulk: