Add index-time scripts to date field mapper

docs/reference/mapping/types/date.asciidoc

@@ -132,7 +132,8 @@ The following parameters are accepted by `date` fields:
 <<ignore-malformed,`ignore_malformed`>>::
 
     If `true`, malformed numbers are ignored. If `false` (default), malformed
-    numbers throw an exception and reject the whole document.
+    numbers throw an exception and reject the whole document.  Note that this
+    cannot be set if the `script` parameter is used.
 
 <<mapping-index,`index`>>::
 
@@ -142,7 +143,29 @@ The following parameters are accepted by `date` fields:
 
     Accepts a date value in one of the configured +format+'s as the field
     which is substituted for any explicit `null` values. Defaults to `null`,
-    which means the field is treated as missing.
+    which means the field is treated as missing.  Note that this cannot be
+    set of the `script` parameter is used.
+
+
+`on_script_error`::
+
+    Defines what to do if the script defined by the `script` parameter
+    throws an error at indexing time. Accepts `reject` (default), which
+    will cause the entire document to be rejected, and `ignore`, which
+    will register the field in the document's
+    <<mapping-ignored-field,`_ignored`>> metadata field and continue
+    indexing. This parameter can only be set if the `script` field is
+    also set.
+
+`script`::
+
+    If this parameter is set, then the field will index values generated
+    by this script, rather than reading the values directly from the
+    source. If a value is set for this field on the input document, then
+    the document will be rejected with an error.
+    Scripts are in the same format as their
+    <<runtime-mapping-fields,runtime equivalent>>, and should emit
+    long-valued timestamps.
 
 <<mapping-store,`store`>>::
 

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/43_date_calculated_at_index.yml

@@ -0,0 +1,178 @@
+---
+setup:
+  - do:
+      indices.create:
+        index: sensor
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              tomorrow:
+                type: date
+                script:
+                  source: |
+                    for (def dt : doc['timestamp']) {
+                      emit(dt.plus(params.days, ChronoUnit.DAYS).toEpochMilli());
+                    }
+                  params:
+                    days: 1
+              # Test fetching from _source and parsing
+              tomorrow_from_source:
+                type: date
+                script:
+                  source: |
+                    Instant instant = Instant.ofEpochMilli(parse(params._source.timestamp));
+                    ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC"));
+                    emit(dt.plus(1, ChronoUnit.DAYS).toEpochMilli());
+              # Test returning millis
+              the_past:
+                type: date
+                script:
+                  source: |
+                    for (def dt : doc['timestamp']) {
+                      emit(dt.toInstant().toEpochMilli() - 1000);
+                    }
+              # Test fetching many values
+              all_week:
+                type: date
+                script:
+                  source: |
+                    for (def dt : doc['timestamp']) {
+                      for (int i = 0; i < 7; i++) {
+                        emit(dt.plus(i, ChronoUnit.DAYS).toEpochMilli());
+                      }
+                    }
+              # Test format parameter
+              formatted_tomorrow:
+                type: date
+                format: yyyy-MM-dd
+                script: |
+                  for (def dt : doc['timestamp']) {
+                    emit(dt.plus(1, ChronoUnit.DAYS).toEpochMilli());
+                  }
+              timestamp:
+                type: date
+              temperature:
+                type: long
+              voltage:
+                type: double
+              node:
+                type: keyword
+
+  - do:
+      bulk:
+        index: sensor
+        refresh: true
+        body: |
+          {"index":{}}
+          {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"}
+          {"index":{}}
+          {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"}
+          {"index":{}}
+          {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"}
+          {"index":{}}
+          {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"}
+          {"index":{}}
+          {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"}
+          {"index":{}}
+          {"timestamp": "2018-01-18T17:41:34.000Z", "temperature": 202, "voltage": 4.0, "node": "c"}
+
+---
+"get mapping":
+  - do:
+      indices.get_mapping:
+        index: sensor
+  - match: {sensor.mappings.properties.tomorrow.type: date }
+  - match:
+      sensor.mappings.properties.tomorrow.script.source: |
+        for (def dt : doc['timestamp']) {
+          emit(dt.plus(params.days, ChronoUnit.DAYS).toEpochMilli());
+        }
+  - match: {sensor.mappings.properties.tomorrow.script.params: {days: 1} }
+  - match: {sensor.mappings.properties.tomorrow.script.lang: painless }
+
+  - match: {sensor.mappings.properties.formatted_tomorrow.type: date }
+  - match:
+      sensor.mappings.properties.formatted_tomorrow.script.source: |
+        for (def dt : doc['timestamp']) {
+          emit(dt.plus(1, ChronoUnit.DAYS).toEpochMilli());
+        }
+  - match: {sensor.mappings.properties.formatted_tomorrow.script.lang: painless }
+  - match: {sensor.mappings.properties.formatted_tomorrow.format: yyyy-MM-dd }
+
+---
+"fetch fields":
+  - do:
+      search:
+        index: sensor
+        body:
+          sort: timestamp
+          fields: [tomorrow, tomorrow_from_source, the_past, all_week, formatted_tomorrow]
+  - match: {hits.total.value: 6}
+  - match: {hits.hits.0.fields.tomorrow: ["2018-01-19T17:41:34.000Z"] }
+  - match: {hits.hits.0.fields.tomorrow_from_source: ["2018-01-19T17:41:34.000Z"] }
+  - match: {hits.hits.0.fields.the_past: ["2018-01-18T17:41:33.000Z"] }
+  - match:
+      hits.hits.0.fields.all_week:
+        - 2018-01-18T17:41:34.000Z
+        - 2018-01-19T17:41:34.000Z
+        - 2018-01-20T17:41:34.000Z
+        - 2018-01-21T17:41:34.000Z
+        - 2018-01-22T17:41:34.000Z
+        - 2018-01-23T17:41:34.000Z
+        - 2018-01-24T17:41:34.000Z
+  - match: {hits.hits.0.fields.formatted_tomorrow: [2018-01-19] }
+
+---
+"docvalue_fields":
+  - do:
+      search:
+        index: sensor
+        body:
+          sort: timestamp
+          docvalue_fields: [tomorrow, tomorrow_from_source, the_past, all_week, formatted_tomorrow]
+  - match: {hits.total.value: 6}
+  - match: {hits.hits.0.fields.tomorrow: ["2018-01-19T17:41:34.000Z"] }
+  - match: {hits.hits.0.fields.tomorrow_from_source: ["2018-01-19T17:41:34.000Z"] }
+  - match: {hits.hits.0.fields.the_past: ["2018-01-18T17:41:33.000Z"] }
+  - match:
+      hits.hits.0.fields.all_week:
+        - 2018-01-18T17:41:34.000Z
+        - 2018-01-19T17:41:34.000Z
+        - 2018-01-20T17:41:34.000Z
+        - 2018-01-21T17:41:34.000Z
+        - 2018-01-22T17:41:34.000Z
+        - 2018-01-23T17:41:34.000Z
+        - 2018-01-24T17:41:34.000Z
+  - match: {hits.hits.0.fields.formatted_tomorrow: [2018-01-19] }
+
+---
+"terms agg":
+  - do:
+      search:
+        index: sensor
+        body:
+          aggs:
+            v10:
+              terms:
+                field: tomorrow
+                format: strict_date_optional_time
+  - match: {hits.total.value: 6}
+  - match: {aggregations.v10.buckets.0.key_as_string: "2018-01-19T17:41:34.000Z"}
+  - match: {aggregations.v10.buckets.0.doc_count: 1}
+  - match: {aggregations.v10.buckets.1.key_as_string: "2018-01-20T17:41:34.000Z"}
+  - match: {aggregations.v10.buckets.1.doc_count: 1}
+
+---
+"term query":
+  - do:
+      search:
+        index: sensor
+        body:
+          query:
+            term:
+              tomorrow: 2018-01-19T17:41:34Z
+  - match: {hits.total.value: 1}
+  - match: {hits.hits.0._source.voltage: 4.0}