diff --git a/docs/reference/mapping/types/date.asciidoc b/docs/reference/mapping/types/date.asciidoc index 34e5c8f4a8aa7..9bcc26a463c6e 100644 --- a/docs/reference/mapping/types/date.asciidoc +++ b/docs/reference/mapping/types/date.asciidoc @@ -132,7 +132,8 @@ The following parameters are accepted by `date` fields: <>:: If `true`, malformed numbers are ignored. If `false` (default), malformed - numbers throw an exception and reject the whole document. + numbers throw an exception and reject the whole document. Note that this + cannot be set if the `script` parameter is used. <>:: @@ -142,7 +143,29 @@ The following parameters are accepted by `date` fields: Accepts a date value in one of the configured +format+'s as the field which is substituted for any explicit `null` values. Defaults to `null`, - which means the field is treated as missing. + which means the field is treated as missing. Note that this cannot be + set of the `script` parameter is used. + + +`on_script_error`:: + + Defines what to do if the script defined by the `script` parameter + throws an error at indexing time. Accepts `reject` (default), which + will cause the entire document to be rejected, and `ignore`, which + will register the field in the document's + <> metadata field and continue + indexing. This parameter can only be set if the `script` field is + also set. + +`script`:: + + If this parameter is set, then the field will index values generated + by this script, rather than reading the values directly from the + source. If a value is set for this field on the input document, then + the document will be rejected with an error. + Scripts are in the same format as their + <>, and should emit + long-valued timestamps. <>:: diff --git a/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/43_date_calculated_at_index.yml b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/43_date_calculated_at_index.yml new file mode 100644 index 0000000000000..0f5ddd1e1b581 --- /dev/null +++ b/modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/43_date_calculated_at_index.yml @@ -0,0 +1,178 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + tomorrow: + type: date + script: + source: | + for (def dt : doc['timestamp']) { + emit(dt.plus(params.days, ChronoUnit.DAYS).toEpochMilli()); + } + params: + days: 1 + # Test fetching from _source and parsing + tomorrow_from_source: + type: date + script: + source: | + Instant instant = Instant.ofEpochMilli(parse(params._source.timestamp)); + ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC")); + emit(dt.plus(1, ChronoUnit.DAYS).toEpochMilli()); + # Test returning millis + the_past: + type: date + script: + source: | + for (def dt : doc['timestamp']) { + emit(dt.toInstant().toEpochMilli() - 1000); + } + # Test fetching many values + all_week: + type: date + script: + source: | + for (def dt : doc['timestamp']) { + for (int i = 0; i < 7; i++) { + emit(dt.plus(i, ChronoUnit.DAYS).toEpochMilli()); + } + } + # Test format parameter + formatted_tomorrow: + type: date + format: yyyy-MM-dd + script: | + for (def dt : doc['timestamp']) { + emit(dt.plus(1, ChronoUnit.DAYS).toEpochMilli()); + } + timestamp: + type: date + temperature: + type: long + voltage: + type: double + node: + type: keyword + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": "2018-01-18T17:41:34.000Z", "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +"get mapping": + - do: + indices.get_mapping: + index: sensor + - match: {sensor.mappings.properties.tomorrow.type: date } + - match: + sensor.mappings.properties.tomorrow.script.source: | + for (def dt : doc['timestamp']) { + emit(dt.plus(params.days, ChronoUnit.DAYS).toEpochMilli()); + } + - match: {sensor.mappings.properties.tomorrow.script.params: {days: 1} } + - match: {sensor.mappings.properties.tomorrow.script.lang: painless } + + - match: {sensor.mappings.properties.formatted_tomorrow.type: date } + - match: + sensor.mappings.properties.formatted_tomorrow.script.source: | + for (def dt : doc['timestamp']) { + emit(dt.plus(1, ChronoUnit.DAYS).toEpochMilli()); + } + - match: {sensor.mappings.properties.formatted_tomorrow.script.lang: painless } + - match: {sensor.mappings.properties.formatted_tomorrow.format: yyyy-MM-dd } + +--- +"fetch fields": + - do: + search: + index: sensor + body: + sort: timestamp + fields: [tomorrow, tomorrow_from_source, the_past, all_week, formatted_tomorrow] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.tomorrow: ["2018-01-19T17:41:34.000Z"] } + - match: {hits.hits.0.fields.tomorrow_from_source: ["2018-01-19T17:41:34.000Z"] } + - match: {hits.hits.0.fields.the_past: ["2018-01-18T17:41:33.000Z"] } + - match: + hits.hits.0.fields.all_week: + - 2018-01-18T17:41:34.000Z + - 2018-01-19T17:41:34.000Z + - 2018-01-20T17:41:34.000Z + - 2018-01-21T17:41:34.000Z + - 2018-01-22T17:41:34.000Z + - 2018-01-23T17:41:34.000Z + - 2018-01-24T17:41:34.000Z + - match: {hits.hits.0.fields.formatted_tomorrow: [2018-01-19] } + +--- +"docvalue_fields": + - do: + search: + index: sensor + body: + sort: timestamp + docvalue_fields: [tomorrow, tomorrow_from_source, the_past, all_week, formatted_tomorrow] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.tomorrow: ["2018-01-19T17:41:34.000Z"] } + - match: {hits.hits.0.fields.tomorrow_from_source: ["2018-01-19T17:41:34.000Z"] } + - match: {hits.hits.0.fields.the_past: ["2018-01-18T17:41:33.000Z"] } + - match: + hits.hits.0.fields.all_week: + - 2018-01-18T17:41:34.000Z + - 2018-01-19T17:41:34.000Z + - 2018-01-20T17:41:34.000Z + - 2018-01-21T17:41:34.000Z + - 2018-01-22T17:41:34.000Z + - 2018-01-23T17:41:34.000Z + - 2018-01-24T17:41:34.000Z + - match: {hits.hits.0.fields.formatted_tomorrow: [2018-01-19] } + +--- +"terms agg": + - do: + search: + index: sensor + body: + aggs: + v10: + terms: + field: tomorrow + format: strict_date_optional_time + - match: {hits.total.value: 6} + - match: {aggregations.v10.buckets.0.key_as_string: "2018-01-19T17:41:34.000Z"} + - match: {aggregations.v10.buckets.0.doc_count: 1} + - match: {aggregations.v10.buckets.1.key_as_string: "2018-01-20T17:41:34.000Z"} + - match: {aggregations.v10.buckets.1.doc_count: 1} + +--- +"term query": + - do: + search: + index: sensor + body: + query: + term: + tomorrow: 2018-01-19T17:41:34Z + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 4.0} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 3a1ff30730d31..b213f5828aede 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -12,6 +12,7 @@ import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.PointValues; import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; @@ -35,7 +36,11 @@ import org.elasticsearch.index.query.DateRangeIncludingNowQuery; import org.elasticsearch.index.query.QueryRewriteContext; import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.script.DateFieldScript; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.FieldValues; import org.elasticsearch.search.lookup.SearchLookup; import java.io.IOException; @@ -49,6 +54,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Objects; import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; @@ -226,17 +232,26 @@ public static class Builder extends FieldMapper.Builder { = Parameter.stringParam("null_value", false, m -> toType(m).nullValueAsString, null).acceptsNull(); private final Parameter ignoreMalformed; + private final Parameter