Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a consistent way to parse dates #61105

Merged
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions x-pack/plugin/runtime-fields/qa/rest/build.gradle
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@ yamlRestTest {
'search/10_source_filtering/docvalue_fields with explicit format',
'search.aggregation/200_top_hits_metric/top_hits aggregation with sequence numbers',
'search.aggregation/200_top_hits_metric/top_hits aggregation with nested documents',
'search/140_pre_filter_search_shards/pre_filter_shard_size with shards that have no hit',
/////// TO FIX ///////

/////// NOT SUPPORTED ///////
Original file line number Diff line number Diff line change
@@ -11,6 +11,7 @@

import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.index.mapper.BooleanFieldMapper;
import org.elasticsearch.index.mapper.DateFieldMapper;
import org.elasticsearch.index.mapper.IpFieldMapper;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType;
@@ -183,8 +184,8 @@ private static String painlessToLoadFromSource(String name, String type) {
}

private static final Map<String, String> PAINLESS_TO_EMIT = Map.ofEntries(
// TODO implement dates against the parser
Map.entry(BooleanFieldMapper.CONTENT_TYPE, "value(parse(value));"),
Map.entry(DateFieldMapper.CONTENT_TYPE, "millis(parse(value.toString()));"),
Map.entry(
NumberType.DOUBLE.typeName(),
"value(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));"
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@
package org.elasticsearch.xpack.runtimefields;

import org.apache.lucene.index.LeafReaderContext;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.painless.spi.Whitelist;
import org.elasticsearch.painless.spi.WhitelistLoader;
import org.elasticsearch.script.ScriptContext;
@@ -29,15 +30,18 @@ static List<Whitelist> whitelist() {
public static final String[] PARAMETERS = {};

public interface Factory extends ScriptFactory {
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup);
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup, DateFormatter formatter);
}

public interface LeafFactory {
DateScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException;
}

public DateScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
private final DateFormatter formatter;

public DateScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, DateFormatter formatter, LeafReaderContext ctx) {
super(params, searchLookup, ctx);
this.formatter = formatter;
}

public static class Millis {
@@ -67,4 +71,15 @@ public void date(TemporalAccessor v) {
}
}

public static class Parse {
private final DateScriptFieldScript script;

public Parse(DateScriptFieldScript script) {
this.script = script;
}

public long parse(Object str) {
return script.formatter.parseMillis(str.toString());
}
}
}
Original file line number Diff line number Diff line change
@@ -85,7 +85,7 @@ public ScriptDateFieldData.Builder fielddataBuilder(String fullyQualifiedIndexNa
}

private DateScriptFieldScript.LeafFactory leafFactory(SearchLookup lookup) {
return scriptFactory.newFactory(script.getParams(), lookup);
return scriptFactory.newFactory(script.getParams(), lookup, dateTimeFormatter);
}

@Override
Original file line number Diff line number Diff line change
@@ -12,6 +12,7 @@ class org.elasticsearch.xpack.runtimefields.DateScriptFieldScript @no_import {
static_import {
void millis(org.elasticsearch.xpack.runtimefields.DateScriptFieldScript, long) bound_to org.elasticsearch.xpack.runtimefields.DateScriptFieldScript$Millis
void date(org.elasticsearch.xpack.runtimefields.DateScriptFieldScript, TemporalAccessor) bound_to org.elasticsearch.xpack.runtimefields.DateScriptFieldScript$Date
long parse(org.elasticsearch.xpack.runtimefields.DateScriptFieldScript, def) bound_to org.elasticsearch.xpack.runtimefields.DateScriptFieldScript$Parse
}

# This import is required to make painless happy and it isn't 100% clear why
Original file line number Diff line number Diff line change
@@ -349,7 +349,12 @@ public void execute() {
};
}
if (context == DateScriptFieldScript.CONTEXT) {
return (DateScriptFieldScript.Factory) (params, lookup) -> ctx -> new DateScriptFieldScript(params, lookup, ctx) {
return (DateScriptFieldScript.Factory) (params, lookup, formatter) -> ctx -> new DateScriptFieldScript(
params,
lookup,
formatter,
ctx
) {
@Override
public void execute() {
new DateScriptFieldScript.Millis(this).millis(1595431354874L);
Original file line number Diff line number Diff line change
@@ -389,7 +389,7 @@ protected ScriptDateMappedFieldType simpleMappedFieldType() throws IOException {
}

private ScriptDateMappedFieldType coolFormattedFieldType() throws IOException {
return build(simpleMappedFieldType().script, DateFormatter.forPattern("yyyy-MM-dd(-■_■)HH:mm:ss.SSSz"));
return build(simpleMappedFieldType().script, DateFormatter.forPattern("yyyy-MM-dd(-■_■)HH:mm:ss.SSSz||epoch_millis"));
}

@Override
@@ -435,16 +435,17 @@ public <FactoryType> FactoryType compile(
private DateScriptFieldScript.Factory factory(String code) {
switch (code) {
case "read_timestamp":
return (params, lookup) -> (ctx) -> new DateScriptFieldScript(params, lookup, ctx) {
return (params, lookup, formatter) -> ctx -> new DateScriptFieldScript(params, lookup, formatter, ctx) {
@Override
public void execute() {
for (Object timestamp : (List<?>) getSource().get("timestamp")) {
new DateScriptFieldScript.Millis(this).millis(((Number) timestamp).longValue());
DateScriptFieldScript.Parse parse = new DateScriptFieldScript.Parse(this);
new DateScriptFieldScript.Millis(this).millis(parse.parse(timestamp));
}
}
};
case "add_days":
return (params, lookup) -> (ctx) -> new DateScriptFieldScript(params, lookup, ctx) {
return (params, lookup, formatter) -> ctx -> new DateScriptFieldScript(params, lookup, formatter, ctx) {
@Override
public void execute() {
for (Object timestamp : (List<?>) getSource().get("timestamp")) {
Original file line number Diff line number Diff line change
@@ -87,7 +87,7 @@ public void testMatches() throws IOException {
try (DirectoryReader reader = iw.getReader()) {
IndexSearcher searcher = newSearcher(reader);
CheckedFunction<LeafReaderContext, AbstractLongScriptFieldScript, IOException> leafFactory =
ctx -> new DateScriptFieldScript(Map.of(), new SearchLookup(null, null), ctx) {
ctx -> new DateScriptFieldScript(Map.of(), new SearchLookup(null, null), null, ctx) {
@Override
public void execute() {
for (Object timestamp : (List<?>) getSource().get("timestamp")) {
Original file line number Diff line number Diff line change
@@ -27,13 +27,13 @@ setup:
}
params:
days: 1
# Test fetching from _source
# Test fetching from _source and parsing
tomorrow_from_source:
type: runtime_script
runtime_type: date
script:
source: |
Instant instant = Instant.ofEpochMilli(source['timestamp']);
Instant instant = Instant.ofEpochMilli(parse(source['timestamp']));
ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC"));
date(dt.plus(1, ChronoUnit.DAYS));
# Test returning millis
@@ -82,7 +82,7 @@ setup:
{"index":{}}
{"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"}
{"index":{}}
{"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"}
{"timestamp": "2018-01-18T17:41:34.000Z", "temperature": 202, "voltage": 4.0, "node": "c"}

---
"get mapping":