From c0329d874738fd7bb3f1fb985dc9445954d09f1c Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 22 Jul 2020 11:17:08 -0400 Subject: [PATCH] Add runtime_script date field Adds support for `runtime_script` fields with `runtime_type: date`. Doesn't add support for the `format` parameter to the mapper but *does* support it on aggregations and the like. Its a start! --- .../index/mapper/DateFieldMapper.java | 122 +++--- .../query/DistanceFeatureQueryBuilder.java | 1 + .../functionscore/DecayFunctionBuilder.java | 1 + .../AbstractLongScriptFieldScript.java | 63 +++ .../runtimefields/DateScriptFieldScript.java | 70 ++++ .../runtimefields/LongScriptFieldScript.java | 39 +- .../xpack/runtimefields/RuntimeFields.java | 7 +- .../RuntimeFieldsPainlessExtension.java | 1 + .../fielddata/ScriptDateFieldData.java | 113 ++++++ .../fielddata/ScriptLongDocValues.java | 6 +- .../mapper/RuntimeScriptFieldMapper.java | 15 + .../mapper/ScriptDateMappedFieldType.java | 157 ++++++++ .../mapper/ScriptLongMappedFieldType.java | 6 +- .../query/AbstractLongScriptFieldQuery.java | 10 +- .../query/LongScriptFieldExistsQuery.java | 4 +- .../query/LongScriptFieldRangeQuery.java | 4 +- .../query/LongScriptFieldTermQuery.java | 4 +- .../query/LongScriptFieldTermsQuery.java | 9 +- .../xpack/runtimefields/date_whitelist.txt | 19 + .../mapper/RuntimeScriptFieldMapperTests.java | 23 +- .../ScriptDateMappedFieldTypeTests.java | 378 ++++++++++++++++++ .../ScriptLongMappedFieldTypeTests.java | 4 +- .../AbstractLongScriptFieldQueryTestCase.java | 4 +- .../test/runtime_fields/40_date.yml | 142 +++++++ 24 files changed, 1091 insertions(+), 111 deletions(-) create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractLongScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDateFieldData.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldType.java create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/date_whitelist.txt create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldTypeTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 6f56b28e93759..577cb8c64d409 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -31,10 +31,10 @@ import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; -import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.common.time.DateFormatters; import org.elasticsearch.common.time.DateMathParser; @@ -57,6 +57,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.function.BiFunction; import java.util.function.Function; import java.util.function.LongSupplier; @@ -301,60 +302,83 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser; + return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { + Query query = LongPoint.newRangeQuery(name(), l, u); + if (hasDocValues()) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + + if (context.indexSortedOnField(name())) { + query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + } + } + return query; + }); + } + + public static Query dateRangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + @Nullable ZoneId timeZone, + DateMathParser parser, + QueryShardContext context, + Resolution resolution, + BiFunction builder + ) { + return handleNow(context, nowSupplier -> { + long l, u; + if (lowerTerm == null) { + l = Long.MIN_VALUE; + } else { + l = parseToLong(lowerTerm, !includeLower, timeZone, parser, nowSupplier, resolution); + if (includeLower == false) { + ++l; + } + } + if (upperTerm == null) { + u = Long.MAX_VALUE; + } else { + u = parseToLong(upperTerm, includeUpper, timeZone, parser, nowSupplier, resolution); + if (includeUpper == false) { + --u; + } + } + return builder.apply(l, u); + }); + } + + /** + * Handle {@code now} in queries. + * @param context context from which to read the current time + * @param builder build the query + * @return the result of the builder, wrapped in {@link DateRangeIncludingNowQuery} if {@code now} was used. + */ + public static Query handleNow(QueryShardContext context, Function builder) { boolean[] nowUsed = new boolean[1]; LongSupplier nowSupplier = () -> { nowUsed[0] = true; return context.nowInMillis(); }; - long l, u; - if (lowerTerm == null) { - l = Long.MIN_VALUE; - } else { - l = parseToLong(lowerTerm, !includeLower, timeZone, parser, nowSupplier); - if (includeLower == false) { - ++l; - } - } - if (upperTerm == null) { - u = Long.MAX_VALUE; - } else { - u = parseToLong(upperTerm, includeUpper, timeZone, parser, nowSupplier); - if (includeUpper == false) { - --u; - } - } - - Query query = LongPoint.newRangeQuery(name(), l, u); - if (hasDocValues()) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); - query = new IndexOrDocValuesQuery(query, dvQuery); - - if (context.indexSortedOnField(name())) { - query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); - } - } - - if (nowUsed[0]) { - query = new DateRangeIncludingNowQuery(query); - } - return query; + Query query = builder.apply(nowSupplier); + return nowUsed[0] ? new DateRangeIncludingNowQuery(query) : query; } - public long parseToLong(Object value, boolean roundUp, - @Nullable ZoneId zone, @Nullable DateMathParser forcedDateParser, LongSupplier now) { - DateMathParser dateParser = dateMathParser(); - if (forcedDateParser != null) { - dateParser = forcedDateParser; - } + public long parseToLong(Object value, boolean roundUp, @Nullable ZoneId zone, DateMathParser dateParser, LongSupplier now) { + dateParser = dateParser == null ? dateMathParser() : dateParser; + return parseToLong(value, roundUp, zone, dateParser, now, resolution); + } - String strValue; - if (value instanceof BytesRef) { - strValue = ((BytesRef) value).utf8ToString(); - } else { - strValue = value.toString(); - } - Instant instant = dateParser.parse(strValue, now, roundUp, zone); - return resolution.convert(instant); + public static long parseToLong( + Object value, + boolean roundUp, + @Nullable ZoneId zone, + DateMathParser dateParser, + LongSupplier now, + Resolution resolution + ) { + return resolution.convert(dateParser.parse(BytesRefs.toString(value), now, roundUp, zone)); } @Override @@ -367,7 +391,7 @@ public Relation isFieldWithinQuery(IndexReader reader, long fromInclusive = Long.MIN_VALUE; if (from != null) { - fromInclusive = parseToLong(from, !includeLower, timeZone, dateParser, context::nowInMillis); + fromInclusive = parseToLong(from, !includeLower, timeZone, dateParser, context::nowInMillis, resolution); if (includeLower == false) { if (fromInclusive == Long.MAX_VALUE) { return Relation.DISJOINT; @@ -378,7 +402,7 @@ public Relation isFieldWithinQuery(IndexReader reader, long toInclusive = Long.MAX_VALUE; if (to != null) { - toInclusive = parseToLong(to, includeUpper, timeZone, dateParser, context::nowInMillis); + toInclusive = parseToLong(to, includeUpper, timeZone, dateParser, context::nowInMillis, resolution); if (includeUpper == false) { if (toInclusive == Long.MIN_VALUE) { return Relation.DISJOINT; diff --git a/server/src/main/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilder.java index ec7debe5b9348..e06a3201e7c11 100644 --- a/server/src/main/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/DistanceFeatureQueryBuilder.java @@ -120,6 +120,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException { return Queries.newMatchNoDocsQuery("Can't run [" + NAME + "] query on unmapped fields!"); } Object originObj = origin.origin(); + // TODO these ain't gonna work with runtime fields if (fieldType instanceof DateFieldType) { long originLong = ((DateFieldType) fieldType).parseToLong(originObj, true, null, null, context::nowInMillis); TimeValue pivotVal = TimeValue.parseTimeValue(pivot, DistanceFeatureQueryBuilder.class.getSimpleName() + ".pivot"); diff --git a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java index 0b4177490b17b..47e08b1286ea3 100644 --- a/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/functionscore/DecayFunctionBuilder.java @@ -210,6 +210,7 @@ private AbstractDistanceScoreFunction parseVariable(String fieldName, XContentPa // dates and time and geo need special handling parser.nextToken(); + // TODO these ain't gonna work with runtime fields if (fieldType instanceof DateFieldMapper.DateFieldType) { return parseDateVariable(parser, context, fieldType, mode); } else if (fieldType instanceof GeoPointFieldType) { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractLongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractLongScriptFieldScript.java new file mode 100644 index 0000000000000..9e01271fdeb0c --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractLongScriptFieldScript.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.Map; + +/** + * Common base class for script field scripts that return long values. + */ +public abstract class AbstractLongScriptFieldScript extends AbstractScriptFieldScript { + public interface LeafFactory { + AbstractLongScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + private long[] values = new long[1]; + private int count; + + public AbstractLongScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + /** + * Execute the script for the provided {@code docId}. + */ + public final void runForDoc(int docId) { + count = 0; + setDocument(docId); + execute(); + } + + /** + * Values from the last time {@link #runForDoc(int)} was called. This array + * is mutable and will change with the next call of {@link #runForDoc(int)}. + * It is also oversized and will contain garbage at all indices at and + * above {@link #count()}. + */ + public final long[] values() { + return values; + } + + /** + * The number of results produced the last time {@link #runForDoc(int)} was called. + */ + public final int count() { + return count; + } + + protected void collectValue(long v) { + if (values.length < count + 1) { + values = ArrayUtil.grow(values, count + 1); + } + values[count++] = v; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java new file mode 100644 index 0000000000000..88a3de0199c47 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DateScriptFieldScript.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.util.List; +import java.util.Map; + +public abstract class DateScriptFieldScript extends AbstractLongScriptFieldScript { + public static final ScriptContext CONTEXT = new ScriptContext<>("date_script_field", Factory.class); + + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "date_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SearchLookup searchLookup); + } + + public interface LeafFactory { + DateScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + public DateScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + public static class Millis { + private final DateScriptFieldScript script; + + public Millis(DateScriptFieldScript script) { + this.script = script; + } + + public void millis(long v) { + script.collectValue(v); + } + } + + public static class Date { + private final DateScriptFieldScript script; + + public Date(DateScriptFieldScript script) { + this.script = script; + } + + public void date(TemporalAccessor v) { + // TemporalAccessor is a nanos API so we have to convert. + long millis = Math.multiplyExact(v.getLong(ChronoField.INSTANT_SECONDS), 1000); + millis = Math.addExact(millis, v.get(ChronoField.NANO_OF_SECOND) / 1_000_000); + script.collectValue(millis); + } + } + +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java index ba15b66f2396e..7f48f7d9d57df 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.runtimefields; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; @@ -18,7 +17,7 @@ import java.util.List; import java.util.Map; -public abstract class LongScriptFieldScript extends AbstractScriptFieldScript { +public abstract class LongScriptFieldScript extends AbstractLongScriptFieldScript { public static final ScriptContext CONTEXT = new ScriptContext<>("long_script_field", Factory.class); static List whitelist() { @@ -35,46 +34,10 @@ public interface LeafFactory { LongScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; } - private long[] values = new long[1]; - private int count; - public LongScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { super(params, searchLookup, ctx); } - /** - * Execute the script for the provided {@code docId}. - */ - public final void runForDoc(int docId) { - count = 0; - setDocument(docId); - execute(); - } - - /** - * Values from the last time {@link #runForDoc(int)} was called. This array - * is mutable and will change with the next call of {@link #runForDoc(int)}. - * It is also oversized and will contain garbage at all indices at and - * above {@link #count()}. - */ - public final long[] values() { - return values; - } - - /** - * The number of results produced the last time {@link #runForDoc(int)} was called. - */ - public final int count() { - return count; - } - - private void collectValue(long v) { - if (values.length < count + 1) { - values = ArrayUtil.grow(values, count + 1); - } - values[count++] = v; - } - public static class Value { private final LongScriptFieldScript script; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 6a63f87c8c885..dd73a504c4e66 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -26,6 +26,11 @@ public Map getMappers() { @Override public List> getContexts() { - return List.of(DoubleScriptFieldScript.CONTEXT, LongScriptFieldScript.CONTEXT, StringScriptFieldScript.CONTEXT); + return List.of( + DateScriptFieldScript.CONTEXT, + DoubleScriptFieldScript.CONTEXT, + LongScriptFieldScript.CONTEXT, + StringScriptFieldScript.CONTEXT + ); } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java index 0ccbeee8069a1..92508058750f8 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java @@ -17,6 +17,7 @@ public class RuntimeFieldsPainlessExtension implements PainlessExtension { @Override public Map, List> getContextWhitelists() { return Map.ofEntries( + Map.entry(DateScriptFieldScript.CONTEXT, DateScriptFieldScript.whitelist()), Map.entry(DoubleScriptFieldScript.CONTEXT, DoubleScriptFieldScript.whitelist()), Map.entry(LongScriptFieldScript.CONTEXT, LongScriptFieldScript.whitelist()), Map.entry(StringScriptFieldScript.CONTEXT, StringScriptFieldScript.whitelist()) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDateFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDateFieldData.java new file mode 100644 index 0000000000000..e838c7cd8f4d7 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptDateFieldData.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.IndexNumericFieldData; +import org.elasticsearch.index.fielddata.SearchLookupAware; +import org.elasticsearch.index.fielddata.plain.LeafLongFieldData; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.xpack.runtimefields.DateScriptFieldScript; + +import java.io.IOException; + +public final class ScriptDateFieldData extends IndexNumericFieldData implements SearchLookupAware { + + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final Script script; + private final DateScriptFieldScript.Factory scriptFactory; + + public Builder(String name, Script script, DateScriptFieldScript.Factory scriptFactory) { + this.name = name; + this.script = script; + this.scriptFactory = scriptFactory; + } + + @Override + public ScriptDateFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptDateFieldData(name, script, scriptFactory); + } + } + + private final String fieldName; + private final Script script; + private final DateScriptFieldScript.Factory scriptFactory; + private final SetOnce leafFactory = new SetOnce<>(); + + private ScriptDateFieldData(String fieldName, Script script, DateScriptFieldScript.Factory scriptFactory) { + this.fieldName = fieldName; + this.script = script; + this.scriptFactory = scriptFactory; + } + + @Override + public void setSearchLookup(SearchLookup searchLookup) { + this.leafFactory.set(scriptFactory.newFactory(script.getParams(), searchLookup)); + } + + @Override + public String getFieldName() { + return fieldName; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.DATE; + } + + @Override + public ScriptDateLeafFieldData load(LeafReaderContext context) { + try { + return loadDirect(context); + } catch (Exception e) { + throw ExceptionsHelper.convertToElastic(e); + } + } + + @Override + public ScriptDateLeafFieldData loadDirect(LeafReaderContext context) throws IOException { + return new ScriptDateLeafFieldData(new ScriptLongDocValues(leafFactory.get().newInstance(context))); + } + + @Override + public NumericType getNumericType() { + return NumericType.DATE; + } + + @Override + protected boolean sortRequiresCustomComparator() { + return true; + } + + @Override + public void clear() {} + + public static class ScriptDateLeafFieldData extends LeafLongFieldData { + private final ScriptLongDocValues scriptLongDocValues; + + ScriptDateLeafFieldData(ScriptLongDocValues scriptLongDocValues) { + super(0, NumericType.DATE); + this.scriptLongDocValues = scriptLongDocValues; + } + + @Override + public SortedNumericDocValues getLongValues() { + return scriptLongDocValues; + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongDocValues.java index 2aa348cbcac9b..3f7da9cec5a5c 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongDocValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptLongDocValues.java @@ -7,16 +7,16 @@ package org.elasticsearch.xpack.runtimefields.fielddata; import org.elasticsearch.index.fielddata.AbstractSortedNumericDocValues; -import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.AbstractLongScriptFieldScript; import java.io.IOException; import java.util.Arrays; public final class ScriptLongDocValues extends AbstractSortedNumericDocValues { - private final LongScriptFieldScript script; + private final AbstractLongScriptFieldScript script; private int cursor; - ScriptLongDocValues(LongScriptFieldScript script) { + ScriptLongDocValues(AbstractLongScriptFieldScript script) { this.script = script; } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java index 754c70104fc1c..50bfcd38b2928 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.runtimefields.mapper; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; @@ -17,6 +18,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.xpack.runtimefields.DateScriptFieldScript; import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; @@ -66,6 +68,19 @@ protected String contentType() { public static class Builder extends ParametrizedFieldMapper.Builder { static final Map> FIELD_TYPE_RESOLVER = Map.of( + DateFieldMapper.CONTENT_TYPE, + (builder, context) -> { + DateScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + DateScriptFieldScript.CONTEXT + ); + return new ScriptDateMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + builder.meta.getValue() + ); + }, NumberType.DOUBLE.typeName(), (builder, context) -> { DoubleScriptFieldScript.Factory factory = builder.scriptCompiler.compile( diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldType.java new file mode 100644 index 0000000000000..0c42c8d722baf --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldType.java @@ -0,0 +1,157 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import com.carrotsearch.hppc.LongHashSet; +import com.carrotsearch.hppc.LongSet; + +import org.apache.lucene.search.Query; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.lucene.search.Queries; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.DateMathParser; +import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType; +import org.elasticsearch.index.mapper.DateFieldMapper.Resolution; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.xpack.runtimefields.AbstractLongScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.DateScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.fielddata.ScriptDateFieldData; +import org.elasticsearch.xpack.runtimefields.query.LongScriptFieldExistsQuery; +import org.elasticsearch.xpack.runtimefields.query.LongScriptFieldRangeQuery; +import org.elasticsearch.xpack.runtimefields.query.LongScriptFieldTermQuery; +import org.elasticsearch.xpack.runtimefields.query.LongScriptFieldTermsQuery; + +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Map; + +public class ScriptDateMappedFieldType extends AbstractScriptMappedFieldType { + private final DateScriptFieldScript.Factory scriptFactory; + + ScriptDateMappedFieldType(String name, Script script, DateScriptFieldScript.Factory scriptFactory, Map meta) { + super(name, script, meta); + this.scriptFactory = scriptFactory; + } + + private DateFormatter dateTimeFormatter() { + return DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER; // TODO make configurable + } + + @Override + protected String runtimeType() { + return DateFieldMapper.CONTENT_TYPE; + } + + @Override + public Object valueForDisplay(Object value) { + Long val = (Long) value; + if (val == null) { + return null; + } + return dateTimeFormatter().format(Resolution.MILLISECONDS.toInstant(val).atZone(ZoneOffset.UTC)); + } + + @Override + public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { + DateFormatter dateTimeFormatter = dateTimeFormatter(); + if (format != null) { + dateTimeFormatter = DateFormatter.forPattern(format).withLocale(dateTimeFormatter.locale()); + } + if (timeZone == null) { + timeZone = ZoneOffset.UTC; + } + return new DocValueFormat.DateTime(dateTimeFormatter, timeZone, Resolution.MILLISECONDS); + } + + @Override + public ScriptDateFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { + // TODO once we get SearchLookup as an argument, we can already call scriptFactory.newFactory here and pass through the result + return new ScriptDateFieldData.Builder(name(), script, scriptFactory); + } + + private AbstractLongScriptFieldScript.LeafFactory leafFactory(QueryShardContext context) { + DateScriptFieldScript.LeafFactory delegate = scriptFactory.newFactory(script.getParams(), context.lookup()); + return ctx -> delegate.newInstance(ctx); + } + + @Override + public Query existsQuery(QueryShardContext context) { + checkAllowExpensiveQueries(context); + return new LongScriptFieldExistsQuery(script, leafFactory(context), name()); + } + + @Override + public Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + ShapeRelation relation, + ZoneId timeZone, + @Nullable DateMathParser parser, + QueryShardContext context + ) { + parser = parser == null ? DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.toDateMathParser() : parser; + checkAllowExpensiveQueries(context); + return DateFieldType.dateRangeQuery( + lowerTerm, + upperTerm, + includeLower, + includeUpper, + timeZone, + parser, + context, + DateFieldMapper.Resolution.MILLISECONDS, + (l, u) -> new LongScriptFieldRangeQuery(script, leafFactory(context), name(), l, u) + ); + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + return DateFieldType.handleNow(context, now -> { + long l = DateFieldType.parseToLong( + value, + false, + null, + dateTimeFormatter().toDateMathParser(), + now, + DateFieldMapper.Resolution.MILLISECONDS + ); + checkAllowExpensiveQueries(context); + return new LongScriptFieldTermQuery(script, leafFactory(context), name(), l); + }); + } + + @Override + public Query termsQuery(List values, QueryShardContext context) { + if (values.isEmpty()) { + return Queries.newMatchAllQuery(); + } + return DateFieldType.handleNow(context, now -> { + LongSet terms = new LongHashSet(values.size()); + for (Object value : values) { + terms.add( + DateFieldType.parseToLong( + value, + false, + null, + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.toDateMathParser(), + now, + DateFieldMapper.Resolution.MILLISECONDS + ) + ); + } + checkAllowExpensiveQueries(context); + return new LongScriptFieldTermsQuery(script, leafFactory(context), name(), terms); + }); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldType.java index 8ee09e3b67d31..56769d952d177 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldType.java @@ -16,6 +16,7 @@ import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.AbstractLongScriptFieldScript; import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; import org.elasticsearch.xpack.runtimefields.fielddata.ScriptLongFieldData; import org.elasticsearch.xpack.runtimefields.query.LongScriptFieldExistsQuery; @@ -51,8 +52,9 @@ public ScriptLongFieldData.Builder fielddataBuilder(String fullyQualifiedIndexNa return new ScriptLongFieldData.Builder(name(), script, scriptFactory); } - private LongScriptFieldScript.LeafFactory leafFactory(QueryShardContext context) { - return scriptFactory.newFactory(script.getParams(), context.lookup()); + private AbstractLongScriptFieldScript.LeafFactory leafFactory(QueryShardContext context) { + LongScriptFieldScript.LeafFactory delegate = scriptFactory.newFactory(script.getParams(), context.lookup()); + return ctx -> delegate.newInstance(ctx); } @Override diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractLongScriptFieldQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractLongScriptFieldQuery.java index 89ffd5452fd48..7366452ab8cdd 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractLongScriptFieldQuery.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractLongScriptFieldQuery.java @@ -17,18 +17,18 @@ import org.apache.lucene.search.TwoPhaseIterator; import org.apache.lucene.search.Weight; import org.elasticsearch.script.Script; -import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.AbstractLongScriptFieldScript; import java.io.IOException; import java.util.Objects; /** - * Abstract base class for building queries based on {@link LongScriptFieldScript}. + * Abstract base class for building queries based on {@link AbstractLongScriptFieldScript}. */ abstract class AbstractLongScriptFieldQuery extends AbstractScriptFieldQuery { - private final LongScriptFieldScript.LeafFactory leafFactory; + private final AbstractLongScriptFieldScript.LeafFactory leafFactory; - AbstractLongScriptFieldQuery(Script script, LongScriptFieldScript.LeafFactory leafFactory, String fieldName) { + AbstractLongScriptFieldQuery(Script script, AbstractLongScriptFieldScript.LeafFactory leafFactory, String fieldName) { super(script, fieldName); this.leafFactory = Objects.requireNonNull(leafFactory); } @@ -48,7 +48,7 @@ public boolean isCacheable(LeafReaderContext ctx) { @Override public Scorer scorer(LeafReaderContext ctx) throws IOException { - LongScriptFieldScript script = leafFactory.newInstance(ctx); + AbstractLongScriptFieldScript script = leafFactory.newInstance(ctx); DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc()); TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { @Override diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldExistsQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldExistsQuery.java index 4ee8e58aa6ed7..455fb7f9cdda3 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldExistsQuery.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldExistsQuery.java @@ -7,10 +7,10 @@ package org.elasticsearch.xpack.runtimefields.query; import org.elasticsearch.script.Script; -import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.AbstractLongScriptFieldScript; public class LongScriptFieldExistsQuery extends AbstractLongScriptFieldQuery { - public LongScriptFieldExistsQuery(Script script, LongScriptFieldScript.LeafFactory leafFactory, String fieldName) { + public LongScriptFieldExistsQuery(Script script, AbstractLongScriptFieldScript.LeafFactory leafFactory, String fieldName) { super(script, leafFactory, fieldName); } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldRangeQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldRangeQuery.java index db403cffc50c3..690eba72dea58 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldRangeQuery.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldRangeQuery.java @@ -7,7 +7,7 @@ package org.elasticsearch.xpack.runtimefields.query; import org.elasticsearch.script.Script; -import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.AbstractLongScriptFieldScript; import java.util.Objects; @@ -17,7 +17,7 @@ public class LongScriptFieldRangeQuery extends AbstractLongScriptFieldQuery { public LongScriptFieldRangeQuery( Script script, - LongScriptFieldScript.LeafFactory leafFactory, + AbstractLongScriptFieldScript.LeafFactory leafFactory, String fieldName, long lowerValue, long upperValue diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldTermQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldTermQuery.java index 9f50edfb5bec7..67015a985bce6 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldTermQuery.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldTermQuery.java @@ -7,14 +7,14 @@ package org.elasticsearch.xpack.runtimefields.query; import org.elasticsearch.script.Script; -import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.AbstractLongScriptFieldScript; import java.util.Objects; public class LongScriptFieldTermQuery extends AbstractLongScriptFieldQuery { private final long term; - public LongScriptFieldTermQuery(Script script, LongScriptFieldScript.LeafFactory leafFactory, String fieldName, long term) { + public LongScriptFieldTermQuery(Script script, AbstractLongScriptFieldScript.LeafFactory leafFactory, String fieldName, long term) { super(script, leafFactory, fieldName); this.term = term; } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldTermsQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldTermsQuery.java index 4176286ee6516..f67142e077bb5 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldTermsQuery.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/LongScriptFieldTermsQuery.java @@ -9,14 +9,19 @@ import com.carrotsearch.hppc.LongSet; import org.elasticsearch.script.Script; -import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.AbstractLongScriptFieldScript; import java.util.Objects; public class LongScriptFieldTermsQuery extends AbstractLongScriptFieldQuery { private final LongSet terms; - public LongScriptFieldTermsQuery(Script script, LongScriptFieldScript.LeafFactory leafFactory, String fieldName, LongSet terms) { + public LongScriptFieldTermsQuery( + Script script, + AbstractLongScriptFieldScript.LeafFactory leafFactory, + String fieldName, + LongSet terms + ) { super(script, leafFactory, fieldName); this.terms = terms; } diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/date_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/date_whitelist.txt new file mode 100644 index 0000000000000..0ddb6a2a92458 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/date_whitelist.txt @@ -0,0 +1,19 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# The whitelist for date-valued runtime fields + +class org.elasticsearch.xpack.runtimefields.DateScriptFieldScript @no_import { +} + +static_import { + void millis(org.elasticsearch.xpack.runtimefields.DateScriptFieldScript, long) bound_to org.elasticsearch.xpack.runtimefields.DateScriptFieldScript$Millis + void date(org.elasticsearch.xpack.runtimefields.DateScriptFieldScript, TemporalAccessor) bound_to org.elasticsearch.xpack.runtimefields.DateScriptFieldScript$Date +} + +# This import is required to make painless happy and it isn't 100% clear why +class org.elasticsearch.xpack.runtimefields.DateScriptFieldScript$Factory @no_import { +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java index c2349631b7785..7ba5ebf709857 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.xpack.runtimefields.DateScriptFieldScript; import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; import org.elasticsearch.xpack.runtimefields.RuntimeFields; @@ -137,6 +138,13 @@ public void testLong() throws IOException { assertEquals(Strings.toString(mapping("long")), Strings.toString(mapperService.documentMapper())); } + public void testDate() throws IOException { + MapperService mapperService = createIndex("test", Settings.EMPTY, mapping("date")).mapperService(); + FieldMapper mapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + assertThat(mapper, instanceOf(RuntimeScriptFieldMapper.class)); + assertEquals(Strings.toString(mapping("date")), Strings.toString(mapperService.documentMapper())); + } + public void testFieldCaps() throws Exception { for (String runtimeType : runtimeTypes) { String scriptIndex = "test_" + runtimeType + "_script"; @@ -219,6 +227,14 @@ public FactoryType compile( } private Object dummyScriptFactory(ScriptContext context) { + if (context == DateScriptFieldScript.CONTEXT) { + return (DateScriptFieldScript.Factory) (params, lookup) -> ctx -> new DateScriptFieldScript(params, lookup, ctx) { + @Override + public void execute() { + new DateScriptFieldScript.Millis(this).millis(1595431354874L); + } + }; + } if (context == DoubleScriptFieldScript.CONTEXT) { return (DoubleScriptFieldScript.Factory) (params, lookup) -> ctx -> new DoubleScriptFieldScript( params, @@ -256,7 +272,12 @@ public void execute() { @Override public Set> getSupportedContexts() { - return Set.of(StringScriptFieldScript.CONTEXT, LongScriptFieldScript.CONTEXT); + return Set.of( + DateScriptFieldScript.CONTEXT, + DoubleScriptFieldScript.CONTEXT, + StringScriptFieldScript.CONTEXT, + LongScriptFieldScript.CONTEXT + ); } }; } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldTypeTests.java new file mode 100644 index 0000000000000..b542c8f485824 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptDateMappedFieldTypeTests.java @@ -0,0 +1,378 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopFieldDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScoreScript; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.xpack.runtimefields.DateScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.RuntimeFields; +import org.elasticsearch.xpack.runtimefields.fielddata.ScriptDateFieldData; + +import java.io.IOException; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; + +import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.equalTo; + +public class ScriptDateMappedFieldTypeTests extends AbstractNonTextScriptMappedFieldTypeTestCase { + public void testFormat() throws IOException { + assertThat(simpleMappedFieldType().docValueFormat("date", null).format(1595432181354L), equalTo("2020-07-22")); + assertThat( + simpleMappedFieldType().docValueFormat("strict_date_optional_time", null).format(1595432181354L), + equalTo("2020-07-22T15:36:21.354Z") + ); + assertThat( + simpleMappedFieldType().docValueFormat("strict_date_optional_time", ZoneId.of("America/New_York")).format(1595432181354L), + equalTo("2020-07-22T11:36:21.354-04:00") + ); + assertThat( + simpleMappedFieldType().docValueFormat(null, ZoneId.of("America/New_York")).format(1595432181354L), + equalTo("2020-07-22T11:36:21.354-04:00") + ); + } + + @Override + public void testDocValues() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181354]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181356, 1595432181351]}")))); + List results = new ArrayList<>(); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + ScriptDateMappedFieldType ft = build("add_days", Map.of("days", 1)); + ScriptDateFieldData ifd = ft.fielddataBuilder("test").build(null, null, null); + ifd.setSearchLookup(mockContext().lookup()); + searcher.search(new MatchAllDocsQuery(), new Collector() { + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + SortedNumericDocValues dv = ifd.load(context).getLongValues(); + return new LeafCollector() { + @Override + public void setScorer(Scorable scorer) throws IOException {} + + @Override + public void collect(int doc) throws IOException { + if (dv.advanceExact(doc)) { + for (int i = 0; i < dv.docValueCount(); i++) { + results.add(dv.nextValue()); + } + } + } + }; + } + }); + assertThat(results, equalTo(List.of(1595518581354L, 1595518581351L, 1595518581356L))); + } + } + } + + @Override + public void testSort() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181354]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181351]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181356]}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + ScriptDateFieldData ifd = simpleMappedFieldType().fielddataBuilder("test").build(null, null, null); + ifd.setSearchLookup(mockContext().lookup()); + SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); + TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); + assertThat( + reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), + equalTo("{\"timestamp\": [1595432181351]}") + ); + assertThat( + reader.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(), + equalTo("{\"timestamp\": [1595432181354]}") + ); + assertThat( + reader.document(docs.scoreDocs[2].doc).getBinaryValue("_source").utf8ToString(), + equalTo("{\"timestamp\": [1595432181356]}") + ); + } + } + } + + @Override + public void testUsedInScript() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181354]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181351]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181356]}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + QueryShardContext qsc = mockContext(true, simpleMappedFieldType()); + assertThat(searcher.count(new ScriptScoreQuery(new MatchAllDocsQuery(), new Script("test"), new ScoreScript.LeafFactory() { + @Override + public boolean needs_score() { + return false; + } + + @Override + public ScoreScript newInstance(LeafReaderContext ctx) throws IOException { + return new ScoreScript(Map.of(), qsc.lookup(), ctx) { + @Override + public double execute(ExplanationHolder explanation) { + ScriptDocValues.Dates dates = (ScriptDocValues.Dates) getDoc().get("test"); + return dates.get(0).toInstant().toEpochMilli() % 1000; + } + }; + } + }, 354.5f, "test", 0, Version.CURRENT)), equalTo(1)); + } + } + } + + @Override + public void testExistsQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181356]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": []}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + assertThat(searcher.count(simpleMappedFieldType().existsQuery(mockContext())), equalTo(1)); + } + } + } + + @Override + public void testExistsQueryIsExpensive() throws IOException { + checkExpensiveQuery(ScriptDateMappedFieldType::existsQuery); + } + + @Override + public void testRangeQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181354]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181351]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181356]}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + MappedFieldType ft = simpleMappedFieldType(); + assertThat( + searcher.count( + ft.rangeQuery("2020-07-22T15:36:21.356Z", "2020-07-23T00:00:00.000Z", true, true, null, null, null, mockContext()) + ), + equalTo(1) + ); + assertThat( + searcher.count( + ft.rangeQuery("2020-07-22T00:00:00.00Z", "2020-07-22T15:36:21.354Z", true, true, null, null, null, mockContext()) + ), + equalTo(2) + ); + assertThat( + searcher.count(ft.rangeQuery(1595432181351L, 1595432181356L, true, true, null, null, null, mockContext())), + equalTo(3) + ); + assertThat( + searcher.count( + ft.rangeQuery("2020-07-22T15:36:21.356Z", "2020-07-23T00:00:00.000Z", true, false, null, null, null, mockContext()) + ), + equalTo(1) + ); + assertThat( + searcher.count( + ft.rangeQuery("2020-07-22T15:36:21.356Z", "2020-07-23T00:00:00.000Z", false, false, null, null, null, mockContext()) + ), + equalTo(0) + ); + } + } + } + + @Override + public void testRangeQueryIsExpensive() throws IOException { + checkExpensiveQuery( + (ft, ctx) -> ft.rangeQuery(randomLong(), randomLong(), randomBoolean(), randomBoolean(), null, null, null, ctx) + ); + } + + @Override + public void testTermQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181354]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181355]}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + assertThat(searcher.count(simpleMappedFieldType().termQuery("2020-07-22T15:36:21.354Z", mockContext())), equalTo(1)); + assertThat(searcher.count(simpleMappedFieldType().termQuery("1595432181355", mockContext())), equalTo(1)); + assertThat(searcher.count(simpleMappedFieldType().termQuery(1595432181354L, mockContext())), equalTo(1)); + assertThat(searcher.count(simpleMappedFieldType().termQuery(2595432181354L, mockContext())), equalTo(0)); + assertThat( + searcher.count(build("add_days", Map.of("days", 1)).termQuery("2020-07-23T15:36:21.354Z", mockContext())), + equalTo(1) + ); + } + } + } + + @Override + public void testTermQueryIsExpensive() throws IOException { + checkExpensiveQuery((ft, ctx) -> ft.termQuery(0, ctx)); + } + + @Override + public void testTermsQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181354]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"timestamp\": [1595432181355]}")))); + try (DirectoryReader reader = iw.getReader()) { + MappedFieldType ft = simpleMappedFieldType(); + IndexSearcher searcher = newSearcher(reader); + assertThat(searcher.count(ft.termsQuery(List.of("2020-07-22T15:36:21.354Z"), mockContext())), equalTo(1)); + assertThat(searcher.count(ft.termsQuery(List.of("1595432181354"), mockContext())), equalTo(1)); + assertThat(searcher.count(ft.termsQuery(List.of(1595432181354L), mockContext())), equalTo(1)); + assertThat(searcher.count(ft.termsQuery(List.of(2595432181354L), mockContext())), equalTo(0)); + assertThat(searcher.count(ft.termsQuery(List.of(1595432181354L, 2595432181354L), mockContext())), equalTo(1)); + assertThat(searcher.count(ft.termsQuery(List.of(2595432181354L, 1595432181354L), mockContext())), equalTo(1)); + assertThat(searcher.count(ft.termsQuery(List.of(1595432181355L, 1595432181354L), mockContext())), equalTo(2)); + } + } + } + + @Override + public void testTermsQueryIsExpensive() throws IOException { + checkExpensiveQuery((ft, ctx) -> ft.termsQuery(List.of(0), ctx)); + } + + @Override + protected ScriptDateMappedFieldType simpleMappedFieldType() throws IOException { + return build("read_timestamp"); + } + + @Override + protected String runtimeType() { + return "date"; + } + + private static ScriptDateMappedFieldType build(String code) throws IOException { + return build(code, Map.of()); + } + + private static ScriptDateMappedFieldType build(String code, Map params) throws IOException { + return build(new Script(ScriptType.INLINE, "test", code, params)); + } + + private static ScriptDateMappedFieldType build(Script script) throws IOException { + ScriptPlugin scriptPlugin = new ScriptPlugin() { + @Override + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { + return new ScriptEngine() { + @Override + public String getType() { + return "test"; + } + + @Override + public Set> getSupportedContexts() { + return Set.of(DateScriptFieldScript.CONTEXT); + } + + @Override + public FactoryType compile( + String name, + String code, + ScriptContext context, + Map params + ) { + @SuppressWarnings("unchecked") + FactoryType factory = (FactoryType) factory(code); + return factory; + } + + private DateScriptFieldScript.Factory factory(String code) { + switch (code) { + case "read_timestamp": + return (params, lookup) -> (ctx) -> new DateScriptFieldScript(params, lookup, ctx) { + @Override + public void execute() { + for (Object timestamp : (List) getSource().get("timestamp")) { + new DateScriptFieldScript.Millis(this).millis((Long) timestamp); + } + } + }; + case "add_days": + return (params, lookup) -> (ctx) -> new DateScriptFieldScript(params, lookup, ctx) { + @Override + public void execute() { + for (Object timestamp : (List) getSource().get("timestamp")) { + long epoch = (Long) timestamp; + ZonedDateTime dt = ZonedDateTime.ofInstant(Instant.ofEpochMilli(epoch), ZoneId.of("UTC")); + dt = dt.plus(((Number) params.get("days")).longValue(), ChronoUnit.DAYS); + new DateScriptFieldScript.Date(this).date(dt); + } + } + }; + default: + throw new IllegalArgumentException("unsupported script [" + code + "]"); + } + } + }; + } + }; + ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(scriptPlugin, new RuntimeFields())); + try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { + DateScriptFieldScript.Factory factory = scriptService.compile(script, DateScriptFieldScript.CONTEXT); + return new ScriptDateMappedFieldType("test", script, factory, emptyMap()); + } + } + + private void checkExpensiveQuery(BiConsumer queryBuilder) throws IOException { + ScriptDateMappedFieldType ft = simpleMappedFieldType(); + Exception e = expectThrows(ElasticsearchException.class, () -> queryBuilder.accept(ft, mockContext(false))); + assertThat( + e.getMessage(), + equalTo("queries cannot be executed against [runtime_script] fields while [search.allow_expensive_queries] is set to [false].") + ); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldTypeTests.java index 606a59d101e93..d4e9428023a63 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptLongMappedFieldTypeTests.java @@ -131,8 +131,8 @@ public ScoreScript newInstance(LeafReaderContext ctx) throws IOException { return new ScoreScript(Map.of(), qsc.lookup(), ctx) { @Override public double execute(ExplanationHolder explanation) { - ScriptDocValues.Longs doubles = (ScriptDocValues.Longs) getDoc().get("test"); - return doubles.get(0); + ScriptDocValues.Longs longs = (ScriptDocValues.Longs) getDoc().get("test"); + return longs.get(0); } }; } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractLongScriptFieldQueryTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractLongScriptFieldQueryTestCase.java index 8c42ddcff7ea5..50a1bf85658d1 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractLongScriptFieldQueryTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractLongScriptFieldQueryTestCase.java @@ -10,7 +10,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.util.automaton.ByteRunAutomaton; -import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.AbstractLongScriptFieldScript; import java.util.ArrayList; import java.util.List; @@ -21,7 +21,7 @@ public abstract class AbstractLongScriptFieldQueryTestCase extends AbstractScriptFieldQueryTestCase< T> { - protected final LongScriptFieldScript.LeafFactory leafFactory = mock(LongScriptFieldScript.LeafFactory.class); + protected final AbstractLongScriptFieldScript.LeafFactory leafFactory = mock(AbstractLongScriptFieldScript.LeafFactory.class); @Override public final void testVisit() { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml new file mode 100644 index 0000000000000..8cd6f2c8ec4ab --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/40_date.yml @@ -0,0 +1,142 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + temperature: + type: long + voltage: + type: double + node: + type: keyword + tomorrow: + type: runtime_script + runtime_type: date + script: + source: | + for (def dt : doc['timestamp']) { + date(dt.plus(params.days, ChronoUnit.DAYS)); + } + params: + days: 1 + # Test fetching from _source + tomorrow_from_source: + type: runtime_script + runtime_type: date + script: + source: | + Instant instant = Instant.ofEpochMilli(source['timestamp']); + ZonedDateTime dt = ZonedDateTime.ofInstant(instant, ZoneId.of("UTC")); + date(dt.plus(1, ChronoUnit.DAYS)); + # Test returning millis + the_past: + type: runtime_script + runtime_type: date + script: + source: | + for (def dt : doc['timestamp']) { + millis(dt.toInstant().toEpochMilli() - 1000); + } + # Test fetching many values + all_week: + type: runtime_script + runtime_type: date + script: + source: | + for (def dt : doc['timestamp']) { + for (int i = 0; i < 7; i++) { + date(dt.plus(i, ChronoUnit.DAYS)); + } + } + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +"get mapping": + - do: + indices.get_mapping: + index: sensor + - match: {sensor.mappings.properties.tomorrow.type: runtime_script } + - match: {sensor.mappings.properties.tomorrow.runtime_type: date } + - match: + sensor.mappings.properties.tomorrow.script.source: | + for (def dt : doc['timestamp']) { + date(dt.plus(params.days, ChronoUnit.DAYS)); + } + - match: {sensor.mappings.properties.tomorrow.script.params: {days: 1} } + - match: {sensor.mappings.properties.tomorrow.script.lang: painless } + +--- +"docvalue_fields": + - do: + search: + index: sensor + body: + sort: timestamp + docvalue_fields: [tomorrow, tomorrow_from_source, the_past, all_week] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.tomorrow: [2018-01-19T17:41:34.000Z] } + - match: {hits.hits.0.fields.tomorrow_from_source: [2018-01-19T17:41:34.000Z] } + - match: {hits.hits.0.fields.the_past: [2018-01-18T17:41:33.000Z] } + - match: + hits.hits.0.fields.all_week: + - 2018-01-18T17:41:34.000Z + - 2018-01-19T17:41:34.000Z + - 2018-01-20T17:41:34.000Z + - 2018-01-21T17:41:34.000Z + - 2018-01-22T17:41:34.000Z + - 2018-01-23T17:41:34.000Z + - 2018-01-24T17:41:34.000Z + +--- +"terms agg": + - do: + search: + index: sensor + body: + aggs: + v10: + terms: + field: tomorrow + format: strict_date_optional_time + - match: {hits.total.value: 6} + - match: {aggregations.v10.buckets.0.key_as_string: 2018-01-19T17:41:34.000Z} + - match: {aggregations.v10.buckets.0.doc_count: 1} + - match: {aggregations.v10.buckets.1.key_as_string: 2018-01-20T17:41:34.000Z} + - match: {aggregations.v10.buckets.1.doc_count: 1} + +--- +"term query": + - do: + search: + index: sensor + body: + query: + term: + tomorrow: 2018-01-19T17:41:34Z + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 4.0}