Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add runtime_script date field #60092

Merged
merged 3 commits into from
Jul 29, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,10 @@
import org.apache.lucene.search.IndexSortSortedNumericDocValuesRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.geo.ShapeRelation;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.time.DateFormatters;
import org.elasticsearch.common.time.DateMathParser;
Expand All @@ -59,6 +59,7 @@
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.LongSupplier;
import java.util.function.Supplier;
Expand Down Expand Up @@ -305,60 +306,83 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower
DateMathParser parser = forcedDateParser == null
? dateMathParser
: forcedDateParser;
return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> {
Query query = LongPoint.newRangeQuery(name(), l, u);
if (hasDocValues()) {
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u);
query = new IndexOrDocValuesQuery(query, dvQuery);

if (context.indexSortedOnField(name())) {
query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query);
}
}
return query;
});
}

public static Query dateRangeQuery(
Object lowerTerm,
Object upperTerm,
boolean includeLower,
boolean includeUpper,
@Nullable ZoneId timeZone,
DateMathParser parser,
QueryShardContext context,
Resolution resolution,
BiFunction<Long, Long, Query> builder
) {
return handleNow(context, nowSupplier -> {
long l, u;
if (lowerTerm == null) {
l = Long.MIN_VALUE;
} else {
l = parseToLong(lowerTerm, !includeLower, timeZone, parser, nowSupplier, resolution);
if (includeLower == false) {
++l;
}
}
if (upperTerm == null) {
u = Long.MAX_VALUE;
} else {
u = parseToLong(upperTerm, includeUpper, timeZone, parser, nowSupplier, resolution);
if (includeUpper == false) {
--u;
}
}
return builder.apply(l, u);
});
}

/**
* Handle {@code now} in queries.
* @param context context from which to read the current time
* @param builder build the query
* @return the result of the builder, wrapped in {@link DateRangeIncludingNowQuery} if {@code now} was used.
*/
public static Query handleNow(QueryShardContext context, Function<LongSupplier, Query> builder) {
boolean[] nowUsed = new boolean[1];
LongSupplier nowSupplier = () -> {
nowUsed[0] = true;
return context.nowInMillis();
};
long l, u;
if (lowerTerm == null) {
l = Long.MIN_VALUE;
} else {
l = parseToLong(lowerTerm, !includeLower, timeZone, parser, nowSupplier);
if (includeLower == false) {
++l;
}
}
if (upperTerm == null) {
u = Long.MAX_VALUE;
} else {
u = parseToLong(upperTerm, includeUpper, timeZone, parser, nowSupplier);
if (includeUpper == false) {
--u;
}
}

Query query = LongPoint.newRangeQuery(name(), l, u);
if (hasDocValues()) {
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u);
query = new IndexOrDocValuesQuery(query, dvQuery);

if (context.indexSortedOnField(name())) {
query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query);
}
}

if (nowUsed[0]) {
query = new DateRangeIncludingNowQuery(query);
}
return query;
Query query = builder.apply(nowSupplier);
return nowUsed[0] ? new DateRangeIncludingNowQuery(query) : query;
}

public long parseToLong(Object value, boolean roundUp,
@Nullable ZoneId zone, @Nullable DateMathParser forcedDateParser, LongSupplier now) {
DateMathParser dateParser = dateMathParser();
if (forcedDateParser != null) {
dateParser = forcedDateParser;
}
public long parseToLong(Object value, boolean roundUp, @Nullable ZoneId zone, DateMathParser dateParser, LongSupplier now) {
dateParser = dateParser == null ? dateMathParser() : dateParser;
return parseToLong(value, roundUp, zone, dateParser, now, resolution);
}

String strValue;
if (value instanceof BytesRef) {
strValue = ((BytesRef) value).utf8ToString();
} else {
strValue = value.toString();
}
Instant instant = dateParser.parse(strValue, now, roundUp, zone);
return resolution.convert(instant);
public static long parseToLong(
Object value,
boolean roundUp,
@Nullable ZoneId zone,
DateMathParser dateParser,
LongSupplier now,
Resolution resolution
) {
return resolution.convert(dateParser.parse(BytesRefs.toString(value), now, roundUp, zone));
}

@Override
Expand All @@ -371,7 +395,7 @@ public Relation isFieldWithinQuery(IndexReader reader,

long fromInclusive = Long.MIN_VALUE;
if (from != null) {
fromInclusive = parseToLong(from, !includeLower, timeZone, dateParser, context::nowInMillis);
fromInclusive = parseToLong(from, !includeLower, timeZone, dateParser, context::nowInMillis, resolution);
if (includeLower == false) {
if (fromInclusive == Long.MAX_VALUE) {
return Relation.DISJOINT;
Expand All @@ -382,7 +406,7 @@ public Relation isFieldWithinQuery(IndexReader reader,

long toInclusive = Long.MAX_VALUE;
if (to != null) {
toInclusive = parseToLong(to, includeUpper, timeZone, dateParser, context::nowInMillis);
toInclusive = parseToLong(to, includeUpper, timeZone, dateParser, context::nowInMillis, resolution);
if (includeUpper == false) {
if (toInclusive == Long.MIN_VALUE) {
return Relation.DISJOINT;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ protected Query doToQuery(QueryShardContext context) throws IOException {
return Queries.newMatchNoDocsQuery("Can't run [" + NAME + "] query on unmapped fields!");
}
Object originObj = origin.origin();
// TODO these ain't gonna work with runtime fields
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

interesting too, at least both of these are around parsing values, which you are making public so that we can reuse for runtime fields?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, we can handle this somehow.

if (fieldType instanceof DateFieldType) {
long originLong = ((DateFieldType) fieldType).parseToLong(originObj, true, null, null, context::nowInMillis);
TimeValue pivotVal = TimeValue.parseTimeValue(pivot, DistanceFeatureQueryBuilder.class.getSimpleName() + ".pivot");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,7 @@ private AbstractDistanceScoreFunction parseVariable(String fieldName, XContentPa

// dates and time and geo need special handling
parser.nextToken();
// TODO these ain't gonna work with runtime fields
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

interesting scenario that I never though about: scoring based on runtime fields. You already added this to the runtime fields meta issue right?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, I added it.

if (fieldType instanceof DateFieldMapper.DateFieldType) {
return parseDateVariable(parser, context, fieldType, mode);
} else if (fieldType instanceof GeoPointFieldType) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.runtimefields;

import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.ArrayUtil;
import org.elasticsearch.search.lookup.SearchLookup;

import java.util.Map;

/**
* Common base class for script field scripts that return long values.
*/
public abstract class AbstractLongScriptFieldScript extends AbstractScriptFieldScript {
private long[] values = new long[1];
private int count;

public AbstractLongScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
super(params, searchLookup, ctx);
}

/**
* Execute the script for the provided {@code docId}.
*/
public final void runForDoc(int docId) {
count = 0;
setDocument(docId);
execute();
}

/**
* Values from the last time {@link #runForDoc(int)} was called. This array
* is mutable and will change with the next call of {@link #runForDoc(int)}.
* It is also oversized and will contain garbage at all indices at and
* above {@link #count()}.
*/
public final long[] values() {
return values;
}

/**
* The number of results produced the last time {@link #runForDoc(int)} was called.
*/
public final int count() {
return count;
}

protected void collectValue(long v) {
if (values.length < count + 1) {
values = ArrayUtil.grow(values, count + 1);
}
values[count++] = v;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.runtimefields;

import org.apache.lucene.index.LeafReaderContext;
import org.elasticsearch.painless.spi.Whitelist;
import org.elasticsearch.painless.spi.WhitelistLoader;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptFactory;
import org.elasticsearch.search.lookup.SearchLookup;

import java.io.IOException;
import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
import java.util.List;
import java.util.Map;

public abstract class DateScriptFieldScript extends AbstractLongScriptFieldScript {
public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("date_script_field", Factory.class);

static List<Whitelist> whitelist() {
return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "date_whitelist.txt"));
}

public static final String[] PARAMETERS = {};

public interface Factory extends ScriptFactory {
LeafFactory newFactory(Map<String, Object> params, SearchLookup searchLookup);
}

public interface LeafFactory {
DateScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException;
}

public DateScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
super(params, searchLookup, ctx);
}

public static class Millis {
private final DateScriptFieldScript script;

public Millis(DateScriptFieldScript script) {
this.script = script;
}

public void millis(long v) {
script.collectValue(v);
}
}

public static class Date {
private final DateScriptFieldScript script;

public Date(DateScriptFieldScript script) {
this.script = script;
}

public void date(TemporalAccessor v) {
// TemporalAccessor is a nanos API so we have to convert.
long millis = Math.multiplyExact(v.getLong(ChronoField.INSTANT_SECONDS), 1000);
millis = Math.addExact(millis, v.get(ChronoField.NANO_OF_SECOND) / 1_000_000);
script.collectValue(millis);
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
package org.elasticsearch.xpack.runtimefields;

import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.ArrayUtil;
import org.elasticsearch.painless.spi.Whitelist;
import org.elasticsearch.painless.spi.WhitelistLoader;
import org.elasticsearch.script.ScriptContext;
Expand All @@ -18,7 +17,7 @@
import java.util.List;
import java.util.Map;

public abstract class LongScriptFieldScript extends AbstractScriptFieldScript {
public abstract class LongScriptFieldScript extends AbstractLongScriptFieldScript {
public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("long_script_field", Factory.class);

static List<Whitelist> whitelist() {
Expand All @@ -35,46 +34,10 @@ public interface LeafFactory {
LongScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException;
}

private long[] values = new long[1];
private int count;

public LongScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
super(params, searchLookup, ctx);
}

/**
* Execute the script for the provided {@code docId}.
*/
public final void runForDoc(int docId) {
count = 0;
setDocument(docId);
execute();
}

/**
* Values from the last time {@link #runForDoc(int)} was called. This array
* is mutable and will change with the next call of {@link #runForDoc(int)}.
* It is also oversized and will contain garbage at all indices at and
* above {@link #count()}.
*/
public final long[] values() {
return values;
}

/**
* The number of results produced the last time {@link #runForDoc(int)} was called.
*/
public final int count() {
return count;
}

private void collectValue(long v) {
if (values.length < count + 1) {
values = ArrayUtil.grow(values, count + 1);
}
values[count++] = v;
}

public static class Value {
private final LongScriptFieldScript script;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ public Map<String, Mapper.TypeParser> getMappers() {

@Override
public List<ScriptContext<?>> getContexts() {
return List.of(DoubleScriptFieldScript.CONTEXT, LongScriptFieldScript.CONTEXT, StringScriptFieldScript.CONTEXT);
return List.of(
DateScriptFieldScript.CONTEXT,
DoubleScriptFieldScript.CONTEXT,
LongScriptFieldScript.CONTEXT,
StringScriptFieldScript.CONTEXT
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ public class RuntimeFieldsPainlessExtension implements PainlessExtension {
@Override
public Map<ScriptContext<?>, List<Whitelist>> getContextWhitelists() {
return Map.ofEntries(
Map.entry(DateScriptFieldScript.CONTEXT, DateScriptFieldScript.whitelist()),
Map.entry(DoubleScriptFieldScript.CONTEXT, DoubleScriptFieldScript.whitelist()),
Map.entry(LongScriptFieldScript.CONTEXT, LongScriptFieldScript.whitelist()),
Map.entry(StringScriptFieldScript.CONTEXT, StringScriptFieldScript.whitelist())
Expand Down
Loading