Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[ESQL] Basic casting for Date Nanos #111850

Merged
merged 9 commits into from
Aug 14, 2024
Merged

[ESQL] Basic casting for Date Nanos #111850

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
274d7b1
date nanos cases, WIP
not-napoleon Aug 8, 2024
47bc891
Merge branch 'main' into esql-date-nanos-casting
not-napoleon Aug 12, 2024
62e1895
To String
not-napoleon Aug 12, 2024
420e8d3
ToLong
not-napoleon Aug 13, 2024
60ff770
ToDatetime
not-napoleon Aug 13, 2024
4b3e03d
spotless apply
not-napoleon Aug 13, 2024
0276dc4
fix meta tests
not-napoleon Aug 14, 2024
3874584
apparently the analyzer tests validate the ToString error message
not-napoleon Aug 14, 2024
886ec57
grumble
not-napoleon Aug 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions x-pack/plugin/esql/qa/testFixtures/src/main/resources/meta.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,21 +92,21 @@ double tau()
"boolean to_boolean(field:boolean|keyword|text|double|long|unsigned_long|integer)"
"cartesian_point to_cartesianpoint(field:cartesian_point|keyword|text)"
"cartesian_shape to_cartesianshape(field:cartesian_point|cartesian_shape|keyword|text)"
"date to_datetime(field:date|keyword|text|double|long|unsigned_long|integer)"
"date to_datetime(field:date|date_nanos|keyword|text|double|long|unsigned_long|integer)"
"double to_dbl(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long)"
"double to_degrees(number:double|integer|long|unsigned_long)"
"double to_double(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long)"
"date to_dt(field:date|keyword|text|double|long|unsigned_long|integer)"
"date to_dt(field:date|date_nanos|keyword|text|double|long|unsigned_long|integer)"
"geo_point to_geopoint(field:geo_point|keyword|text)"
"geo_shape to_geoshape(field:geo_point|geo_shape|keyword|text)"
"integer to_int(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer)"
"integer to_integer(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer)"
"ip to_ip(field:ip|keyword|text)"
"long to_long(field:boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer|counter_long)"
"long to_long(field:boolean|date|date_nanos|keyword|text|double|long|unsigned_long|integer|counter_integer|counter_long)"
"keyword|text to_lower(str:keyword|text)"
"double to_radians(number:double|integer|long|unsigned_long)"
"keyword to_str(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
"keyword to_string(field:boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
"keyword to_str(field:boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
"keyword to_string(field:boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version)"
"unsigned_long to_ul(field:boolean|date|keyword|text|double|long|unsigned_long|integer)"
"unsigned_long to_ulong(field:boolean|date|keyword|text|double|long|unsigned_long|integer)"
"unsigned_long to_unsigned_long(field:boolean|date|keyword|text|double|long|unsigned_long|integer)"
Expand Down Expand Up @@ -215,21 +215,21 @@ to_bool |field |"boolean|keyword|text|double
to_boolean |field |"boolean|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression.
to_cartesianpo|field |"cartesian_point|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression.
to_cartesiansh|field |"cartesian_point|cartesian_shape|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression.
to_datetime |field |"date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression.
to_datetime |field |"date|date_nanos|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression.
to_dbl |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression.
to_degrees |number |"double|integer|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression.
to_double |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_double|counter_integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression.
to_dt |field |"date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression.
to_dt |field |"date|date_nanos|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression.
to_geopoint |field |"geo_point|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression.
to_geoshape |field |"geo_point|geo_shape|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression.
to_int |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer" |Input value. The input can be a single- or multi-valued column or an expression.
to_integer |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer" |Input value. The input can be a single- or multi-valued column or an expression.
to_ip |field |"ip|keyword|text" |Input value. The input can be a single- or multi-valued column or an expression.
to_long |field |"boolean|date|keyword|text|double|long|unsigned_long|integer|counter_integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression.
to_long |field |"boolean|date|date_nanos|keyword|text|double|long|unsigned_long|integer|counter_integer|counter_long" |Input value. The input can be a single- or multi-valued column or an expression.
to_lower |str |"keyword|text" |String expression. If `null`, the function returns `null`.
to_radians |number |"double|integer|long|unsigned_long" |Input value. The input can be a single- or multi-valued column or an expression.
to_str |field |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version" |Input value. The input can be a single- or multi-valued column or an expression.
to_string |field |"boolean|cartesian_point|cartesian_shape|date|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version" |Input value. The input can be a single- or multi-valued column or an expression.
to_str |field |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version" |Input value. The input can be a single- or multi-valued column or an expression.
to_string |field |"boolean|cartesian_point|cartesian_shape|date|date_nanos|double|geo_point|geo_shape|integer|ip|keyword|long|text|unsigned_long|version" |Input value. The input can be a single- or multi-valued column or an expression.
to_ul |field |"boolean|date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression.
to_ulong |field |"boolean|date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression.
to_unsigned_lo|field |"boolean|date|keyword|text|double|long|unsigned_long|integer" |Input value. The input can be a single- or multi-valued column or an expression.
Expand Down
12 changes: 11 additions & 1 deletion ...main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetime.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.time.DateUtils;
import org.elasticsearch.compute.ann.ConvertEvaluator;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.tree.NodeInfo;
Expand All @@ -24,6 +25,7 @@
import java.util.Map;

import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
Expand All @@ -41,6 +43,7 @@ public class ToDatetime extends AbstractConvertFunction {

private static final Map<DataType, BuildFactory> EVALUATORS = Map.ofEntries(
Map.entry(DATETIME, (field, source) -> field),
Map.entry(DATE_NANOS, ToDatetimeFromDateNanosEvaluator.Factory::new),
Map.entry(LONG, (field, source) -> field),
Map.entry(KEYWORD, ToDatetimeFromStringEvaluator.Factory::new),
Map.entry(TEXT, ToDatetimeFromStringEvaluator.Factory::new),
Expand All @@ -55,6 +58,8 @@ public class ToDatetime extends AbstractConvertFunction {
Converts an input value to a date value.
A string will only be successfully converted if it's respecting the format `yyyy-MM-dd'T'HH:mm:ss.SSS'Z'`.
To convert dates in other formats, use <<esql-date_parse>>.""",
note = "Note that when converting from nanosecond resolution to millisecond resolution with this function, the nanosecond date is"
+ "truncated, not rounded.",
examples = {
@Example(file = "date", tag = "to_datetime-str", explanation = """
Note that in this example, the last value in the source multi-valued field has not been converted.
Expand All @@ -81,7 +86,7 @@ public ToDatetime(
Source source,
@Param(
name = "field",
type = { "date", "keyword", "text", "double", "long", "unsigned_long", "integer" },
type = { "date", "date_nanos", "keyword", "text", "double", "long", "unsigned_long", "integer" },
description = "Input value. The input can be a single- or multi-valued column or an expression."
) Expression field
) {
Expand Down Expand Up @@ -121,4 +126,9 @@ protected NodeInfo<? extends Expression> info() {
static long fromKeyword(BytesRef in) {
return dateTimeToLong(in.utf8ToString());
}

@ConvertEvaluator(extraName = "FromDateNanos", warnExceptions = { IllegalArgumentException.class })
static long fromDatenanos(long in) {
return DateUtils.toMilliSeconds(in);
}
}
3 changes: 3 additions & 0 deletions ...src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToLong.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@

import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER;
import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD;
Expand All @@ -42,6 +43,7 @@ public class ToLong extends AbstractConvertFunction {
private static final Map<DataType, BuildFactory> EVALUATORS = Map.ofEntries(
Map.entry(LONG, (fieldEval, source) -> fieldEval),
Map.entry(DATETIME, (fieldEval, source) -> fieldEval),
Map.entry(DATE_NANOS, (fieldEval, source) -> fieldEval),
Map.entry(BOOLEAN, ToLongFromBooleanEvaluator.Factory::new),
Map.entry(KEYWORD, ToLongFromStringEvaluator.Factory::new),
Map.entry(TEXT, ToLongFromStringEvaluator.Factory::new),
Expand Down Expand Up @@ -76,6 +78,7 @@ public ToLong(
type = {
"boolean",
"date",
"date_nanos",
"keyword",
"text",
"double",
Expand Down
9 changes: 9 additions & 0 deletions ...c/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_POINT;
import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE;
import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE;
import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT;
import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_SHAPE;
Expand All @@ -40,6 +41,7 @@
import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString;
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString;
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.nanoTimeToString;
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.numericBooleanToString;
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.spatialToString;
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.unsignedLongToString;
Expand All @@ -52,6 +54,7 @@ public class ToString extends AbstractConvertFunction implements EvaluatorMapper
Map.entry(KEYWORD, (fieldEval, source) -> fieldEval),
Map.entry(BOOLEAN, ToStringFromBooleanEvaluator.Factory::new),
Map.entry(DATETIME, ToStringFromDatetimeEvaluator.Factory::new),
Map.entry(DATE_NANOS, ToStringFromDateNanosEvaluator.Factory::new),
Map.entry(IP, ToStringFromIPEvaluator.Factory::new),
Map.entry(DOUBLE, ToStringFromDoubleEvaluator.Factory::new),
Map.entry(LONG, ToStringFromLongEvaluator.Factory::new),
Expand Down Expand Up @@ -81,6 +84,7 @@ public ToString(
"cartesian_point",
"cartesian_shape",
"date",
"date_nanos",
"double",
"geo_point",
"geo_shape",
Expand Down Expand Up @@ -141,6 +145,11 @@ static BytesRef fromDatetime(long datetime) {
return new BytesRef(dateTimeToString(datetime));
}

@ConvertEvaluator(extraName = "FromDateNanos")
static BytesRef fromDateNanos(long datetime) {
return new BytesRef(nanoTimeToString(datetime));
}

@ConvertEvaluator(extraName = "FromDouble")
static BytesRef fromDouble(double dbl) {
return numericBooleanToString(dbl);
Expand Down
42 changes: 42 additions & 0 deletions ...esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import org.apache.lucene.document.InetAddressPoint;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.network.InetAddresses;
import org.elasticsearch.common.time.DateUtils;
import org.elasticsearch.geo.GeometryTestUtils;
import org.elasticsearch.geo.ShapeTestUtils;
import org.elasticsearch.logging.LogManager;
Expand Down Expand Up @@ -625,6 +626,26 @@ public static void forUnaryDatetime(
);
}

/**
* Generate positive test cases for a unary function operating on an {@link DataType#DATE_NANOS}.
*/
public static void forUnaryDateNanos(
List<TestCaseSupplier> suppliers,
String expectedEvaluatorToString,
DataType expectedType,
Function<Instant, Object> expectedValue,
List<String> warnings
) {
unaryNumeric(
suppliers,
expectedEvaluatorToString,
dateNanosCases(),
expectedType,
n -> expectedValue.apply(DateUtils.toInstant((long) n)),
warnings
);
}

/**
* Generate positive test cases for a unary function operating on an {@link DataType#GEO_POINT}.
*/
Expand Down Expand Up @@ -1030,6 +1051,27 @@ public static List<TypedDataSupplier> dateCases() {
);
}

/**
* Generate cases for {@link DataType#DATE_NANOS}.
*
*/
public static List<TypedDataSupplier> dateNanosCases() {
return List.of(
new TypedDataSupplier("<1970-01-01T00:00:00.000000000Z>", () -> 0L, DataType.DATE_NANOS),
new TypedDataSupplier("<date nanos>", () -> ESTestCase.randomLongBetween(0, 10 * (long) 10e11), DataType.DATE_NANOS),
new TypedDataSupplier(
"<far future date nanos>",
() -> ESTestCase.randomLongBetween(10 * (long) 10e11, Long.MAX_VALUE),
DataType.DATE_NANOS
),
new TypedDataSupplier(
"<nanos near the end of time>",
() -> ESTestCase.randomLongBetween(Long.MAX_VALUE / 100 * 99, Long.MAX_VALUE),
DataType.DATE_NANOS
)
);
}

public static List<TypedDataSupplier> datePeriodCases() {
return List.of(
new TypedDataSupplier("<zero date period>", () -> Period.ZERO, DataType.DATE_PERIOD, true),
Expand Down
10 changes: 9 additions & 1 deletion ...java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDatetimeTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;

import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.time.DateUtils;
import org.elasticsearch.xpack.esql.core.expression.Expression;
import org.elasticsearch.xpack.esql.core.tree.Source;
import org.elasticsearch.xpack.esql.core.type.DataType;
Expand All @@ -37,6 +38,13 @@ public static Iterable<Object[]> parameters() {
final List<TestCaseSupplier> suppliers = new ArrayList<>();

TestCaseSupplier.forUnaryDatetime(suppliers, read, DataType.DATETIME, Instant::toEpochMilli, emptyList());
TestCaseSupplier.forUnaryDateNanos(
suppliers,
"ToDatetimeFromDateNanosEvaluator[field=" + read + "]",
DataType.DATETIME,
i -> DateUtils.toMilliSeconds(DateUtils.toLong(i)),
emptyList()
);

TestCaseSupplier.forUnaryInt(
suppliers,
Expand Down Expand Up @@ -162,7 +170,7 @@ public static Iterable<Object[]> parameters() {
)
);

return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "datetime or numeric or string");
return parameterSuppliersFromTypedDataWithDefaultChecks(true, suppliers, (v, p) -> "date_nanos or datetime or numeric or string");
}

private static String randomDateString(long from, long to) {
Expand Down
Loading