From e28ac783a62f42853b98460d1fbf0ed8aed4b018 Mon Sep 17 00:00:00 2001 From: Yury-Fridlyand Date: Fri, 23 Sep 2022 16:29:20 -0700 Subject: [PATCH] Add implementation of `now`, `sysdate`, `localtime` and similar functions (#754) * Add implementation of `now`, `sysdate`, `localtime` and similar functions (#92) Signed-off-by: Yury-Fridlyand * Rework on `now` function implementation (#113) Signed-off-by: Yury-Fridlyand * Minor SQL ANTLR clean-up. Signed-off-by: Yury-Fridlyand Signed-off-by: Yury-Fridlyand --- .../sql/common/utils/QueryContext.java | 3 + .../sql/analysis/AnalysisContext.java | 16 ++ .../sql/analysis/ExpressionAnalyzer.java | 14 ++ .../sql/ast/AbstractNodeVisitor.java | 5 + .../org/opensearch/sql/ast/dsl/AstDSL.java | 5 + .../sql/ast/expression/ConstantFunction.java | 28 +++ .../org/opensearch/sql/expression/DSL.java | 36 ++++ .../expression/datetime/DateTimeFunction.java | 102 ++++++++++ .../function/BuiltinFunctionName.java | 11 +- .../sql/analysis/ExpressionAnalyzerTest.java | 46 +++++ .../datetime/NowLikeFunctionTest.java | 128 ++++++++++++ docs/user/dql/functions.rst | 184 +++++++++++++++++- docs/user/ppl/functions/datetime.rst | 184 +++++++++++++++++- .../sql/ppl/DateTimeFunctionIT.java | 184 +++++++++++++++++- .../sql/sql/DateTimeFunctionIT.java | 179 ++++++++++++++++- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 15 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 42 +++- .../sql/ppl/parser/AstExpressionBuilder.java | 53 +++-- .../ppl/antlr/NowLikeFunctionParserTest.java | 71 +++++++ .../ppl/parser/AstExpressionBuilderTest.java | 20 ++ .../ppl/parser/AstNowLikeFunctionTest.java | 111 +++++++++++ sql/src/main/antlr/OpenSearchSQLLexer.g4 | 10 + sql/src/main/antlr/OpenSearchSQLParser.g4 | 27 ++- .../sql/sql/parser/AstExpressionBuilder.java | 28 ++- .../sql/sql/antlr/SQLSyntaxParserTest.java | 34 ++++ .../sql/sql/parser/AstBuilderTest.java | 61 +++++- 26 files changed, 1562 insertions(+), 35 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/expression/ConstantFunction.java create mode 100644 core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/antlr/NowLikeFunctionParserTest.java create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/parser/AstNowLikeFunctionTest.java diff --git a/common/src/main/java/org/opensearch/sql/common/utils/QueryContext.java b/common/src/main/java/org/opensearch/sql/common/utils/QueryContext.java index 372dbae387..ab11029d73 100644 --- a/common/src/main/java/org/opensearch/sql/common/utils/QueryContext.java +++ b/common/src/main/java/org/opensearch/sql/common/utils/QueryContext.java @@ -6,6 +6,7 @@ package org.opensearch.sql.common.utils; +import java.time.LocalDateTime; import java.util.Map; import java.util.Optional; import java.util.UUID; @@ -13,6 +14,8 @@ /** * Utility class for recording and accessing context for the query being executed. + * Implementation Details: context variables is being persisted statically in the thread context + * @see: @ThreadContext */ public class QueryContext { diff --git a/core/src/main/java/org/opensearch/sql/analysis/AnalysisContext.java b/core/src/main/java/org/opensearch/sql/analysis/AnalysisContext.java index 2d3ee1a52c..f3fd623371 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/AnalysisContext.java +++ b/core/src/main/java/org/opensearch/sql/analysis/AnalysisContext.java @@ -7,9 +7,12 @@ package org.opensearch.sql.analysis; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import lombok.Getter; +import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.NamedExpression; /** @@ -23,13 +26,26 @@ public class AnalysisContext { @Getter private final List namedParseExpressions; + /** + * Storage for values of functions which return a constant value. + * We are storing the values there to use it in sequential calls to those functions. + * For example, `now` function should the same value during processing a query. + */ + @Getter + private final Map constantFunctionValues; + public AnalysisContext() { this(new TypeEnvironment(null)); } + /** + * Class CTOR. + * @param environment Env to set to a new instance. + */ public AnalysisContext(TypeEnvironment environment) { this.environment = environment; this.namedParseExpressions = new ArrayList<>(); + this.constantFunctionValues = new HashMap<>(); } /** diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index 670da5c85c..ef9d73b7f5 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -24,6 +24,7 @@ import org.opensearch.sql.ast.expression.Case; import org.opensearch.sql.ast.expression.Cast; import org.opensearch.sql.ast.expression.Compare; +import org.opensearch.sql.ast.expression.ConstantFunction; import org.opensearch.sql.ast.expression.EqualTo; import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Function; @@ -169,6 +170,19 @@ public Expression visitRelevanceFieldList(RelevanceFieldList node, AnalysisConte ImmutableMap.copyOf(node.getFieldList()))); } + @Override + public Expression visitConstantFunction(ConstantFunction node, AnalysisContext context) { + var valueName = node.getFuncName(); + if (context.getConstantFunctionValues().containsKey(valueName)) { + return context.getConstantFunctionValues().get(valueName); + } + + var value = visitFunction(node, context); + value = DSL.literal(value.valueOf(null)); + context.getConstantFunctionValues().put(valueName, value); + return value; + } + @Override public Expression visitFunction(Function node, AnalysisContext context) { FunctionName functionName = FunctionName.of(node.getFuncName()); diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index 17321bc473..e75f8f4ce5 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -15,6 +15,7 @@ import org.opensearch.sql.ast.expression.Case; import org.opensearch.sql.ast.expression.Cast; import org.opensearch.sql.ast.expression.Compare; +import org.opensearch.sql.ast.expression.ConstantFunction; import org.opensearch.sql.ast.expression.EqualTo; import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Function; @@ -116,6 +117,10 @@ public T visitRelevanceFieldList(RelevanceFieldList node, C context) { return visitChildren(node, context); } + public T visitConstantFunction(ConstantFunction node, C context) { + return visitChildren(node, context); + } + public T visitUnresolvedAttribute(UnresolvedAttribute node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index 99d8aaa882..c13dc53ea3 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -19,6 +19,7 @@ import org.opensearch.sql.ast.expression.Case; import org.opensearch.sql.ast.expression.Cast; import org.opensearch.sql.ast.expression.Compare; +import org.opensearch.sql.ast.expression.ConstantFunction; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.EqualTo; import org.opensearch.sql.ast.expression.Field; @@ -234,6 +235,10 @@ public static Function function(String funcName, UnresolvedExpression... funcArg return new Function(funcName, Arrays.asList(funcArgs)); } + public static Function constantFunction(String funcName, UnresolvedExpression... funcArgs) { + return new ConstantFunction(funcName, Arrays.asList(funcArgs)); + } + /** * CASE * WHEN search_condition THEN result_expr diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/ConstantFunction.java b/core/src/main/java/org/opensearch/sql/ast/expression/ConstantFunction.java new file mode 100644 index 0000000000..f14e65eeb2 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/expression/ConstantFunction.java @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.ast.expression; + +import java.util.List; +import lombok.EqualsAndHashCode; +import org.opensearch.sql.ast.AbstractNodeVisitor; + +/** + * Expression node that holds a function which should be replaced by its constant[1] value. + * [1] Constant at execution time. + */ +@EqualsAndHashCode(callSuper = false) +public class ConstantFunction extends Function { + + public ConstantFunction(String funcName, List funcArgs) { + super(funcName, funcArgs); + } + + @Override + public R accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitConstantFunction(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index bd2d075613..a094d2e487 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -682,6 +682,42 @@ public FunctionExpression match_bool_prefix(Expression... args) { return compile(BuiltinFunctionName.MATCH_BOOL_PREFIX, args); } + public FunctionExpression now(Expression... args) { + return compile(BuiltinFunctionName.NOW, args); + } + + public FunctionExpression current_timestamp(Expression... args) { + return compile(BuiltinFunctionName.CURRENT_TIMESTAMP, args); + } + + public FunctionExpression localtimestamp(Expression... args) { + return compile(BuiltinFunctionName.LOCALTIMESTAMP, args); + } + + public FunctionExpression localtime(Expression... args) { + return compile(BuiltinFunctionName.LOCALTIME, args); + } + + public FunctionExpression sysdate(Expression... args) { + return compile(BuiltinFunctionName.SYSDATE, args); + } + + public FunctionExpression curtime(Expression... args) { + return compile(BuiltinFunctionName.CURTIME, args); + } + + public FunctionExpression current_time(Expression... args) { + return compile(BuiltinFunctionName.CURRENT_TIME, args); + } + + public FunctionExpression curdate(Expression... args) { + return compile(BuiltinFunctionName.CURDATE, args); + } + + public FunctionExpression current_date(Expression... args) { + return compile(BuiltinFunctionName.CURRENT_DATE, args); + } + private FunctionExpression compile(BuiltinFunctionName bfn, Expression... args) { return (FunctionExpression) repository.compile(bfn.getName(), Arrays.asList(args.clone())); } diff --git a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java index 469f7e2011..d8dc7fc85f 100644 --- a/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/datetime/DateTimeFunction.java @@ -19,12 +19,16 @@ import static org.opensearch.sql.expression.function.FunctionDSL.impl; import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling; +import java.math.BigDecimal; +import java.math.RoundingMode; import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.LocalTime; import java.time.format.DateTimeFormatter; import java.time.format.TextStyle; import java.util.Locale; import java.util.concurrent.TimeUnit; +import javax.annotation.Nullable; import lombok.experimental.UtilityClass; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDatetimeValue; @@ -85,6 +89,84 @@ public void register(BuiltinFunctionRepository repository) { repository.register(to_days()); repository.register(week()); repository.register(year()); + + repository.register(now()); + repository.register(current_timestamp()); + repository.register(localtimestamp()); + repository.register(localtime()); + repository.register(sysdate()); + repository.register(curtime()); + repository.register(current_time()); + repository.register(curdate()); + repository.register(current_date()); + } + + /** + * NOW() returns a constant time that indicates the time at which the statement began to execute. + * `fsp` argument support is removed until refactoring to avoid bug where `now()`, `now(x)` and + * `now(y) return different values. + */ + private FunctionResolver now(FunctionName functionName) { + return define(functionName, + impl(() -> new ExprDatetimeValue(formatNow(null)), DATETIME) + ); + } + + private FunctionResolver now() { + return now(BuiltinFunctionName.NOW.getName()); + } + + private FunctionResolver current_timestamp() { + return now(BuiltinFunctionName.CURRENT_TIMESTAMP.getName()); + } + + private FunctionResolver localtimestamp() { + return now(BuiltinFunctionName.LOCALTIMESTAMP.getName()); + } + + private FunctionResolver localtime() { + return now(BuiltinFunctionName.LOCALTIME.getName()); + } + + /** + * SYSDATE() returns the time at which it executes. + */ + private FunctionResolver sysdate() { + return define(BuiltinFunctionName.SYSDATE.getName(), + impl(() -> new ExprDatetimeValue(formatNow(null)), DATETIME), + impl((v) -> new ExprDatetimeValue(formatNow(v.integerValue())), DATETIME, INTEGER) + ); + } + + /** + * Synonym for @see `now`. + */ + private FunctionResolver curtime(FunctionName functionName) { + return define(functionName, + impl(() -> new ExprTimeValue(formatNow(null).toLocalTime()), TIME) + ); + } + + private FunctionResolver curtime() { + return curtime(BuiltinFunctionName.CURTIME.getName()); + } + + private FunctionResolver current_time() { + return curtime(BuiltinFunctionName.CURRENT_TIME.getName()); + } + + private FunctionResolver curdate(FunctionName functionName) { + return define(functionName, + impl(() -> new ExprDateValue(formatNow(null).toLocalDate()), DATE) + ); + } + + private FunctionResolver curdate() { + return curdate(BuiltinFunctionName.CURDATE.getName()); + } + + private FunctionResolver current_date() { + return curdate(BuiltinFunctionName.CURRENT_DATE.getName()); } /** @@ -742,4 +824,24 @@ private ExprValue exprYear(ExprValue date) { return new ExprIntegerValue(date.dateValue().getYear()); } + /** + * Prepare LocalDateTime value. Truncate fractional second part according to the argument. + * @param fsp argument is given to specify a fractional seconds precision from 0 to 6, + * the return value includes a fractional seconds part of that many digits. + * @return LocalDateTime object. + */ + private LocalDateTime formatNow(@Nullable Integer fsp) { + var res = LocalDateTime.now(); + if (fsp == null) { + fsp = 0; + } + var defaultPrecision = 9; // There are 10^9 nanoseconds in one second + if (fsp < 0 || fsp > 6) { // Check that the argument is in the allowed range [0, 6] + throw new IllegalArgumentException( + String.format("Invalid `fsp` value: %d, allowed 0 to 6", fsp)); + } + var nano = new BigDecimal(res.getNano()) + .setScale(fsp - defaultPrecision, RoundingMode.DOWN).intValue(); + return res.withNano(nano); + } } diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index b3821d6e41..3a3db4201e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -84,7 +84,16 @@ public enum BuiltinFunctionName { TO_DAYS(FunctionName.of("to_days")), WEEK(FunctionName.of("week")), YEAR(FunctionName.of("year")), - + // `now`-like functions + NOW(FunctionName.of("now")), + CURDATE(FunctionName.of("curdate")), + CURRENT_DATE(FunctionName.of("current_date")), + CURTIME(FunctionName.of("curtime")), + CURRENT_TIME(FunctionName.of("current_time")), + LOCALTIME(FunctionName.of("localtime")), + CURRENT_TIMESTAMP(FunctionName.of("current_timestamp")), + LOCALTIMESTAMP(FunctionName.of("localtimestamp")), + SYSDATE(FunctionName.of("sysdate")), /** * Text Functions. */ diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java index c8ce70c418..e3b1ac7e6a 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java @@ -8,7 +8,9 @@ import static java.util.Collections.emptyList; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.ast.dsl.AstDSL.field; import static org.opensearch.sql.ast.dsl.AstDSL.floatLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.function; @@ -27,6 +29,7 @@ import com.google.common.collect.ImmutableMap; import java.util.Collections; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -46,7 +49,9 @@ import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.FunctionExpression; import org.opensearch.sql.expression.HighlightExpression; +import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.config.ExpressionConfig; import org.opensearch.sql.expression.window.aggregation.AggregateWindowFunction; import org.springframework.context.annotation.Configuration; @@ -546,6 +551,47 @@ public void match_phrase_prefix_all_params() { ); } + @Test + public void constant_function_is_calculated_on_analyze() { + // Actually, we can call any function as ConstantFunction to be calculated on analyze stage + assertTrue(analyze(AstDSL.constantFunction("now")) instanceof LiteralExpression); + assertTrue(analyze(AstDSL.constantFunction("localtime")) instanceof LiteralExpression); + } + + @Test + public void function_isnt_calculated_on_analyze() { + assertTrue(analyze(function("now")) instanceof FunctionExpression); + assertTrue(analyze(AstDSL.function("localtime")) instanceof FunctionExpression); + } + + @Test + public void constant_function_returns_constant_cached_value() { + var values = List.of(analyze(AstDSL.constantFunction("now")), + analyze(AstDSL.constantFunction("now")), analyze(AstDSL.constantFunction("now"))); + assertTrue(values.stream().allMatch(v -> + v.valueOf(null) == analyze(AstDSL.constantFunction("now")).valueOf(null))); + } + + @Test + public void function_returns_non_constant_value() { + // Even a function returns the same values - they are calculated on each call + // `sysdate()` which returns `LocalDateTime.now()` shouldn't be cached and should return always + // different values + var values = List.of(analyze(function("sysdate")), analyze(function("sysdate")), + analyze(function("sysdate")), analyze(function("sysdate"))); + var referenceValue = analyze(function("sysdate")).valueOf(null); + assertTrue(values.stream().noneMatch(v -> v.valueOf(null) == referenceValue)); + } + + @Test + public void now_as_a_function_not_cached() { + // // We can call `now()` as a function, in that case nothing should be cached + var values = List.of(analyze(function("now")), analyze(function("now")), + analyze(function("now")), analyze(function("now"))); + var referenceValue = analyze(function("now")).valueOf(null); + assertTrue(values.stream().noneMatch(v -> v.valueOf(null) == referenceValue)); + } + @Test void highlight() { assertAnalyzeEqual(new HighlightExpression(DSL.literal("fieldA")), diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java new file mode 100644 index 0000000000..e8f5c16025 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/NowLikeFunctionTest.java @@ -0,0 +1,128 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.expression.datetime; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.opensearch.sql.data.type.ExprCoreType.DATE; +import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; +import static org.opensearch.sql.data.type.ExprCoreType.TIME; + +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.Period; +import java.time.temporal.Temporal; +import java.util.List; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Stream; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.expression.DSL; +import org.opensearch.sql.expression.Expression; +import org.opensearch.sql.expression.ExpressionTestBase; +import org.opensearch.sql.expression.FunctionExpression; +import org.opensearch.sql.expression.config.ExpressionConfig; + + +public class NowLikeFunctionTest extends ExpressionTestBase { + private static Stream functionNames() { + var dsl = new DSL(new ExpressionConfig().functionRepository()); + return Stream.of( + Arguments.of((Function)dsl::now, + "now", DATETIME, false, (Supplier)LocalDateTime::now), + Arguments.of((Function)dsl::current_timestamp, + "current_timestamp", DATETIME, false, (Supplier)LocalDateTime::now), + Arguments.of((Function)dsl::localtimestamp, + "localtimestamp", DATETIME, false, (Supplier)LocalDateTime::now), + Arguments.of((Function)dsl::localtime, + "localtime", DATETIME, false, (Supplier)LocalDateTime::now), + Arguments.of((Function)dsl::sysdate, + "sysdate", DATETIME, true, (Supplier)LocalDateTime::now), + Arguments.of((Function)dsl::curtime, + "curtime", TIME, false, (Supplier)LocalTime::now), + Arguments.of((Function)dsl::current_time, + "current_time", TIME, false, (Supplier)LocalTime::now), + Arguments.of((Function)dsl::curdate, + "curdate", DATE, false, (Supplier)LocalDate::now), + Arguments.of((Function)dsl::current_date, + "current_date", DATE, false, (Supplier)LocalDate::now)); + } + + private Temporal extractValue(FunctionExpression func) { + switch ((ExprCoreType)func.type()) { + case DATE: return func.valueOf(null).dateValue(); + case DATETIME: return func.valueOf(null).datetimeValue(); + case TIME: return func.valueOf(null).timeValue(); + // unreachable code + default: throw new IllegalArgumentException(String.format("%s", func.type())); + } + } + + private long getDiff(Temporal sample, Temporal reference) { + if (sample instanceof LocalDate) { + return Period.between((LocalDate) sample, (LocalDate) reference).getDays(); + } + return Duration.between(sample, reference).toSeconds(); + } + + /** + * Check how NOW-like functions are processed. + * @param function Function + * @param name Function name + * @param resType Return type + * @param hasFsp Whether function has fsp argument + * @param referenceGetter A callback to get reference value + */ + @ParameterizedTest(name = "{1}") + @MethodSource("functionNames") + public void test_now_like_functions(Function function, + @SuppressWarnings("unused") // Used in the test name above + String name, + ExprCoreType resType, + Boolean hasFsp, + Supplier referenceGetter) { + // Check return types: + // `func()` + FunctionExpression expr = function.apply(new Expression[]{}); + assertEquals(resType, expr.type()); + if (hasFsp) { + // `func(fsp = 0)` + expr = function.apply(new Expression[]{DSL.literal(0)}); + assertEquals(resType, expr.type()); + // `func(fsp = 6)` + expr = function.apply(new Expression[]{DSL.literal(6)}); + assertEquals(resType, expr.type()); + + for (var wrongFspValue: List.of(-1, 10)) { + var exception = assertThrows(IllegalArgumentException.class, + () -> function.apply(new Expression[]{DSL.literal(wrongFspValue)}).valueOf(null)); + assertEquals(String.format("Invalid `fsp` value: %d, allowed 0 to 6", wrongFspValue), + exception.getMessage()); + } + } + + // Check how calculations are precise: + // `func()` + assertTrue(Math.abs(getDiff( + extractValue(function.apply(new Expression[]{})), + referenceGetter.get() + )) <= 1); + if (hasFsp) { + // `func(fsp)` + assertTrue(Math.abs(getDiff( + extractValue(function.apply(new Expression[]{DSL.literal(0)})), + referenceGetter.get() + )) <= 1); + } + } +} diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 736b1f148e..e914803243 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -1383,9 +1383,189 @@ NOW Description >>>>>>>>>>> -Specifications: +Returns the current date and time as a value in 'YYYY-MM-DD hh:mm:ss.nnnnnn' format. The value is expressed in the cluster time zone. +`NOW()` returns a constant time that indicates the time at which the statement began to execute. This differs from the behavior for `SYSDATE() <#sysdate>`_, which returns the exact time at which it executes. + +Return type: DATETIME + +Specification: NOW() -> DATETIME + +Example:: + + > SELECT NOW() as value_1, NOW() as value_2; + fetched rows / total rows = 1/1 + +----------------------------+----------------------------+ + | value_1 | value_2 | + |----------------------------+----------------------------| + | 2022-08-02 15:39:05.173069 | 2022-08-02 15:39:05.173069 | + +----------------------------+----------------------------+ + + +CURRENT_TIMESTAMP +----------------- + +Description +>>>>>>>>>>> + +`CURRENT_TIMESTAMP` and `CURRENT_TIMESTAMP()` are synonyms for `NOW() <#now>`_. + +Example:: + + > SELECT CURRENT_TIMESTAMP(), CURRENT_TIMESTAMP; + fetched rows / total rows = 1/1 + +----------------------------+----------------------------+ + | CURRENT_TIMESTAMP() | CURRENT_TIMESTAMP | + |----------------------------+----------------------------| + | 2022-08-02 15:54:19.209361 | 2022-08-02 15:54:19.209361 | + +----------------------------+----------------------------+ + + +LOCALTIMESTAMP +-------------- + +Description +>>>>>>>>>>> + +`LOCALTIMESTAMP` and `LOCALTIMESTAMP()` are synonyms for `NOW() <#now>`_. + +Example:: + + > SELECT LOCALTIMESTAMP(), LOCALTIMESTAMP; + fetched rows / total rows = 1/1 + +----------------------------+----------------------------+ + | LOCALTIMESTAMP() | LOCALTIMESTAMP | + |----------------------------+----------------------------| + | 2022-08-02 15:54:19.209361 | 2022-08-02 15:54:19.209361 | + +----------------------------+----------------------------+ + + +LOCALTIME +--------- + +Description +>>>>>>>>>>> + +`LOCALTIME` and `LOCALTIME()` are synonyms for `NOW() <#now>`_. + +Example:: + + > SELECT LOCALTIME(), LOCALTIME; + fetched rows / total rows = 1/1 + +----------------------------+----------------------------+ + | LOCALTIME() | LOCALTIME | + |----------------------------+----------------------------| + | 2022-08-02 15:54:19.209361 | 2022-08-02 15:54:19.209361 | + +----------------------------+----------------------------+ + + +SYSDATE +------- + +Description +>>>>>>>>>>> + +Returns the current date and time as a value in 'YYYY-MM-DD hh:mm:ss.nnnnnn'. +SYSDATE() returns the time at which it executes. This differs from the behavior for `NOW() <#now>`_, which returns a constant time that indicates the time at which the statement began to execute. + +Return type: DATETIME -1. NOW() -> DATE +Specification: SYSDATE() -> DATETIME + +Example:: + + > SELECT SYSDATE() as value_1, SYSDATE() as value_2; + fetched rows / total rows = 1/1 + +----------------------------+----------------------------+ + | value_1 | value_2 | + |----------------------------+----------------------------| + | 2022-08-02 15:39:05.173069 | 2022-08-02 15:39:05.173142 | + +----------------------------+----------------------------+ + + +CURTIME +------- + +Description +>>>>>>>>>>> + +Returns the current time as a value in 'hh:mm:ss.nnnnnn'. +CURTIME() returns the time at which the statement began to execute as `NOW() <#now>`_ does. + +Return type: TIME + +Specification: CURTIME() -> TIME + +Example:: + + > SELECT CURTIME() as value_1, CURTIME() as value_2; + fetched rows / total rows = 1/1 + +-----------------+-----------------+ + | value_1 | value_2 | + |-----------------+-----------------| + | 15:39:05.173069 | 15:39:05.173069 | + +-----------------+-----------------+ + + +CURRENT_TIME +------------ + +Description +>>>>>>>>>>> + +`CURRENT_TIME` and `CURRENT_TIME()` are synonyms for `CURTIME() <#curtime>`_. + +Example:: + + > SELECT CURRENT_TIME(), CURRENT_TIME; + fetched rows / total rows = 1/1 + +-----------------+-----------------+ + | CURRENT_TIME() | CURRENT_TIME | + |-----------------+-----------------| + | 15:39:05.173069 | 15:39:05.173069 | + +-----------------+-----------------+ + + +CURDATE +------- + +Description +>>>>>>>>>>> + +Returns the current time as a value in 'YYYY-MM-DD'. +CURDATE() returns the time at which it executes as `SYSDATE() <#sysdate>`_ does. + +Return type: DATE + +Specification: CURDATE() -> DATE + +Example:: + + > SELECT CURDATE(); + fetched rows / total rows = 1/1 + +-------------+ + | CURDATE() | + |-------------| + | 2022-08-02 | + +-------------+ + + +CURRENT_DATE +------------ + +Description +>>>>>>>>>>> + +`CURRENT_DATE` and `CURRENT_DATE()` are synonyms for `CURDATE() <#curdate>`_. + +Example:: + + > SELECT CURRENT_DATE(), CURRENT_DATE; + fetched rows / total rows = 1/1 + +------------------+----------------+ + | CURRENT_DATE() | CURRENT_DATE | + |------------------+----------------| + | 2022-08-02 | 2022-08-02 | + +------------------+----------------+ QUARTER diff --git a/docs/user/ppl/functions/datetime.rst b/docs/user/ppl/functions/datetime.rst index 3680dc2272..dbbc2abe21 100644 --- a/docs/user/ppl/functions/datetime.rst +++ b/docs/user/ppl/functions/datetime.rst @@ -552,9 +552,189 @@ NOW Description >>>>>>>>>>> -Specifications: +Returns the current date and time as a value in 'YYYY-MM-DD hh:mm:ss.nnnnnn' format. The value is expressed in the cluster time zone. +`NOW()` returns a constant time that indicates the time at which the statement began to execute. This differs from the behavior for `SYSDATE() <#sysdate>`_, which returns the exact time at which it executes. + +Return type: DATETIME + +Specification: NOW() -> DATETIME + +Example:: + + > source=people | eval `value_1` = NOW(), `value_2` = NOW() | fields `value_1`, `value_2` + fetched rows / total rows = 1/1 + +----------------------------+----------------------------+ + | value_1 | value_2 | + |----------------------------+----------------------------| + | 2022-08-02 15:39:05.173069 | 2022-08-02 15:39:05.173069 | + +----------------------------+----------------------------+ + + +CURRENT_TIMESTAMP +----------------- + +Description +>>>>>>>>>>> + +`CURRENT_TIMESTAMP` and `CURRENT_TIMESTAMP()` are synonyms for `NOW() <#now>`_. + +Example:: + + > source=people | eval `CURRENT_TIMESTAMP()` = CURRENT_TIMESTAMP(), `CURRENT_TIMESTAMP` = CURRENT_TIMESTAMP | fields `CURRENT_TIMESTAMP()`, `CURRENT_TIMESTAMP` + fetched rows / total rows = 1/1 + +----------------------------+----------------------------+ + | CURRENT_TIMESTAMP() | CURRENT_TIMESTAMP | + |----------------------------+----------------------------| + | 2022-08-02 15:54:19.209361 | 2022-08-02 15:54:19.209361 | + +----------------------------+----------------------------+ + + +LOCALTIMESTAMP +-------------- + +Description +>>>>>>>>>>> + +`LOCALTIMESTAMP` and `LOCALTIMESTAMP()` are synonyms for `NOW() <#now>`_. + +Example:: + + > source=people | eval `LOCALTIMESTAMP()` = LOCALTIMESTAMP(), `LOCALTIMESTAMP` = LOCALTIMESTAMP | fields `LOCALTIMESTAMP()`, `LOCALTIMESTAMP` + fetched rows / total rows = 1/1 + +----------------------------+----------------------------+ + | LOCALTIMESTAMP() | LOCALTIMESTAMP | + |----------------------------+----------------------------| + | 2022-08-02 15:54:19.209361 | 2022-08-02 15:54:19.209361 | + +----------------------------+----------------------------+ + + +LOCALTIME +--------- + +Description +>>>>>>>>>>> + +`LOCALTIME` and `LOCALTIME()` are synonyms for `NOW() <#now>`_. + +Example:: + + > source=people | eval `LOCALTIME()` = LOCALTIME(), `LOCALTIME` = LOCALTIME | fields `LOCALTIME()`, `LOCALTIME` + fetched rows / total rows = 1/1 + +----------------------------+----------------------------+ + | LOCALTIME() | LOCALTIME | + |----------------------------+----------------------------| + | 2022-08-02 15:54:19.209361 | 2022-08-02 15:54:19.209361 | + +----------------------------+----------------------------+ + + +SYSDATE +------- + +Description +>>>>>>>>>>> + +Returns the current date and time as a value in 'YYYY-MM-DD hh:mm:ss.nnnnnn'. +SYSDATE() returns the time at which it executes. This differs from the behavior for `NOW() <#now>`_, which returns a constant time that indicates the time at which the statement began to execute. + +Return type: DATETIME + +Specification: SYSDATE() -> DATETIME + +Example:: -1. NOW() -> DATE + > source=people | eval `value_1` = SYSDATE(), `value_2` = SYSDATE() | fields `value_1`, `value_2` + fetched rows / total rows = 1/1 + +----------------------------+----------------------------+ + | value_1 | value_2 | + |----------------------------+----------------------------| + | 2022-08-02 15:39:05.173069 | 2022-08-02 15:39:05.173142 | + +----------------------------+----------------------------+ + + +CURTIME +------- + +Description +>>>>>>>>>>> + +Returns the current time as a value in 'hh:mm:ss.nnnnnn'. +CURTIME() returns the time at which the statement began to execute as `NOW() <#now>`_ does. + +Return type: TIME + +Specification: CURTIME() -> TIME + +Example:: + + > source=people | eval `value_1` = CURTIME(), `value_2` = CURTIME() | fields `value_1`, `value_2` + fetched rows / total rows = 1/1 + +-----------------+-----------------+ + | value_1 | value_2 | + |-----------------+-----------------| + | 15:39:05.173069 | 15:39:05.173069 | + +-----------------+-----------------+ + + +CURRENT_TIME +------------ + +Description +>>>>>>>>>>> + +`CURRENT_TIME` and `CURRENT_TIME()` are synonyms for `CURTIME() <#curtime>`_. + +Example:: + + > source=people | eval `CURRENT_TIME()` = CURRENT_TIME(), `CURRENT_TIME` = CURRENT_TIME | fields `CURRENT_TIME()`, `CURRENT_TIME` + fetched rows / total rows = 1/1 + +------------------+-----------------+ + | CURRENT_TIME() | CURRENT_TIME | + |------------------+-----------------| + | 15:39:05.173069 | 15:39:05.173069 | + +------------------+-----------------+ + + +CURDATE +------- + +Description +>>>>>>>>>>> + +Returns the current time as a value in 'YYYY-MM-DD'. +CURDATE() returns the time at which it executes as `SYSDATE() <#sysdate>`_ does. + +Return type: DATE + +Specification: CURDATE() -> DATE + +Example:: + + > source=people | eval `CURDATE()` = CURDATE() | fields `CURDATE()` + fetched rows / total rows = 1/1 + +-------------+ + | CURDATE() | + |-------------| + | 2022-08-02 | + +-------------+ + + +CURRENT_DATE +------------ + +Description +>>>>>>>>>>> + +`CURRENT_DATE` and `CURRENT_DATE()` are synonyms for `CURDATE() <#curdate>`_. + +Example:: + + > source=people | eval `CURRENT_DATE()` = CURRENT_DATE(), `CURRENT_DATE` = CURRENT_DATE | fields `CURRENT_DATE()`, `CURRENT_DATE` + fetched rows / total rows = 1/1 + +------------------+----------------+ + | CURRENT_DATE() | CURRENT_DATE | + |------------------+----------------| + | 2022-08-02 | 2022-08-02 | + +------------------+----------------+ QUARTER diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java index 7e0169d174..a0b0e8673b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/DateTimeFunctionIT.java @@ -7,6 +7,7 @@ package org.opensearch.sql.ppl; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_PEOPLE2; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifySchema; @@ -14,7 +15,24 @@ import java.io.IOException; import java.time.LocalTime; - +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.Period; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.time.temporal.Temporal; +import java.util.ArrayList; +import java.util.List; +import java.util.TimeZone; +import java.util.function.BiFunction; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import com.google.common.collect.ImmutableMap; +import org.json.JSONArray; +import java.time.LocalTime; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.common.utils.StringUtils; @@ -24,6 +42,7 @@ public class DateTimeFunctionIT extends PPLIntegTestCase { @Override public void init() throws IOException { loadIndex(Index.DATE); + loadIndex(Index.PEOPLE2); } @Test @@ -480,4 +499,167 @@ public void testMakeDate() throws IOException { verifySchema(result, schema("f1", null, "date"), schema("f2", null, "date")); verifySome(result.getJSONArray("datarows"), rows("1945-01-06", "1989-06-06")); } + + private List> nowLikeFunctionsData() { + return List.of( + ImmutableMap.builder() + .put("name", "now") + .put("hasFsp", false) + .put("hasShortcut", false) + .put("constValue", true) + .put("referenceGetter", (Supplier) LocalDateTime::now) + .put("parser", (BiFunction) LocalDateTime::parse) + .put("serializationPattern", "uuuu-MM-dd HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "current_timestamp") + .put("hasFsp", false) + .put("hasShortcut", true) + .put("constValue", true) + .put("referenceGetter", (Supplier) LocalDateTime::now) + .put("parser", (BiFunction) LocalDateTime::parse) + .put("serializationPattern", "uuuu-MM-dd HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "localtimestamp") + .put("hasFsp", false) + .put("hasShortcut", true) + .put("constValue", true) + .put("referenceGetter", (Supplier) LocalDateTime::now) + .put("parser", (BiFunction) LocalDateTime::parse) + .put("serializationPattern", "uuuu-MM-dd HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "localtime") + .put("hasFsp", false) + .put("hasShortcut", true) + .put("constValue", true) + .put("referenceGetter", (Supplier) LocalDateTime::now) + .put("parser", (BiFunction) LocalDateTime::parse) + .put("serializationPattern", "uuuu-MM-dd HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "sysdate") + .put("hasFsp", true) + .put("hasShortcut", false) + .put("constValue", false) + .put("referenceGetter", (Supplier) LocalDateTime::now) + .put("parser", (BiFunction) LocalDateTime::parse) + .put("serializationPattern", "uuuu-MM-dd HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "curtime") + .put("hasFsp", false) + .put("hasShortcut", false) + .put("constValue", false) + .put("referenceGetter", (Supplier) LocalTime::now) + .put("parser", (BiFunction) LocalTime::parse) + .put("serializationPattern", "HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "current_time") + .put("hasFsp", false) + .put("hasShortcut", true) + .put("constValue", false) + .put("referenceGetter", (Supplier) LocalTime::now) + .put("parser", (BiFunction) LocalTime::parse) + .put("serializationPattern", "HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "curdate") + .put("hasFsp", false) + .put("hasShortcut", false) + .put("constValue", false) + .put("referenceGetter", (Supplier) LocalDate::now) + .put("parser", (BiFunction) LocalDate::parse) + .put("serializationPattern", "uuuu-MM-dd") + .build(), + ImmutableMap.builder() + .put("name", "current_date") + .put("hasFsp", false) + .put("hasShortcut", true) + .put("constValue", false) + .put("referenceGetter", (Supplier) LocalDate::now) + .put("parser", (BiFunction) LocalDate::parse) + .put("serializationPattern", "uuuu-MM-dd") + .build() + ); + } + + private long getDiff(Temporal sample, Temporal reference) { + if (sample instanceof LocalDate) { + return Period.between((LocalDate) sample, (LocalDate) reference).getDays(); + } + return Duration.between(sample, reference).toSeconds(); + } + + @Test + public void testNowLikeFunctions() throws IOException { + // Integration test framework sets for OpenSearch instance a random timezone. + // If server's TZ doesn't match localhost's TZ, time measurements for `now` would differ. + // We should set localhost's TZ now and recover the value back in the end of the test. + var testTz = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone(System.getProperty("user.timezone"))); + + for (var funcData : nowLikeFunctionsData()) { + String name = (String) funcData.get("name"); + Boolean hasFsp = (Boolean) funcData.get("hasFsp"); + Boolean hasShortcut = (Boolean) funcData.get("hasShortcut"); + Boolean constValue = (Boolean) funcData.get("constValue"); + Supplier referenceGetter = (Supplier) funcData.get("referenceGetter"); + BiFunction parser = + (BiFunction) funcData.get("parser"); + String serializationPatternStr = (String) funcData.get("serializationPattern"); + + var serializationPattern = new DateTimeFormatterBuilder() + .appendPattern(serializationPatternStr) + .optionalStart() + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .toFormatter(); + + Temporal reference = referenceGetter.get(); + double delta = 2d; // acceptable time diff, secs + if (reference instanceof LocalDate) + delta = 1d; // Max date delta could be 1 if test runs on the very edge of two days + // We ignore probability of a test run on edge of month or year to simplify the checks + + var calls = new ArrayList() {{ + add(name + "()"); + }}; + if (hasShortcut) + calls.add(name); + if (hasFsp) + calls.add(name + "(0)"); + + // Column order is: func(), func, func(0) + // shortcut ^ fsp ^ + // Query looks like: + // source=people2 | eval `now()`=now() | fields `now()`; + JSONObject result = executeQuery("source=" + TEST_INDEX_PEOPLE2 + + " | eval " + calls.stream().map(c -> String.format("`%s`=%s", c, c)).collect(Collectors.joining(",")) + + " | fields " + calls.stream().map(c -> String.format("`%s`", c)).collect(Collectors.joining(","))); + + var rows = result.getJSONArray("datarows"); + JSONArray firstRow = rows.getJSONArray(0); + for (int i = 0; i < rows.length(); i++) { + var row = rows.getJSONArray(i); + if (constValue) + assertTrue(firstRow.similar(row)); + + int column = 0; + assertEquals(0, + getDiff(reference, parser.apply(row.getString(column++), serializationPattern)), delta); + + if (hasShortcut) { + assertEquals(0, + getDiff(reference, parser.apply(row.getString(column++), serializationPattern)), delta); + } + if (hasFsp) { + assertEquals(0, + getDiff(reference, parser.apply(row.getString(column), serializationPattern)), delta); + } + } + } + TimeZone.setDefault(testTz); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java index 7c6bd7efe2..db4bec2540 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/DateTimeFunctionIT.java @@ -8,6 +8,8 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_PEOPLE2; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATE; import static org.opensearch.sql.legacy.plugin.RestSqlAction.QUERY_API_ENDPOINT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -17,7 +19,23 @@ import static org.opensearch.sql.util.TestUtils.getResponseBody; import java.io.IOException; +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.Period; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.time.temporal.Temporal; +import java.util.ArrayList; +import java.util.List; import java.util.Locale; +import java.util.TimeZone; +import java.util.function.BiFunction; +import java.util.function.Supplier; +import com.google.common.collect.ImmutableMap; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; @@ -32,6 +50,7 @@ public class DateTimeFunctionIT extends SQLIntegTestCase { public void init() throws Exception { super.init(); loadIndex(Index.BANK); + loadIndex(Index.PEOPLE2); } @Test @@ -454,7 +473,6 @@ public void testDateFormat() throws IOException { verifyDateFormat(date, "date", dateFormat, dateFormatted); } - @Test public void testMakeTime() throws IOException { var result = executeQuery(String.format( @@ -471,6 +489,165 @@ public void testMakeDate() throws IOException { verifySome(result.getJSONArray("datarows"), rows("1945-01-06", "1989-06-06")); } + private List> nowLikeFunctionsData() { + return List.of( + ImmutableMap.builder() + .put("name", "now") + .put("hasFsp", false) + .put("hasShortcut", false) + .put("constValue", true) + .put("referenceGetter", (Supplier) LocalDateTime::now) + .put("parser", (BiFunction) LocalDateTime::parse) + .put("serializationPattern", "uuuu-MM-dd HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "current_timestamp") + .put("hasFsp", false) + .put("hasShortcut", true) + .put("constValue", true) + .put("referenceGetter", (Supplier) LocalDateTime::now) + .put("parser", (BiFunction) LocalDateTime::parse) + .put("serializationPattern", "uuuu-MM-dd HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "localtimestamp") + .put("hasFsp", false) + .put("hasShortcut", true) + .put("constValue", true) + .put("referenceGetter", (Supplier) LocalDateTime::now) + .put("parser", (BiFunction) LocalDateTime::parse) + .put("serializationPattern", "uuuu-MM-dd HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "localtime") + .put("hasFsp", false) + .put("hasShortcut", true) + .put("constValue", true) + .put("referenceGetter", (Supplier) LocalDateTime::now) + .put("parser", (BiFunction) LocalDateTime::parse) + .put("serializationPattern", "uuuu-MM-dd HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "sysdate") + .put("hasFsp", true) + .put("hasShortcut", false) + .put("constValue", false) + .put("referenceGetter", (Supplier) LocalDateTime::now) + .put("parser", (BiFunction) LocalDateTime::parse) + .put("serializationPattern", "uuuu-MM-dd HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "curtime") + .put("hasFsp", false) + .put("hasShortcut", false) + .put("constValue", false) + .put("referenceGetter", (Supplier) LocalTime::now) + .put("parser", (BiFunction) LocalTime::parse) + .put("serializationPattern", "HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "current_time") + .put("hasFsp", false) + .put("hasShortcut", true) + .put("constValue", false) + .put("referenceGetter", (Supplier) LocalTime::now) + .put("parser", (BiFunction) LocalTime::parse) + .put("serializationPattern", "HH:mm:ss") + .build(), + ImmutableMap.builder() + .put("name", "curdate") + .put("hasFsp", false) + .put("hasShortcut", false) + .put("constValue", false) + .put("referenceGetter", (Supplier) LocalDate::now) + .put("parser", (BiFunction) LocalDate::parse) + .put("serializationPattern", "uuuu-MM-dd") + .build(), + ImmutableMap.builder() + .put("name", "current_date") + .put("hasFsp", false) + .put("hasShortcut", true) + .put("constValue", false) + .put("referenceGetter", (Supplier) LocalDate::now) + .put("parser", (BiFunction) LocalDate::parse) + .put("serializationPattern", "uuuu-MM-dd") + .build() + ); + } + + private long getDiff(Temporal sample, Temporal reference) { + if (sample instanceof LocalDate) { + return Period.between((LocalDate) sample, (LocalDate) reference).getDays(); + } + return Duration.between(sample, reference).toSeconds(); + } + + @Test + public void testNowLikeFunctions() throws IOException { + // Integration test framework sets for OpenSearch instance a random timezone. + // If server's TZ doesn't match localhost's TZ, time measurements for `now` would differ. + // We should set localhost's TZ now and recover the value back in the end of the test. + var testTz = TimeZone.getDefault(); + TimeZone.setDefault(TimeZone.getTimeZone(System.getProperty("user.timezone"))); + + for (var funcData : nowLikeFunctionsData()) { + String name = (String) funcData.get("name"); + Boolean hasFsp = (Boolean) funcData.get("hasFsp"); + Boolean hasShortcut = (Boolean) funcData.get("hasShortcut"); + Boolean constValue = (Boolean) funcData.get("constValue"); + Supplier referenceGetter = (Supplier) funcData.get("referenceGetter"); + BiFunction parser = + (BiFunction) funcData.get("parser"); + String serializationPatternStr = (String) funcData.get("serializationPattern"); + + var serializationPattern = new DateTimeFormatterBuilder() + .appendPattern(serializationPatternStr) + .optionalStart() + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .toFormatter(); + + Temporal reference = referenceGetter.get(); + double delta = 2d; // acceptable time diff, secs + if (reference instanceof LocalDate) + delta = 1d; // Max date delta could be 1 if test runs on the very edge of two days + // We ignore probability of a test run on edge of month or year to simplify the checks + + var calls = new ArrayList() {{ + add(name + "()"); + }}; + if (hasShortcut) + calls.add(name); + if (hasFsp) + calls.add(name + "(0)"); + + // Column order is: func(), func, func(0) + // shortcut ^ fsp ^ + JSONObject result = executeQuery("select " + String.join(", ", calls) + " from " + TEST_INDEX_PEOPLE2); + + var rows = result.getJSONArray("datarows"); + JSONArray firstRow = rows.getJSONArray(0); + for (int i = 0; i < rows.length(); i++) { + var row = rows.getJSONArray(i); + if (constValue) + assertTrue(firstRow.similar(row)); + + int column = 0; + assertEquals(0, + getDiff(reference, parser.apply(row.getString(column++), serializationPattern)), delta); + + if (hasShortcut) { + assertEquals(0, + getDiff(reference, parser.apply(row.getString(column++), serializationPattern)), delta); + } + if (hasFsp) { + assertEquals(0, + getDiff(reference, parser.apply(row.getString(column), serializationPattern)), delta); + } + } + } + TimeZone.setDefault(testTz); + } + protected JSONObject executeQuery(String query) throws IOException { Request request = new Request("POST", QUERY_API_ENDPOINT); request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 93df64d0b3..4a48f00964 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -223,23 +223,36 @@ TAN: 'TAN'; // DATE AND TIME FUNCTIONS ADDDATE: 'ADDDATE'; +CURDATE: 'CURDATE'; +CURRENT_DATE: 'CURRENT_DATE'; +CURRENT_TIME: 'CURRENT_TIME'; +CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; +CURTIME: 'CURTIME'; DATE: 'DATE'; DATE_ADD: 'DATE_ADD'; +DATE_FORMAT: 'DATE_FORMAT'; DATE_SUB: 'DATE_SUB'; DAYOFMONTH: 'DAYOFMONTH'; DAYOFWEEK: 'DAYOFWEEK'; DAYOFYEAR: 'DAYOFYEAR'; DAYNAME: 'DAYNAME'; FROM_DAYS: 'FROM_DAYS'; +LOCALTIME: 'LOCALTIME'; +LOCALTIMESTAMP: 'LOCALTIMESTAMP'; MAKEDATE: 'MAKEDATE'; MAKETIME: 'MAKETIME'; MONTHNAME: 'MONTHNAME'; +NOW: 'NOW'; SUBDATE: 'SUBDATE'; +SYSDATE: 'SYSDATE'; TIME: 'TIME'; TIME_TO_SEC: 'TIME_TO_SEC'; TIMESTAMP: 'TIMESTAMP'; -DATE_FORMAT: 'DATE_FORMAT'; TO_DAYS: 'TO_DAYS'; +UTC_DATE: 'UTC_DATE'; +UTC_TIME: 'UTC_TIME'; +UTC_TIMESTAMP: 'UTC_TIMESTAMP'; + // TEXT FUNCTIONS SUBSTR: 'SUBSTR'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index c83297459d..1bd1140d8a 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -221,6 +221,11 @@ primaryExpression | dataTypeFunctionCall | fieldExpression | literalValue + | constantFunction + ; + +constantFunction + : constantFunctionName LT_PRTHS functionArgs? RT_PRTHS ; booleanExpression @@ -373,9 +378,14 @@ trigonometricFunctionName ; dateAndTimeFunctionBase - : ADDDATE | DATE | DATE_ADD | DATE_SUB | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK | DAYOFYEAR | FROM_DAYS - | HOUR | MICROSECOND | MINUTE | MONTH | MONTHNAME | QUARTER | SECOND | SUBDATE | TIME | TIME_TO_SEC - | TIMESTAMP | TO_DAYS | YEAR | WEEK | DATE_FORMAT | MAKETIME | MAKEDATE + : ADDDATE | DATE | DATE_ADD | DATE_FORMAT | DATE_SUB | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK + | DAYOFYEAR | FROM_DAYS | HOUR | MAKEDATE | MAKETIME | MICROSECOND | MINUTE | MONTH | MONTHNAME + | QUARTER | SECOND | SUBDATE | SYSDATE | TIME | TIME_TO_SEC| TIMESTAMP | TO_DAYS | WEEK | YEAR + ; + +constantFunctionName + : datetimeConstantLiteral + | CURDATE | CURTIME | NOW ; /** condition function return boolean value */ @@ -419,6 +429,7 @@ literalValue | integerLiteral | decimalLiteral | booleanLiteral + | datetimeLiteral //#datetime ; intervalLiteral @@ -441,6 +452,31 @@ booleanLiteral : TRUE | FALSE ; +// Date and Time Literal, follow ANSI 92 +datetimeLiteral + : dateLiteral + | timeLiteral + | timestampLiteral + | datetimeConstantLiteral + ; + +dateLiteral + : DATE date=stringLiteral + ; + +timeLiteral + : TIME time=stringLiteral + ; + +timestampLiteral + : TIMESTAMP timestamp=stringLiteral + ; + +// Actually, these constants are shortcuts to the corresponding functions +datetimeConstantLiteral + : CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | LOCALTIME | LOCALTIMESTAMP | UTC_TIMESTAMP | UTC_DATE | UTC_TIME + ; + pattern : stringLiteral ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 99483d2403..5df1c4ec56 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -14,14 +14,17 @@ import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BooleanLiteralContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.BySpanClauseContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.CompareExprContext; +import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.ConstantFunctionContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.ConvertedDataTypeContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.CountAllFunctionCallContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DataTypeFunctionCallContext; +import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DatetimeConstantLiteralContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DecimalLiteralContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DistinctCountFunctionCallContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.EvalClauseContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.EvalFunctionCallContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FieldExpressionContext; +import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FunctionArgsContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.IdentsAsQualifiedNameContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.IdentsAsWildcardQualifiedNameContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.InExprContext; @@ -59,6 +62,7 @@ import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.Cast; import org.opensearch.sql.ast.expression.Compare; +import org.opensearch.sql.ast.expression.ConstantFunction; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Function; @@ -216,14 +220,8 @@ public UnresolvedExpression visitPercentileAggFunction(PercentileAggFunctionCont @Override public UnresolvedExpression visitBooleanFunctionCall(BooleanFunctionCallContext ctx) { final String functionName = ctx.conditionFunctionBase().getText(); - - return new Function( - FUNCTION_NAME_MAPPING.getOrDefault(functionName, functionName), - ctx.functionArgs() - .functionArg() - .stream() - .map(this::visitFunctionArg) - .collect(Collectors.toList())); + return visitFunction(FUNCTION_NAME_MAPPING.getOrDefault(functionName, functionName), + ctx.functionArgs()); } /** @@ -231,13 +229,7 @@ public UnresolvedExpression visitBooleanFunctionCall(BooleanFunctionCallContext */ @Override public UnresolvedExpression visitEvalFunctionCall(EvalFunctionCallContext ctx) { - return new Function( - ctx.evalFunctionName().getText(), - ctx.functionArgs() - .functionArg() - .stream() - .map(this::visitFunctionArg) - .collect(Collectors.toList())); + return visitFunction(ctx.evalFunctionName().getText(), ctx.functionArgs()); } /** @@ -253,6 +245,37 @@ public UnresolvedExpression visitConvertedDataType(ConvertedDataTypeContext ctx) return AstDSL.stringLiteral(ctx.getText()); } + @Override + public UnresolvedExpression visitDatetimeConstantLiteral(DatetimeConstantLiteralContext ctx) { + return visitConstantFunction(ctx.getText(), null); + } + + public UnresolvedExpression visitConstantFunction(ConstantFunctionContext ctx) { + return visitConstantFunction(ctx.constantFunctionName().getText(), + ctx.functionArgs()); + } + + private UnresolvedExpression visitConstantFunction(String functionName, + FunctionArgsContext args) { + return new ConstantFunction(functionName, + args == null + ? Collections.emptyList() + : args.functionArg() + .stream() + .map(this::visitFunctionArg) + .collect(Collectors.toList())); + } + + private Function visitFunction(String functionName, FunctionArgsContext args) { + return new Function( + functionName, + args.functionArg() + .stream() + .map(this::visitFunctionArg) + .collect(Collectors.toList()) + ); + } + @Override public UnresolvedExpression visitSingleFieldRelevanceFunction( SingleFieldRelevanceFunctionContext ctx) { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/NowLikeFunctionParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/NowLikeFunctionParserTest.java new file mode 100644 index 0000000000..dda404f29a --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/NowLikeFunctionParserTest.java @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.ppl.antlr; + +import static org.junit.Assert.assertNotEquals; + +import java.util.List; +import org.antlr.v4.runtime.tree.ParseTree; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class NowLikeFunctionParserTest { + + private final PPLSyntaxParser parser = new PPLSyntaxParser(); + + /** + * Set parameterized values used in test. + * @param name Function name + * @param hasFsp Whether function has fsp argument + * @param hasShortcut Whether function has shortcut (call without `()`) + */ + public NowLikeFunctionParserTest(String name, Boolean hasFsp, Boolean hasShortcut) { + this.name = name; + this.hasFsp = hasFsp; + this.hasShortcut = hasShortcut; + } + + /** + * Returns function data to test. + * @return An iterable. + */ + @Parameterized.Parameters(name = "{0}") + public static Iterable functionNames() { + return List.of(new Object[][]{ + {"now", true, false}, + {"current_timestamp", true, true}, + {"localtimestamp", true, true}, + {"localtime", true, true}, + {"sysdate", true, false}, + {"curtime", true, false}, + {"current_time", true, true}, + {"curdate", false, false}, + {"current_date", false, true} + }); + } + + private final String name; + private final Boolean hasFsp; + private final Boolean hasShortcut; + + @Test + public void test_now_like_functions() { + for (var call : hasShortcut ? List.of(name, name + "()") : List.of(name + "()")) { + ParseTree tree = parser.parse("source=t | eval r=" + call); + assertNotEquals(null, tree); + + tree = parser.parse("search source=t | where a=" + call); + assertNotEquals(null, tree); + } + if (hasFsp) { + ParseTree tree = parser.parse("search source=t | where a=" + name + "(0)"); + assertNotEquals(null, tree); + } + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index bb3315d5c8..1becf086ac 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -7,6 +7,7 @@ package org.opensearch.sql.ppl.parser; import static java.util.Collections.emptyList; +import static org.junit.Assert.assertEquals; import static org.opensearch.sql.ast.dsl.AstDSL.agg; import static org.opensearch.sql.ast.dsl.AstDSL.aggregate; import static org.opensearch.sql.ast.dsl.AstDSL.alias; @@ -47,11 +48,13 @@ import java.util.Collections; import org.junit.Ignore; import org.junit.Test; +import org.opensearch.sql.ast.Node; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.RelevanceFieldList; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; public class AstExpressionBuilderTest extends AstBuilderTest { @@ -168,6 +171,18 @@ public void testEvalFunctionExpr() { )); } + @Test + public void testEvalFunctionExprNoArgs() { + assertEqual("source=t | eval f=PI()", + eval( + relation("t"), + let( + field("f"), + function("PI") + ) + )); + } + @Test public void testEvalBinaryOperationExpr() { assertEqual("source=t | eval f=a+b", @@ -715,4 +730,9 @@ public void canBuildQuery_stringRelevanceFunctionWithArguments() { ) ); } + + private Node buildExprAst(String query) { + AstBuilder astBuilder = new AstBuilder(new AstExpressionBuilder(), query); + return astBuilder.visit(new PPLSyntaxParser().parse(query)); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstNowLikeFunctionTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstNowLikeFunctionTest.java new file mode 100644 index 0000000000..6c6233a17f --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstNowLikeFunctionTest.java @@ -0,0 +1,111 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.ppl.parser; + +import static org.junit.Assert.assertEquals; +import static org.opensearch.sql.ast.dsl.AstDSL.compare; +import static org.opensearch.sql.ast.dsl.AstDSL.constantFunction; +import static org.opensearch.sql.ast.dsl.AstDSL.eval; +import static org.opensearch.sql.ast.dsl.AstDSL.field; +import static org.opensearch.sql.ast.dsl.AstDSL.filter; +import static org.opensearch.sql.ast.dsl.AstDSL.function; +import static org.opensearch.sql.ast.dsl.AstDSL.intLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.let; +import static org.opensearch.sql.ast.dsl.AstDSL.relation; + +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; + +@RunWith(Parameterized.class) +public class AstNowLikeFunctionTest { + + private final PPLSyntaxParser parser = new PPLSyntaxParser(); + + /** + * Set parameterized values used in test. + * @param name Function name + * @param hasFsp Whether function has fsp argument + * @param hasShortcut Whether function has shortcut (call without `()`) + * @param isConstantFunction Whether function has constant value + */ + public AstNowLikeFunctionTest(String name, Boolean hasFsp, Boolean hasShortcut, + Boolean isConstantFunction) { + this.name = name; + this.hasFsp = hasFsp; + this.hasShortcut = hasShortcut; + this.isConstantFunction = isConstantFunction; + } + + /** + * Returns function data to test. + * @return An iterable. + */ + @Parameterized.Parameters(name = "{0}") + public static Iterable functionNames() { + return List.of(new Object[][]{ + {"now", false, false, true}, + {"current_timestamp", false, true, true}, + {"localtimestamp", false, true, true}, + {"localtime", false, true, true}, + {"sysdate", true, false, false}, + {"curtime", false, false, true}, + {"current_time", false, true, true}, + {"curdate", false, false, true}, + {"current_date", false, true, true} + }); + } + + private final String name; + private final Boolean hasFsp; + private final Boolean hasShortcut; + private final Boolean isConstantFunction; + + @Test + public void test_now_like_functions() { + for (var call : hasShortcut ? List.of(name, name + "()") : List.of(name + "()")) { + assertEqual("source=t | eval r=" + call, + eval( + relation("t"), + let( + field("r"), + (isConstantFunction ? constantFunction(name) : function(name)) + ) + )); + + assertEqual("search source=t | where a=" + call, + filter( + relation("t"), + compare("=", field("a"), + (isConstantFunction ? constantFunction(name) : function(name))) + ) + ); + } + // Unfortunately, only real functions (not ConstantFunction) might have `fsp` now. + if (hasFsp) { + assertEqual("search source=t | where a=" + name + "(0)", + filter( + relation("t"), + compare("=", field("a"), function(name, intLiteral(0))) + ) + ); + } + } + + protected void assertEqual(String query, Node expectedPlan) { + Node actualPlan = plan(query); + assertEquals(expectedPlan, actualPlan); + } + + private Node plan(String query) { + AstBuilder astBuilder = new AstBuilder(new AstExpressionBuilder(), query); + return astBuilder.visit(parser.parse(query)); + } +} diff --git a/sql/src/main/antlr/OpenSearchSQLLexer.g4 b/sql/src/main/antlr/OpenSearchSQLLexer.g4 index 6d2d7d8a64..0eaec2a5ee 100644 --- a/sql/src/main/antlr/OpenSearchSQLLexer.g4 +++ b/sql/src/main/antlr/OpenSearchSQLLexer.g4 @@ -191,6 +191,10 @@ COSH: 'COSH'; COT: 'COT'; CRC32: 'CRC32'; CURDATE: 'CURDATE'; +CURTIME: 'CURTIME'; +CURRENT_DATE: 'CURRENT_DATE'; +CURRENT_TIME: 'CURRENT_TIME'; +CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; DATE: 'DATE'; DATE_FORMAT: 'DATE_FORMAT'; DATE_ADD: 'DATE_ADD'; @@ -210,6 +214,8 @@ IFNULL: 'IFNULL'; ISNULL: 'ISNULL'; LENGTH: 'LENGTH'; LN: 'LN'; +LOCALTIME: 'LOCALTIME'; +LOCALTIMESTAMP: 'LOCALTIMESTAMP'; LOCATE: 'LOCATE'; LOG: 'LOG'; LOG10: 'LOG10'; @@ -239,13 +245,17 @@ SINH: 'SINH'; SQRT: 'SQRT'; SUBDATE: 'SUBDATE'; SUBTRACT: 'SUBTRACT'; +SYSDATE: 'SYSDATE'; TAN: 'TAN'; TIME: 'TIME'; TIME_TO_SEC: 'TIME_TO_SEC'; TIMESTAMP: 'TIMESTAMP'; TRUNCATE: 'TRUNCATE'; TO_DAYS: 'TO_DAYS'; +UTC_DATE: 'UTC_DATE'; UPPER: 'UPPER'; +UTC_TIME: 'UTC_TIME'; +UTC_TIMESTAMP: 'UTC_TIMESTAMP'; D: 'D'; T: 'T'; diff --git a/sql/src/main/antlr/OpenSearchSQLParser.g4 b/sql/src/main/antlr/OpenSearchSQLParser.g4 index 40207df82a..a0507c98ef 100644 --- a/sql/src/main/antlr/OpenSearchSQLParser.g4 +++ b/sql/src/main/antlr/OpenSearchSQLParser.g4 @@ -225,6 +225,7 @@ datetimeLiteral : dateLiteral | timeLiteral | timestampLiteral + | datetimeConstantLiteral ; dateLiteral @@ -239,6 +240,11 @@ timestampLiteral : TIMESTAMP timestamp=stringLiteral ; +// Actually, these constants are shortcuts to the corresponding functions +datetimeConstantLiteral + : CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP | LOCALTIME | LOCALTIMESTAMP | UTC_TIMESTAMP | UTC_DATE | UTC_TIME + ; + intervalLiteral : INTERVAL expression intervalUnit ; @@ -294,13 +300,18 @@ nullNotnull ; functionCall - : scalarFunctionName LR_BRACKET functionArgs? RR_BRACKET #scalarFunctionCall + : scalarFunctionName LR_BRACKET functionArgs RR_BRACKET #scalarFunctionCall | specificFunction #specificFunctionCall | windowFunctionClause #windowFunctionCall | aggregateFunction #aggregateFunctionCall | aggregateFunction (orderByClause)? filterClause #filteredAggregationFunctionCall | relevanceFunction #relevanceFunctionCall | highlightFunction #highlightFunctionCall + | constantFunction #constantFunctionCall + ; + +constantFunction + : constantFunctionName LR_BRACKET functionArgs RR_BRACKET ; highlightFunction @@ -383,9 +394,15 @@ trigonometricFunctionName ; dateTimeFunctionName - : ADDDATE | DATE | DATE_ADD | DATE_SUB | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK | DAYOFYEAR | FROM_DAYS - | HOUR | MICROSECOND | MINUTE | MONTH | MONTHNAME | QUARTER | SECOND | SUBDATE | TIME | TIME_TO_SEC - | TIMESTAMP | TO_DAYS | YEAR | WEEK | DATE_FORMAT | MAKETIME | MAKEDATE + : ADDDATE | DATE | DATE_ADD | DATE_FORMAT | DATE_SUB | DAY | DAYNAME | DAYOFMONTH | DAYOFWEEK + | DAYOFYEAR | FROM_DAYS | HOUR | MAKEDATE | MAKETIME | MICROSECOND | MINUTE | MONTH | MONTHNAME + | QUARTER | SECOND | SUBDATE | SYSDATE | TIME | TIME_TO_SEC| TIMESTAMP | TO_DAYS | WEEK | YEAR + ; + +// Functions which value could be cached in scope of a single query +constantFunctionName + : datetimeConstantLiteral + | CURDATE | CURTIME | NOW ; textFunctionName @@ -414,7 +431,7 @@ legacyRelevanceFunctionName ; functionArgs - : functionArg (COMMA functionArg)* + : (functionArg (COMMA functionArg)*)? ; functionArg diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java index 453162e335..006ed5fba2 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstExpressionBuilder.java @@ -18,10 +18,12 @@ import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.CaseFuncAlternativeContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.CaseFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ColumnFilterContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ConstantFunctionContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.ConvertedDataTypeContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.CountStarFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DataTypeFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DateLiteralContext; +import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DatetimeConstantLiteralContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.DistinctCountFunctionCallContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.IsNullPredicateContext; import static org.opensearch.sql.sql.antlr.parser.OpenSearchSQLParser.LikePredicateContext; @@ -61,6 +63,7 @@ import org.opensearch.sql.ast.expression.And; import org.opensearch.sql.ast.expression.Case; import org.opensearch.sql.ast.expression.Cast; +import org.opensearch.sql.ast.expression.ConstantFunction; import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.Function; import org.opensearch.sql.ast.expression.HighlightFunction; @@ -386,9 +389,6 @@ public UnresolvedExpression visitMultiFieldRelevanceFunction( } private Function visitFunction(String functionName, FunctionArgsContext args) { - if (args == null) { - return new Function(functionName, Collections.emptyList()); - } return new Function( functionName, args.functionArg() @@ -398,6 +398,28 @@ private Function visitFunction(String functionName, FunctionArgsContext args) { ); } + @Override + public UnresolvedExpression visitDatetimeConstantLiteral(DatetimeConstantLiteralContext ctx) { + return visitConstantFunction(ctx.getText(), null); + } + + @Override + public UnresolvedExpression visitConstantFunction(ConstantFunctionContext ctx) { + return visitConstantFunction(ctx.constantFunctionName().getText(), + ctx.functionArgs()); + } + + private UnresolvedExpression visitConstantFunction(String functionName, + FunctionArgsContext args) { + return new ConstantFunction(functionName, + args == null + ? Collections.emptyList() + : args.functionArg() + .stream() + .map(this::visitFunctionArg) + .collect(Collectors.toList())); + } + private QualifiedName visitIdentifiers(List identifiers) { return new QualifiedName( identifiers.stream() diff --git a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java index 61bedcf754..af428bdc52 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -22,6 +22,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.opensearch.sql.common.antlr.SyntaxCheckException; @@ -157,6 +158,39 @@ public void canNotParseShowStatementWithoutFilterClause() { assertThrows(SyntaxCheckException.class, () -> parser.parse("SHOW TABLES")); } + private static Stream nowLikeFunctionsData() { + return Stream.of( + Arguments.of("now", true, false), + Arguments.of("current_timestamp", true, true), + Arguments.of("localtimestamp", true, true), + Arguments.of("localtime", true, true), + Arguments.of("sysdate", true, false), + Arguments.of("curtime", true, false), + Arguments.of("current_time", true, true), + Arguments.of("curdate", false, false), + Arguments.of("current_date", false, true) + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("nowLikeFunctionsData") + public void can_parse_now_like_functions(String name, Boolean hasFsp, Boolean hasShortcut) { + var calls = new ArrayList() {{ + add(name + "()"); + }}; + if (hasShortcut) { + calls.add(name); + } + if (hasFsp) { + calls.add(name + "(0)"); + } + + assertNotNull(parser.parse("SELECT " + String.join(", ", calls))); + assertNotNull(parser.parse("SELECT " + String.join(", ", calls) + " FROM test")); + assertNotNull(parser.parse("SELECT " + String.join(", ", calls) + ", id FROM test")); + assertNotNull(parser.parse("SELECT id FROM test WHERE " + String.join(" AND ", calls))); + } + @Test public void can_parse_multi_match_relevance_function() { assertNotNull(parser.parse( diff --git a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java index 8bf38b14a6..c3b9ed245a 100644 --- a/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/org/opensearch/sql/sql/parser/AstBuilderTest.java @@ -14,6 +14,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.alias; import static org.opensearch.sql.ast.dsl.AstDSL.argument; import static org.opensearch.sql.ast.dsl.AstDSL.booleanLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.constantFunction; import static org.opensearch.sql.ast.dsl.AstDSL.doubleLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.field; import static org.opensearch.sql.ast.dsl.AstDSL.filter; @@ -32,8 +33,13 @@ import static org.opensearch.sql.utils.SystemIndexUtils.mappingTable; import com.google.common.collect.ImmutableList; +import java.util.List; +import java.util.stream.Stream; import org.antlr.v4.runtime.tree.ParseTree; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.expression.AllFields; import org.opensearch.sql.ast.tree.UnresolvedPlan; @@ -669,6 +675,60 @@ public void can_build_limit_clause_with_offset() { buildAST("SELECT name FROM test LIMIT 5, 10")); } + private static Stream nowLikeFunctionsData() { + return Stream.of( + Arguments.of("now", false, false, true), + Arguments.of("current_timestamp", false, true, true), + Arguments.of("localtimestamp", false, true, true), + Arguments.of("localtime", false, true, true), + Arguments.of("sysdate", true, false, false), + Arguments.of("curtime", false, false, true), + Arguments.of("current_time", false, true, true), + Arguments.of("curdate", false, false, true), + Arguments.of("current_date", false, true, true) + ); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("nowLikeFunctionsData") + public void test_now_like_functions(String name, Boolean hasFsp, Boolean hasShortcut, + Boolean isConstantFunction) { + for (var call : hasShortcut ? List.of(name, name + "()") : List.of(name + "()")) { + assertEquals( + project( + values(emptyList()), + alias(call, (isConstantFunction ? constantFunction(name) : function(name))) + ), + buildAST("SELECT " + call) + ); + + assertEquals( + project( + filter( + relation("test"), + function( + "=", + qualifiedName("data"), + (isConstantFunction ? constantFunction(name) : function(name))) + ), + AllFields.of() + ), + buildAST("SELECT * FROM test WHERE data = " + call) + ); + } + + // Unfortunately, only real functions (not ConstantFunction) might have `fsp` now. + if (hasFsp) { + assertEquals( + project( + values(emptyList()), + alias(name + "(0)", function(name, intLiteral(0))) + ), + buildAST("SELECT " + name + "(0)") + ); + } + } + @Test public void can_build_qualified_name_highlight() { assertEquals( @@ -691,5 +751,4 @@ private UnresolvedPlan buildAST(String query) { ParseTree parseTree = parser.parse(query); return parseTree.accept(new AstBuilder(query)); } - }