diff --git a/src/main/antlr/OpenDistroSqlLexer.g4 b/src/main/antlr/OpenDistroSqlLexer.g4 index 08ab8b9720..b581c9b34c 100644 --- a/src/main/antlr/OpenDistroSqlLexer.g4 +++ b/src/main/antlr/OpenDistroSqlLexer.g4 @@ -132,6 +132,8 @@ TABLES: 'TABLES'; ABS: 'ABS'; ACOS: 'ACOS'; +ADD: 'ADD'; +ASCII: 'ASCII'; ASIN: 'ASIN'; ATAN: 'ATAN'; ATAN2: 'ATAN2'; @@ -141,26 +143,50 @@ CONCAT: 'CONCAT'; CONCAT_WS: 'CONCAT_WS'; COS: 'COS'; COSH: 'COSH'; +COT: 'COT'; +CURDATE: 'CURDATE'; +DATE: 'DATE'; DATE_FORMAT: 'DATE_FORMAT'; +DAYOFMONTH: 'DAYOFMONTH'; DEGREES: 'DEGREES'; E: 'E'; EXP: 'EXP'; EXPM1: 'EXPM1'; FLOOR: 'FLOOR'; +IF: 'IF'; +IFNULL: 'IFNULL'; +ISNULL: 'ISNULL'; +LENGTH: 'LENGTH'; +LN: 'LN'; +LOCATE: 'LOCATE'; LOG: 'LOG'; LOG10: 'LOG10'; LOG2: 'LOG2'; LOWER: 'LOWER'; +LTRIM: 'LTRIM'; +MAKETIME: 'MAKETIME'; +MODULUS: 'MODULUS'; +MONTH: 'MONTH'; +MONTHNAME: 'MONTHNAME'; +MULTIPLY: 'MULTIPLY'; +NOW: 'NOW'; PI: 'PI'; POW: 'POW'; +POWER: 'POWER'; RADIANS: 'RADIANS'; -RANDOM: 'RANDOM'; +RAND: 'RAND'; +REPLACE: 'REPLACE'; RINT: 'RINT'; ROUND: 'ROUND'; +RTRIM: 'RTRIM'; +SIGN: 'SIGN'; +SIGNUM: 'SIGNUM'; SIN: 'SIN'; SINH: 'SINH'; SQRT: 'SQRT'; +SUBTRACT: 'SUBTRACT'; TAN: 'TAN'; +TIMESTAMP: 'TIMESTAMP'; UPPER: 'UPPER'; D: 'D'; diff --git a/src/main/antlr/OpenDistroSqlParser.g4 b/src/main/antlr/OpenDistroSqlParser.g4 index f485382ff0..c35140e6c0 100644 --- a/src/main/antlr/OpenDistroSqlParser.g4 +++ b/src/main/antlr/OpenDistroSqlParser.g4 @@ -408,14 +408,13 @@ keywordsCanBeId functionNameBase : esFunctionNameBase - | ABS | ASIN | ATAN | ATAN2 | CBRT | CEIL | CONCAT | CONCAT_WS - | COS | COSH | DATE_FORMAT | DEGREES - | E | EXP | EXPM1 | FLOOR | LOG | LOG10 | LOG2 | LOWER - | PI | POW | RADIANS | RANDOM | RINT | ROUND - | SIN | SINH | SQRT | SUBSTRING | TAN | TRIM | UPPER | YEAR - | MONTH | DAYOFMONTH | DATE | MONTHNAME | TIMESTAMP - | MAKETIME | NOW | CURDATE - | IF | IFNULL | ISNULL + | ABS | ACOS | ADD | ASCII | ASIN | ATAN | ATAN2 | CBRT | CEIL | CONCAT | CONCAT_WS + | COS | COSH | COT | CURDATE | DATE | DATE_FORMAT | DAYOFMONTH | DEGREES + | E | EXP | EXPM1 | FLOOR | IF | IFNULL | ISNULL | LEFT | LENGTH | LN | LOCATE | LOG + | LOG10 | LOG2 | LOWER | LTRIM | MAKETIME | MODULUS | MONTH | MONTHNAME | MULTIPLY + | NOW | PI | POW | POWER | RADIANS | RAND | REPLACE | RIGHT | RINT | ROUND | RTRIM + | SIGN | SIGNUM | SIN | SINH | SQRT | SUBSTRING | SUBTRACT | TAN | TIMESTAMP | TRIM + | UPPER | YEAR ; esFunctionNameBase diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ScalarFunction.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ScalarFunction.java index d24669c0c0..453cebce91 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ScalarFunction.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ScalarFunction.java @@ -33,6 +33,8 @@ public enum ScalarFunction implements TypeExpression { ABS(func(T(NUMBER)).to(T)), // translate to Java: T ABS(T) + ACOS(func(T(NUMBER)).to(T)), + ADD(func(T(NUMBER), NUMBER).to(T)), ASCII(func(T(STRING)).to(T)), ASIN(func(T(NUMBER)).to(T)), ATAN(func(T(NUMBER)).to(T)), @@ -44,11 +46,15 @@ public enum ScalarFunction implements TypeExpression { COS(func(T(NUMBER)).to(T)), COSH(func(T(NUMBER)).to(T)), COT(func(T(NUMBER)).to(T)), + CURDATE(func().to(ESDataType.DATE)), + DATE(func(ESDataType.DATE).to(ESDataType.DATE)), DATE_FORMAT( func(ESDataType.DATE, STRING).to(STRING), func(ESDataType.DATE, STRING, STRING).to(STRING) ), + DAYOFMONTH(func(ESDataType.DATE).to(INTEGER)), DEGREES(func(T(NUMBER)).to(T)), + DIVIDE(func(T(NUMBER), NUMBER).to(T)), E(func().to(DOUBLE)), EXP(func(T(NUMBER)).to(T)), EXPM1(func(T(NUMBER)).to(T)), @@ -56,8 +62,9 @@ public enum ScalarFunction implements TypeExpression { IF(func(BOOLEAN, ES_TYPE, ES_TYPE).to(ES_TYPE)), IFNULL(func(ES_TYPE, ES_TYPE).to(ES_TYPE)), ISNULL(func(ES_TYPE).to(INTEGER)), - LENGTH(func(STRING).to(INTEGER) -), + LEFT(func(T(STRING), INTEGER).to(T)), + LENGTH(func(STRING).to(INTEGER)), + LN(func(T(NUMBER)).to(T)), LOCATE( func(STRING, STRING, INTEGER).to(INTEGER), func(STRING, STRING).to(INTEGER) @@ -68,20 +75,33 @@ public enum ScalarFunction implements TypeExpression { ), LOG2(func(T(NUMBER)).to(T)), LOG10(func(T(NUMBER)).to(T)), - LN(func(T(NUMBER)).to(T)), LOWER( func(T(STRING)).to(T), func(T(STRING), STRING).to(T) ), LTRIM(func(T(STRING)).to(T)), + MAKETIME(func(INTEGER, INTEGER, INTEGER).to(ESDataType.DATE)), + MODULUS(func(T(NUMBER), NUMBER).to(T)), + MONTH(func(ESDataType.DATE).to(INTEGER)), + MONTHNAME(func(ESDataType.DATE).to(STRING)), + MULTIPLY(func(T(NUMBER), NUMBER).to(NUMBER)), + NOW(func().to(ESDataType.DATE)), PI(func().to(DOUBLE)), - POW, POWER( + POW( + func(T(NUMBER)).to(T), + func(T(NUMBER), NUMBER).to(T) + ), + POWER( func(T(NUMBER)).to(T), func(T(NUMBER), NUMBER).to(T) ), RADIANS(func(T(NUMBER)).to(T)), - RANDOM(func(T(NUMBER)).to(T)), + RAND( + func().to(NUMBER), + func(T(NUMBER)).to(T) + ), REPLACE(func(T(STRING), STRING, STRING).to(T)), + RIGHT(func(T(STRING), INTEGER).to(T)), RINT(func(T(NUMBER)).to(T)), ROUND(func(T(NUMBER)).to(T)), RTRIM(func(T(STRING)).to(T)), @@ -91,20 +111,15 @@ POW, POWER( SINH(func(T(NUMBER)).to(T)), SQRT(func(T(NUMBER)).to(T)), SUBSTRING(func(T(STRING), INTEGER, INTEGER).to(T)), + SUBTRACT(func(T(NUMBER), NUMBER).to(T)), TAN(func(T(NUMBER)).to(T)), + TIMESTAMP(func(ESDataType.DATE).to(ESDataType.DATE)), + TRIM(func(T(STRING)).to(T)), UPPER( func(T(STRING)).to(T), func(T(STRING), STRING).to(T) ), - YEAR(func(ESDataType.DATE).to(INTEGER)), - MONTH(func(ESDataType.DATE).to(INTEGER)), - MONTHNAME(func(ESDataType.DATE).to(STRING)), - DAYOFMONTH(func(ESDataType.DATE).to(INTEGER)), - DATE(func(ESDataType.DATE).to(ESDataType.DATE)), - TIMESTAMP(func(ESDataType.DATE).to(ESDataType.DATE)), - MAKETIME(func(INTEGER, INTEGER, INTEGER).to(ESDataType.DATE)), - NOW(func().to(ESDataType.DATE)), - CURDATE(func().to(ESDataType.DATE)); + YEAR(func(ESDataType.DATE).to(INTEGER)); private final TypeExpressionSpec[] specifications; diff --git a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java b/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java index 3c126d0df9..5131b35725 100644 --- a/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java +++ b/src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java @@ -42,6 +42,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.format; import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.isQuoted; /** @@ -51,7 +52,7 @@ public class SQLFunctions { private static final Set numberOperators = Sets.newHashSet( "exp", "expm1", "log", "log2", "log10", "ln", "sqrt", "cbrt", "ceil", "floor", "rint", "pow", "power", - "round", "random", "abs", "sign", "signum" + "round", "rand", "abs", "sign", "signum" ); private static final Set mathConstants = Sets.newHashSet("e", "pi"); @@ -246,6 +247,14 @@ public Tuple function(String methodName, List paramers, (SQLExpr) paramers.get(0).value, name); break; + case "rand": + if (paramers.isEmpty()) { + functionStr = rand(); + } else { + functionStr = rand((SQLExpr) paramers.get(0).value); + } + break; + case "cot": // ES does not support the function name cot functionStr = mathSingleValueTemplate("1 / Math.tan", methodName, @@ -354,6 +363,12 @@ public Tuple function(String methodName, List paramers, case "ascii": functionStr = ascii((SQLExpr) paramers.get(0).value); break; + case "left": + functionStr = left((SQLExpr) paramers.get(0).value, (SQLExpr) paramers.get(1).value); + break; + case "right": + functionStr = right((SQLExpr) paramers.get(0).value, (SQLExpr) paramers.get(1).value); + break; case "if": functionStr = ifFunc(paramers); @@ -391,10 +406,10 @@ public Tuple upper(SQLExpr field, String locale, String valueNam String name = nextId("upper"); if (valueName == null) { - return new Tuple<>(name, def(name, upper(getPropertyOrValue(field), locale))); + return new Tuple<>(name, def(name, upper(getPropertyOrStringValue(field), locale))); } else { - return new Tuple<>(name, getPropertyOrValue(field) + "; " - + def(name, valueName + "." + upper(getPropertyOrValue(field), locale))); + return new Tuple<>(name, getPropertyOrStringValue(field) + "; " + + def(name, valueName + "." + upper(getPropertyOrStringValue(field), locale))); } } @@ -402,10 +417,10 @@ public Tuple lower(SQLExpr field, String locale, String valueNam String name = nextId("lower"); if (valueName == null) { - return new Tuple<>(name, def(name, lower(getPropertyOrValue(field), locale))); + return new Tuple<>(name, def(name, lower(getPropertyOrStringValue(field), locale))); } else { - return new Tuple<>(name, getPropertyOrValue(field) + "; " - + def(name, valueName + "." + lower(getPropertyOrValue(field), locale))); + return new Tuple<>(name, getPropertyOrStringValue(field) + "; " + + def(name, valueName + "." + lower(getPropertyOrStringValue(field), locale))); } } @@ -650,14 +665,25 @@ private Tuple radians(SQLExpr field, String valueName) { return mathSingleValueTemplate("Math.toRadians", "radians", field, valueName); } + private Tuple rand(SQLExpr expr) { + String name = nextId("rand"); + return new Tuple<>(name, def(name, format("new Random(%s).nextDouble()", getPropertyOrValue(expr)))); + } + + private Tuple rand() { + String name = nextId("rand"); + return new Tuple<>(name, def(name, "new Random().nextDouble()")); + } + private Tuple mathDoubleValueTemplate(String methodName, String fieldName, SQLExpr val1, String val2, String valueName) { String name = nextId(fieldName); if (valueName == null) { - return new Tuple<>(name, def(name, func(methodName, false, getPropertyOrValue(val1), val2))); + return new Tuple<>(name, def(name, func(methodName, false, getPropertyOrValue(val1), + getPropertyOrValue(val2)))); } else { return new Tuple<>(name, getPropertyOrValue(val1) + "; " - + def(name, func(methodName, false, valueName, val2))); + + def(name, func(methodName, false, valueName, getPropertyOrValue(val2)))); } } @@ -688,9 +714,9 @@ private Tuple mathConstantTemplate(String methodName, String fie private Tuple strSingleValueTemplate(String methodName, SQLExpr field, String valueName) { String name = nextId(methodName); if (valueName == null) { - return new Tuple<>(name, def(name, getPropertyOrValue(field) + "." + func(methodName, false))); + return new Tuple<>(name, def(name, getPropertyOrStringValue(field) + "." + func(methodName, false))); } else { - return new Tuple<>(name, getPropertyOrValue(field) + "; " + return new Tuple<>(name, getPropertyOrStringValue(field) + "; " + def(name, valueName + "." + func(methodName, false))); } @@ -772,6 +798,20 @@ private Tuple ascii(SQLExpr field) { return new Tuple<>(name, def(name, "(int) " + getPropertyOrStringValue(field) + ".charAt(0)")); } + private Tuple left(SQLExpr expr, SQLExpr length) { + String name = nextId("left"); + return new Tuple<>(name, StringUtils.format( + "def len = (int) Math.min(%s, %s.length()); def %s = %s.substring(0, len)", + exprString(length), getPropertyOrStringValue(expr), name, getPropertyOrStringValue(expr))); + } + + private Tuple right(SQLExpr expr, SQLExpr length) { + String name = nextId("right"); + return new Tuple<>(name, StringUtils.format( + "def start = (int) Math.max(0, %s.length()-%s); def %s = %s.substring(start)", + getPropertyOrStringValue(expr), exprString(length), name, getPropertyOrStringValue(expr))); + } + private Tuple date(SQLExpr field) { String name = nextId("date"); return new Tuple<>(name, def(name, diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MathFunctionsIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MathFunctionsIT.java index 172679f954..fc88bf4df5 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MathFunctionsIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MathFunctionsIT.java @@ -228,6 +228,15 @@ public void lnInAggregationShouldPass() { ); } + @Test + public void rand() throws IOException { + SearchHit[] hits = query("SELECT RAND() AS rand", "ORDER BY rand"); + for (SearchHit hit : hits) { + double rand = (double) getField(hit, "rand"); + assertTrue(rand >= 0 && rand < 1); + } + } + private SearchHit[] query(String select, String... statements) throws IOException { final String response = executeQueryWithStringOutput(select + " " + FROM + " " + String.join(" ", statements)); diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java index 5d5fdc5e00..75ecfa623e 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java @@ -482,6 +482,36 @@ public void ascii() throws IOException { ); } + /** + * The following tests for LEFT and RIGHT are ignored because the ES client fails to parse "LEFT"/"RIGHT" in + * the integTest + */ + @Ignore + @Test + public void left() throws IOException { + assertThat( + executeQuery("SELECT LEFT('sample', 2) AS left FROM " + TEST_INDEX_ACCOUNT + " ORDER BY left"), + hitAny(kvString("/fields/left/0", equalTo("sa"))) + ); + assertThat( + executeQuery("SELECT LEFT('sample', 20) AS left FROM " + TEST_INDEX_ACCOUNT + " ORDER BY left"), + hitAny(kvString("/fields/left/0", equalTo("sample"))) + ); + } + + @Ignore + @Test + public void right() throws IOException { + assertThat( + executeQuery("SELECT RIGHT('elastic', 3) AS right FROM " + TEST_INDEX_ACCOUNT + " ORDER BY right"), + hitAny(kvString("/fields/right/0", equalTo("tic"))) + ); + assertThat( + executeQuery("SELECT RIGHT('elastic', 20) AS right FROM " + TEST_INDEX_ACCOUNT + " ORDER BY right"), + hitAny(kvString("/fields/right/0", equalTo("elastic"))) + ); + } + @Test public void ifFuncShouldPassJDBC() { assertThat( diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/MathFunctionsTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/MathFunctionsTest.java index 6cf7ddb0f9..c8f17bffe0 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/MathFunctionsTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/MathFunctionsTest.java @@ -423,4 +423,28 @@ public void lnTest() { scriptFilter, "Math.log(doc['age'].value)")); } + + @Test + public void randWithoutParamTest() { + String query = "SELECT RAND() FROM bank"; + ScriptField scriptField = CheckScriptContents.getScriptFieldFromQuery(query); + assertTrue( + CheckScriptContents.scriptContainsString( + scriptField, + "new Random().nextDouble()" + ) + ); + } + + @Test + public void randWithOneParamTest() { + String query = "SELECT RAND(age) FROM bank"; + ScriptField scriptField = CheckScriptContents.getScriptFieldFromQuery(query); + assertTrue( + CheckScriptContents.scriptContainsString( + scriptField, + "new Random(doc['age'].value).nextDouble()" + ) + ); + } } diff --git a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/StringOperatorsTest.java b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/StringOperatorsTest.java index cd83609249..5f441d2ea5 100644 --- a/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/StringOperatorsTest.java +++ b/src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/StringOperatorsTest.java @@ -194,4 +194,28 @@ public void asciiTest() { ) ); } + + @Test + public void left() { + String query = "SELECT left(lastname, 1) FROM accounts"; + ScriptField scriptField = CheckScriptContents.getScriptFieldFromQuery(query); + assertTrue( + CheckScriptContents.scriptContainsString( + scriptField, + "doc['lastname'].value.substring(0, len)" + ) + ); + } + + @Test + public void right() { + String query = "SELECT right(lastname, 2) FROM accounts"; + ScriptField scriptField = CheckScriptContents.getScriptFieldFromQuery(query); + assertTrue( + CheckScriptContents.scriptContainsString( + scriptField, + "doc['lastname'].value.substring(start)" + ) + ); + } }