Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Commit

Permalink
Fixed functions work improperly with fieldvalue/constant param for cu…
Browse files Browse the repository at this point in the history 
…rrent use (#296)

* Fixed the field input issue; added LEFT and RIGHT function; reorganized the scalar function base

* Added tests
  • Loading branch information
@chloe-zh
chloe-zh authored Nov 20, 2019
1 parent 537f4b9 commit 332cb48
Show file tree
Hide file tree
Showing 8 changed files with 201 additions and 34 deletions.
28 changes: 27 additions & 1 deletion src/main/antlr/OpenDistroSqlLexer.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,8 @@ TABLES: 'TABLES';

ABS: 'ABS';
ACOS: 'ACOS';
ADD: 'ADD';
ASCII: 'ASCII';
ASIN: 'ASIN';
ATAN: 'ATAN';
ATAN2: 'ATAN2';
Expand All @@ -141,26 +143,50 @@ CONCAT: 'CONCAT';
CONCAT_WS: 'CONCAT_WS';
COS: 'COS';
COSH: 'COSH';
COT: 'COT';
CURDATE: 'CURDATE';
DATE: 'DATE';
DATE_FORMAT: 'DATE_FORMAT';
DAYOFMONTH: 'DAYOFMONTH';
DEGREES: 'DEGREES';
E: 'E';
EXP: 'EXP';
EXPM1: 'EXPM1';
FLOOR: 'FLOOR';
IF: 'IF';
IFNULL: 'IFNULL';
ISNULL: 'ISNULL';
LENGTH: 'LENGTH';
LN: 'LN';
LOCATE: 'LOCATE';
LOG: 'LOG';
LOG10: 'LOG10';
LOG2: 'LOG2';
LOWER: 'LOWER';
LTRIM: 'LTRIM';
MAKETIME: 'MAKETIME';
MODULUS: 'MODULUS';
MONTH: 'MONTH';
MONTHNAME: 'MONTHNAME';
MULTIPLY: 'MULTIPLY';
NOW: 'NOW';
PI: 'PI';
POW: 'POW';
POWER: 'POWER';
RADIANS: 'RADIANS';
RANDOM: 'RANDOM';
RAND: 'RAND';
REPLACE: 'REPLACE';
RINT: 'RINT';
ROUND: 'ROUND';
RTRIM: 'RTRIM';
SIGN: 'SIGN';
SIGNUM: 'SIGNUM';
SIN: 'SIN';
SINH: 'SINH';
SQRT: 'SQRT';
SUBTRACT: 'SUBTRACT';
TAN: 'TAN';
TIMESTAMP: 'TIMESTAMP';
UPPER: 'UPPER';

D: 'D';
Expand Down
15 changes: 7 additions & 8 deletions src/main/antlr/OpenDistroSqlParser.g4
View file
Original file line number Diff line number Diff line change
Expand Up @@ -408,14 +408,13 @@ keywordsCanBeId

functionNameBase
: esFunctionNameBase
| ABS | ASIN | ATAN | ATAN2 | CBRT | CEIL | CONCAT | CONCAT_WS
| COS | COSH | DATE_FORMAT | DEGREES
| E | EXP | EXPM1 | FLOOR | LOG | LOG10 | LOG2 | LOWER
| PI | POW | RADIANS | RANDOM | RINT | ROUND
| SIN | SINH | SQRT | SUBSTRING | TAN | TRIM | UPPER | YEAR
| MONTH | DAYOFMONTH | DATE | MONTHNAME | TIMESTAMP
| MAKETIME | NOW | CURDATE
| IF | IFNULL | ISNULL
| ABS | ACOS | ADD | ASCII | ASIN | ATAN | ATAN2 | CBRT | CEIL | CONCAT | CONCAT_WS
| COS | COSH | COT | CURDATE | DATE | DATE_FORMAT | DAYOFMONTH | DEGREES
| E | EXP | EXPM1 | FLOOR | IF | IFNULL | ISNULL | LEFT | LENGTH | LN | LOCATE | LOG
| LOG10 | LOG2 | LOWER | LTRIM | MAKETIME | MODULUS | MONTH | MONTHNAME | MULTIPLY
| NOW | PI | POW | POWER | RADIANS | RAND | REPLACE | RIGHT | RINT | ROUND | RTRIM
| SIGN | SIGNUM | SIN | SINH | SQRT | SUBSTRING | SUBTRACT | TAN | TIMESTAMP | TRIM
| UPPER | YEAR
;

esFunctionNameBase
Expand Down
43 changes: 29 additions & 14 deletions ...m/amazon/opendistroforelasticsearch/sql/antlr/semantic/types/function/ScalarFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@
public enum ScalarFunction implements TypeExpression {

ABS(func(T(NUMBER)).to(T)), // translate to Java: <T extends Number> T ABS(T)
ACOS(func(T(NUMBER)).to(T)),
ADD(func(T(NUMBER), NUMBER).to(T)),
ASCII(func(T(STRING)).to(T)),
ASIN(func(T(NUMBER)).to(T)),
ATAN(func(T(NUMBER)).to(T)),
Expand All @@ -44,20 +46,25 @@ public enum ScalarFunction implements TypeExpression {
COS(func(T(NUMBER)).to(T)),
COSH(func(T(NUMBER)).to(T)),
COT(func(T(NUMBER)).to(T)),
CURDATE(func().to(ESDataType.DATE)),
DATE(func(ESDataType.DATE).to(ESDataType.DATE)),
DATE_FORMAT(
func(ESDataType.DATE, STRING).to(STRING),
func(ESDataType.DATE, STRING, STRING).to(STRING)
),
DAYOFMONTH(func(ESDataType.DATE).to(INTEGER)),
DEGREES(func(T(NUMBER)).to(T)),
DIVIDE(func(T(NUMBER), NUMBER).to(T)),
E(func().to(DOUBLE)),
EXP(func(T(NUMBER)).to(T)),
EXPM1(func(T(NUMBER)).to(T)),
FLOOR(func(T(NUMBER)).to(T)),
IF(func(BOOLEAN, ES_TYPE, ES_TYPE).to(ES_TYPE)),
IFNULL(func(ES_TYPE, ES_TYPE).to(ES_TYPE)),
ISNULL(func(ES_TYPE).to(INTEGER)),
LENGTH(func(STRING).to(INTEGER)
),
LEFT(func(T(STRING), INTEGER).to(T)),
LENGTH(func(STRING).to(INTEGER)),
LN(func(T(NUMBER)).to(T)),
LOCATE(
func(STRING, STRING, INTEGER).to(INTEGER),
func(STRING, STRING).to(INTEGER)
Expand All @@ -68,20 +75,33 @@ public enum ScalarFunction implements TypeExpression {
),
LOG2(func(T(NUMBER)).to(T)),
LOG10(func(T(NUMBER)).to(T)),
LN(func(T(NUMBER)).to(T)),
LOWER(
func(T(STRING)).to(T),
func(T(STRING), STRING).to(T)
),
LTRIM(func(T(STRING)).to(T)),
MAKETIME(func(INTEGER, INTEGER, INTEGER).to(ESDataType.DATE)),
MODULUS(func(T(NUMBER), NUMBER).to(T)),
MONTH(func(ESDataType.DATE).to(INTEGER)),
MONTHNAME(func(ESDataType.DATE).to(STRING)),
MULTIPLY(func(T(NUMBER), NUMBER).to(NUMBER)),
NOW(func().to(ESDataType.DATE)),
PI(func().to(DOUBLE)),
POW, POWER(
POW(
func(T(NUMBER)).to(T),
func(T(NUMBER), NUMBER).to(T)
),
POWER(
func(T(NUMBER)).to(T),
func(T(NUMBER), NUMBER).to(T)
),
RADIANS(func(T(NUMBER)).to(T)),
RANDOM(func(T(NUMBER)).to(T)),
RAND(
func().to(NUMBER),
func(T(NUMBER)).to(T)
),
REPLACE(func(T(STRING), STRING, STRING).to(T)),
RIGHT(func(T(STRING), INTEGER).to(T)),
RINT(func(T(NUMBER)).to(T)),
ROUND(func(T(NUMBER)).to(T)),
RTRIM(func(T(STRING)).to(T)),
Expand All @@ -91,20 +111,15 @@ POW, POWER(
SINH(func(T(NUMBER)).to(T)),
SQRT(func(T(NUMBER)).to(T)),
SUBSTRING(func(T(STRING), INTEGER, INTEGER).to(T)),
SUBTRACT(func(T(NUMBER), NUMBER).to(T)),
TAN(func(T(NUMBER)).to(T)),
TIMESTAMP(func(ESDataType.DATE).to(ESDataType.DATE)),
TRIM(func(T(STRING)).to(T)),
UPPER(
func(T(STRING)).to(T),
func(T(STRING), STRING).to(T)
),
YEAR(func(ESDataType.DATE).to(INTEGER)),
MONTH(func(ESDataType.DATE).to(INTEGER)),
MONTHNAME(func(ESDataType.DATE).to(STRING)),
DAYOFMONTH(func(ESDataType.DATE).to(INTEGER)),
DATE(func(ESDataType.DATE).to(ESDataType.DATE)),
TIMESTAMP(func(ESDataType.DATE).to(ESDataType.DATE)),
MAKETIME(func(INTEGER, INTEGER, INTEGER).to(ESDataType.DATE)),
NOW(func().to(ESDataType.DATE)),
CURDATE(func().to(ESDataType.DATE));
YEAR(func(ESDataType.DATE).to(INTEGER));

private final TypeExpressionSpec[] specifications;

Expand Down
62 changes: 51 additions & 11 deletions src/main/java/com/amazon/opendistroforelasticsearch/sql/utils/SQLFunctions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.format;
import static com.amazon.opendistroforelasticsearch.sql.utils.StringUtils.isQuoted;

/**
Expand All @@ -51,7 +52,7 @@ public class SQLFunctions {

private static final Set<String> numberOperators = Sets.newHashSet(
"exp", "expm1", "log", "log2", "log10", "ln", "sqrt", "cbrt", "ceil", "floor", "rint", "pow", "power",
"round", "random", "abs", "sign", "signum"
"round", "rand", "abs", "sign", "signum"
);

private static final Set<String> mathConstants = Sets.newHashSet("e", "pi");
Expand Down Expand Up @@ -246,6 +247,14 @@ public Tuple<String, String> function(String methodName, List<KVValue> paramers,
(SQLExpr) paramers.get(0).value, name);
break;

case "rand":
if (paramers.isEmpty()) {
functionStr = rand();
} else {
functionStr = rand((SQLExpr) paramers.get(0).value);
}
break;

case "cot":
// ES does not support the function name cot
functionStr = mathSingleValueTemplate("1 / Math.tan", methodName,
Expand Down Expand Up @@ -354,6 +363,12 @@ public Tuple<String, String> function(String methodName, List<KVValue> paramers,
case "ascii":
functionStr = ascii((SQLExpr) paramers.get(0).value);
break;
case "left":
functionStr = left((SQLExpr) paramers.get(0).value, (SQLExpr) paramers.get(1).value);
break;
case "right":
functionStr = right((SQLExpr) paramers.get(0).value, (SQLExpr) paramers.get(1).value);
break;

case "if":
functionStr = ifFunc(paramers);
Expand Down Expand Up @@ -391,21 +406,21 @@ public Tuple<String, String> upper(SQLExpr field, String locale, String valueNam
String name = nextId("upper");

if (valueName == null) {
return new Tuple<>(name, def(name, upper(getPropertyOrValue(field), locale)));
return new Tuple<>(name, def(name, upper(getPropertyOrStringValue(field), locale)));
} else {
return new Tuple<>(name, getPropertyOrValue(field) + "; "
+ def(name, valueName + "." + upper(getPropertyOrValue(field), locale)));
return new Tuple<>(name, getPropertyOrStringValue(field) + "; "
+ def(name, valueName + "." + upper(getPropertyOrStringValue(field), locale)));
}
}

public Tuple<String, String> lower(SQLExpr field, String locale, String valueName) {
String name = nextId("lower");

if (valueName == null) {
return new Tuple<>(name, def(name, lower(getPropertyOrValue(field), locale)));
return new Tuple<>(name, def(name, lower(getPropertyOrStringValue(field), locale)));
} else {
return new Tuple<>(name, getPropertyOrValue(field) + "; "
+ def(name, valueName + "." + lower(getPropertyOrValue(field), locale)));
return new Tuple<>(name, getPropertyOrStringValue(field) + "; "
+ def(name, valueName + "." + lower(getPropertyOrStringValue(field), locale)));
}
}

Expand Down Expand Up @@ -650,14 +665,25 @@ private Tuple<String, String> radians(SQLExpr field, String valueName) {
return mathSingleValueTemplate("Math.toRadians", "radians", field, valueName);
}

private Tuple<String, String> rand(SQLExpr expr) {
String name = nextId("rand");
return new Tuple<>(name, def(name, format("new Random(%s).nextDouble()", getPropertyOrValue(expr))));
}

private Tuple<String, String> rand() {
String name = nextId("rand");
return new Tuple<>(name, def(name, "new Random().nextDouble()"));
}

private Tuple<String, String> mathDoubleValueTemplate(String methodName, String fieldName, SQLExpr val1,
String val2, String valueName) {
String name = nextId(fieldName);
if (valueName == null) {
return new Tuple<>(name, def(name, func(methodName, false, getPropertyOrValue(val1), val2)));
return new Tuple<>(name, def(name, func(methodName, false, getPropertyOrValue(val1),
getPropertyOrValue(val2))));
} else {
return new Tuple<>(name, getPropertyOrValue(val1) + "; "
+ def(name, func(methodName, false, valueName, val2)));
+ def(name, func(methodName, false, valueName, getPropertyOrValue(val2))));
}
}

Expand Down Expand Up @@ -688,9 +714,9 @@ private Tuple<String, String> mathConstantTemplate(String methodName, String fie
private Tuple<String, String> strSingleValueTemplate(String methodName, SQLExpr field, String valueName) {
String name = nextId(methodName);
if (valueName == null) {
return new Tuple<>(name, def(name, getPropertyOrValue(field) + "." + func(methodName, false)));
return new Tuple<>(name, def(name, getPropertyOrStringValue(field) + "." + func(methodName, false)));
} else {
return new Tuple<>(name, getPropertyOrValue(field) + "; "
return new Tuple<>(name, getPropertyOrStringValue(field) + "; "
+ def(name, valueName + "." + func(methodName, false)));
}

Expand Down Expand Up @@ -772,6 +798,20 @@ private Tuple<String, String> ascii(SQLExpr field) {
return new Tuple<>(name, def(name, "(int) " + getPropertyOrStringValue(field) + ".charAt(0)"));
}

private Tuple<String, String> left(SQLExpr expr, SQLExpr length) {
String name = nextId("left");
return new Tuple<>(name, StringUtils.format(
"def len = (int) Math.min(%s, %s.length()); def %s = %s.substring(0, len)",
exprString(length), getPropertyOrStringValue(expr), name, getPropertyOrStringValue(expr)));
}

private Tuple<String, String> right(SQLExpr expr, SQLExpr length) {
String name = nextId("right");
return new Tuple<>(name, StringUtils.format(
"def start = (int) Math.max(0, %s.length()-%s); def %s = %s.substring(start)",
getPropertyOrStringValue(expr), exprString(length), name, getPropertyOrStringValue(expr)));
}

private Tuple<String, String> date(SQLExpr field) {
String name = nextId("date");
return new Tuple<>(name, def(name,
Expand Down
9 changes: 9 additions & 0 deletions src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/MathFunctionsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,15 @@ public void lnInAggregationShouldPass() {
);
}

@Test
public void rand() throws IOException {
SearchHit[] hits = query("SELECT RAND() AS rand", "ORDER BY rand");
for (SearchHit hit : hits) {
double rand = (double) getField(hit, "rand");
assertTrue(rand >= 0 && rand < 1);
}
}

private SearchHit[] query(String select, String... statements) throws IOException {
final String response = executeQueryWithStringOutput(select + " " + FROM + " " + String.join(" ", statements));

Expand Down
30 changes: 30 additions & 0 deletions src/test/java/com/amazon/opendistroforelasticsearch/sql/esintgtest/SQLFunctionsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -482,6 +482,36 @@ public void ascii() throws IOException {
);
}

/**
* The following tests for LEFT and RIGHT are ignored because the ES client fails to parse "LEFT"/"RIGHT" in
* the integTest
*/
@Ignore
@Test
public void left() throws IOException {
assertThat(
executeQuery("SELECT LEFT('sample', 2) AS left FROM " + TEST_INDEX_ACCOUNT + " ORDER BY left"),
hitAny(kvString("/fields/left/0", equalTo("sa")))
);
assertThat(
executeQuery("SELECT LEFT('sample', 20) AS left FROM " + TEST_INDEX_ACCOUNT + " ORDER BY left"),
hitAny(kvString("/fields/left/0", equalTo("sample")))
);
}

@Ignore
@Test
public void right() throws IOException {
assertThat(
executeQuery("SELECT RIGHT('elastic', 3) AS right FROM " + TEST_INDEX_ACCOUNT + " ORDER BY right"),
hitAny(kvString("/fields/right/0", equalTo("tic")))
);
assertThat(
executeQuery("SELECT RIGHT('elastic', 20) AS right FROM " + TEST_INDEX_ACCOUNT + " ORDER BY right"),
hitAny(kvString("/fields/right/0", equalTo("elastic")))
);
}

@Test
public void ifFuncShouldPassJDBC() {
assertThat(
Expand Down
24 changes: 24 additions & 0 deletions src/test/java/com/amazon/opendistroforelasticsearch/sql/unittest/MathFunctionsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -423,4 +423,28 @@ public void lnTest() {
scriptFilter,
"Math.log(doc['age'].value)"));
}

@Test
public void randWithoutParamTest() {
String query = "SELECT RAND() FROM bank";
ScriptField scriptField = CheckScriptContents.getScriptFieldFromQuery(query);
assertTrue(
CheckScriptContents.scriptContainsString(
scriptField,
"new Random().nextDouble()"
)
);
}

@Test
public void randWithOneParamTest() {
String query = "SELECT RAND(age) FROM bank";
ScriptField scriptField = CheckScriptContents.getScriptFieldFromQuery(query);
assertTrue(
CheckScriptContents.scriptContainsString(
scriptField,
"new Random(doc['age'].value).nextDouble()"
)
);
}
}
Loading

0 comments on commit 332cb48

Please sign in to comment.