From 78a51a45364e86f49f62b24b18f85319876d55b1 Mon Sep 17 00:00:00 2001 From: Chloe Date: Wed, 22 Jul 2020 10:06:43 -0700 Subject: [PATCH] Support mathematical functions rand and constants e, pi (#591) * support e, pi, rand * updated doc * update * update * update --- .../sql/expression/DSL.java | 13 +++- .../function/BuiltinFunctionName.java | 3 + .../expression/operator/OperatorUtils.java | 31 ++++++++++ .../arthmetic/MathematicalFunction.java | 58 ++++++++++++++++++ .../arthmetic/MathematicalFunctionTest.java | 60 +++++++++++++++++-- docs/user/dql/functions.rst | 45 +++++++++++--- .../sql/ppl/MathematicalFunctionIT.java | 40 ++++++++++++- .../sql/sql/MathematicalFunctionIT.java | 7 +++ .../correctness/expressions/functions.txt | 27 +++++---- ppl/src/main/antlr/OpenDistroPPLLexer.g4 | 3 + ppl/src/main/antlr/OpenDistroPPLParser.g4 | 6 +- sql/src/main/antlr/OpenDistroSQLParser.g4 | 6 +- 12 files changed, 266 insertions(+), 33 deletions(-) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java index e09b2a0a19..e33eb315e7 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java @@ -21,7 +21,6 @@ import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; -import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; import java.util.Arrays; import lombok.RequiredArgsConstructor; @@ -81,6 +80,10 @@ public FunctionExpression crc32(Expression... expressions) { return function(BuiltinFunctionName.CRC32, expressions); } + public FunctionExpression euler(Expression... expressions) { + return function(BuiltinFunctionName.E, expressions); + } + public FunctionExpression exp(Expression... expressions) { return function(BuiltinFunctionName.EXP, expressions); } @@ -109,6 +112,10 @@ public FunctionExpression mod(Expression... expressions) { return function(BuiltinFunctionName.MOD, expressions); } + public FunctionExpression pi(Expression... expressions) { + return function(BuiltinFunctionName.PI, expressions); + } + public FunctionExpression pow(Expression... expressions) { return function(BuiltinFunctionName.POW, expressions); } @@ -117,6 +124,10 @@ public FunctionExpression power(Expression... expressions) { return function(BuiltinFunctionName.POWER, expressions); } + public FunctionExpression rand(Expression... expressions) { + return function(BuiltinFunctionName.RAND, expressions); + } + public FunctionExpression round(Expression... expressions) { return function(BuiltinFunctionName.ROUND, expressions); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java index 282cc0bb43..bf39c5005f 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/BuiltinFunctionName.java @@ -20,6 +20,7 @@ public enum BuiltinFunctionName { CEILING(FunctionName.of("ceiling")), CONV(FunctionName.of("conv")), CRC32(FunctionName.of("crc32")), + E(FunctionName.of("e")), EXP(FunctionName.of("exp")), FLOOR(FunctionName.of("floor")), LN(FunctionName.of("ln")), @@ -27,8 +28,10 @@ public enum BuiltinFunctionName { LOG10(FunctionName.of("log10")), LOG2(FunctionName.of("log2")), MOD(FunctionName.of("mod")), + PI(FunctionName.of("pi")), POW(FunctionName.of("pow")), POWER(FunctionName.of("power")), + RAND(FunctionName.of("rand")), ROUND(FunctionName.of("round")), SIGN(FunctionName.of("sign")), SQRT(FunctionName.of("sqrt")), diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/OperatorUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/OperatorUtils.java index 568cb6a8e3..a60a4fc48e 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/OperatorUtils.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/OperatorUtils.java @@ -29,6 +29,7 @@ import java.util.function.BiFunction; import java.util.function.BiPredicate; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; import lombok.experimental.UtilityClass; @@ -257,6 +258,36 @@ public String toString() { }; } + /** + * Construct {@link FunctionBuilder} which call function with no argument. + * + * @param functionName function name + * @param function {@link Function} + * @param returnType return type + * @param the type of the result to the function + * @return {@link FunctionBuilder} + */ + public static FunctionBuilder noArgFunction(FunctionName functionName, + Supplier function, + ExprCoreType returnType) { + return arguments -> new FunctionExpression(functionName, arguments) { + @Override + public ExprValue valueOf(Environment valueEnv) { + return ExprValueUtils.fromObjectValue(function.get()); + } + + @Override + public ExprType type() { + return returnType; + } + + @Override + public String toString() { + return String.format("%s()", functionName); + } + }; + } + /** * String comparator. */ diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunction.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunction.java index 0bed318e72..69972d3255 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunction.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunction.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.sql.expression.operator.arthmetic; import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.doubleArgFunc; +import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.noArgFunction; import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.tripleArgFunc; import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.unaryOperator; @@ -31,7 +32,9 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.util.Arrays; +import java.util.Collections; import java.util.Map; +import java.util.Random; import java.util.function.BiFunction; import java.util.function.Function; import java.util.zip.CRC32; @@ -50,6 +53,7 @@ public static void register(BuiltinFunctionRepository repository) { repository.register(ceiling()); repository.register(conv()); repository.register(crc32()); + repository.register(euler()); repository.register(exp()); repository.register(floor()); repository.register(ln()); @@ -63,6 +67,8 @@ public static void register(BuiltinFunctionRepository repository) { repository.register(sign()); repository.register(sqrt()); repository.register(truncate()); + repository.register(pi()); + repository.register(rand()); } /** @@ -164,6 +170,20 @@ private static FunctionResolver crc32() { .build()); } + /** + * Definition of e() function. + * Get the Euler's number. + * () -> DOUBLE + */ + private static FunctionResolver euler() { + FunctionName functionName = BuiltinFunctionName.E.getName(); + return new FunctionResolver(functionName, + new ImmutableMap.Builder() + .put(new FunctionSignature(functionName, Collections.emptyList()), + noArgFunction(functionName, () -> Math.E, ExprCoreType.DOUBLE)) + .build()); + } + /** * Definition of exp(x) function. Calculate exponent function e to the x The supported signature * of exp function is INTEGER/LONG/FLOAT/DOUBLE -> DOUBLE @@ -266,6 +286,20 @@ private static FunctionResolver mod() { (v1, v2) -> v2 == 0 ? null : v1 % v2)); } + /** + * Definition of pi() function. + * Get the value of pi. + * () -> DOUBLE + */ + private static FunctionResolver pi() { + FunctionName functionName = BuiltinFunctionName.PI.getName(); + return new FunctionResolver(functionName, + new ImmutableMap.Builder() + .put(new FunctionSignature(functionName, Collections.emptyList()), + noArgFunction(functionName, () -> Math.PI, ExprCoreType.DOUBLE)) + .build()); + } + /** * Definition of pow(x, y)/power(x, y) function. * Calculate the value of x raised to the power of y @@ -285,6 +319,30 @@ private static FunctionResolver power() { return new FunctionResolver(functionName, doubleArgumentsFunction(functionName, Math::pow)); } + /** + * Definition of rand() and rand(N) function. + * rand() returns a random floating-point value in the range 0 <= value < 1.0 + * If integer N is specified, the seed is initialized prior to execution. + * One implication of this behavior is with identical argument N,rand(N) returns the same value + * each time, and thus produces a repeatable sequence of column values. + * The supported signature of rand function is + * ([INTEGER]) -> FLOAT + */ + private static FunctionResolver rand() { + FunctionName functionName = BuiltinFunctionName.RAND.getName(); + return new FunctionResolver(functionName, + new ImmutableMap.Builder() + .put( + new FunctionSignature(functionName, Collections.emptyList()), + noArgFunction(functionName, () -> new Random().nextFloat(), ExprCoreType.FLOAT)) + .put( + new FunctionSignature(functionName, Arrays.asList(ExprCoreType.INTEGER)), + unaryOperator( + functionName, n -> new Random(n).nextFloat(), ExprValueUtils::getIntegerValue, + ExprCoreType.FLOAT)) + .build()); + } + /** * Definition of round(x)/round(x, d) function. * Rounds the argument x to d decimal places, d defaults to 0 if not specified. diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java index c4ada14b0d..6e5f958a34 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java @@ -22,6 +22,7 @@ import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.STRING_TYPE_MISSING_VALUE_FILED; import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.STRING_TYPE_NULL_VALUE_FILED; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getDoubleValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getFloatValue; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; @@ -35,16 +36,12 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; -import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionTestBase; import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; -import com.google.common.collect.Lists; import java.math.BigDecimal; import java.math.RoundingMode; -import java.util.List; -import java.util.stream.Collectors; +import java.util.Random; import java.util.stream.Stream; import java.util.zip.CRC32; import org.junit.jupiter.api.DisplayNameGeneration; @@ -425,6 +422,15 @@ public void crc32_missing_value() { assertTrue(crc.valueOf(valueEnv()).isMissing()); } + /** + * Test constant e. + */ + @Test + public void test_e() { + FunctionExpression e = dsl.euler(); + assertThat(e.valueOf(valueEnv()), allOf(hasType(DOUBLE), hasValue(Math.E))); + } + /** * Test exp with integer value. */ @@ -1717,4 +1723,48 @@ public void truncate_null_missing() { assertEquals(LONG, truncate.type()); assertTrue(truncate.valueOf(valueEnv()).isMissing()); } + + /** + * Test constant pi. + */ + @Test + public void test_pi() { + FunctionExpression pi = dsl.pi(); + assertThat(pi.valueOf(valueEnv()), allOf(hasType(DOUBLE), hasValue(Math.PI))); + } + + /** + * Test rand with no argument. + */ + @Test + public void rand_no_arg() { + FunctionExpression rand = dsl.rand(); + assertEquals(FLOAT, rand.type()); + assertTrue( + getFloatValue(rand.valueOf(valueEnv())) >= 0 + && getFloatValue(rand.valueOf(valueEnv())) < 1); + assertEquals("rand()", rand.toString()); + } + + /** + * Test rand with integer value. + */ + @ParameterizedTest(name = "rand({0})") + @ValueSource(ints = {2, 3}) + public void rand_int_value(Integer n) { + FunctionExpression rand = dsl.rand(DSL.literal(n)); + assertEquals(FLOAT, rand.type()); + assertTrue( + getFloatValue(rand.valueOf(valueEnv())) >= 0 + && getFloatValue(rand.valueOf(valueEnv())) < 1); + assertEquals(getFloatValue(rand.valueOf(valueEnv())), new Random(n).nextFloat()); + assertEquals(String.format("rand(%s)", n), rand.toString()); + } + + @Test + public void rand_null_value() { + FunctionExpression rand = dsl.rand(DSL.ref(INT_TYPE_NULL_VALUE_FIELD, INTEGER)); + assertEquals(FLOAT, rand.type()); + assertTrue(rand.valueOf(valueEnv()).isNull()); + } } diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index 2904133b5a..dea0bbcf09 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -291,9 +291,19 @@ E Description ----------- -Specifications: +Usage: E() returns the Euler's number -1. E() -> DOUBLE +Return type: DOUBLE + +Example:: + + od> SELECT E() + fetched rows / total rows = 1/1 + +-------------------+ + | e() | + |-------------------| + | 2.718281828459045 | + +-------------------+ EXP @@ -548,9 +558,19 @@ PI Description ----------- -Specifications: +Usage: PI() returns the constant pi + +Return type: DOUBLE -1. PI() -> DOUBLE +Example:: + + od> SELECT PI() + fetched rows / total rows = 1/1 + +-------------------+ + | pi() | + |-------------------| + | 3.141592653589793 | + +-------------------+ POW @@ -616,10 +636,21 @@ RAND Description ----------- -Specifications: +Usage: RAND()/RAND(N) returns a random floating-point value in the range 0 <= value < 1.0. If integer N is specified, the seed is initialized prior to execution. One implication of this behavior is with identical argument N, rand(N) returns the same value each time, and thus produces a repeatable sequence of column values. + +Argument type: INTEGER -1. RAND() -> NUMBER -2. RAND(NUMBER T) -> T +Return type: FLOAT + +Example:: + + od> SELECT RAND(3) + fetched rows / total rows = 1/1 + +------------+ + | rand(3) | + |------------| + | 0.73105735 | + +------------+ REPLACE diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/MathematicalFunctionIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/MathematicalFunctionIT.java index 9c9bf42293..0afe4be28d 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/MathematicalFunctionIT.java @@ -20,7 +20,6 @@ import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.rows; import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRows; -import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifyDataRowsInOrder; import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; @@ -71,6 +70,18 @@ public void testCeiling() throws IOException { rows(32), rows(36), rows(28), rows(33), rows(36), rows(39), rows(34)); } + @Test + public void testE() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval f = e() | fields f", TEST_INDEX_BANK)); + verifySchema(result, schema("f", null, "double")); + verifyDataRows( + result, rows(Math.E), rows(Math.E), rows(Math.E), rows(Math.E), + rows(Math.E), rows(Math.E), rows(Math.E)); + } + @Test public void testExp() throws IOException { JSONObject result = @@ -277,4 +288,31 @@ public void testTruncate() throws IOException { verifyDataRows(result, rows(30), rows(30), rows(20), rows(30), rows(30), rows(30), rows(30)); } + + @Test + public void testPi() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval f = pi() | fields f", TEST_INDEX_BANK)); + verifySchema(result, schema("f", null, "double")); + verifyDataRows( + result, rows(Math.PI), rows(Math.PI), rows(Math.PI), rows(Math.PI), + rows(Math.PI), rows(Math.PI), rows(Math.PI)); + } + + @Test + public void testRand() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | eval f = rand() | fields f", TEST_INDEX_BANK)); + verifySchema(result, schema("f", null, "float")); + + result = + executeQuery( + String.format( + "source=%s | eval f = rand(5) | fields f", TEST_INDEX_BANK)); + verifySchema(result, schema("f", null, "float")); + } } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/MathematicalFunctionIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/MathematicalFunctionIT.java index 405d9058b5..746f81dc01 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/MathematicalFunctionIT.java @@ -58,6 +58,13 @@ public void testCrc32() throws IOException { verifyDataRows(result, rows(3259397556L)); } + @Test + public void testE() throws IOException { + JSONObject result = executeQuery("select e()"); + verifySchema(result, schema("e()", null, "double")); + verifyDataRows(result, rows(Math.E)); + } + @Test public void testMod() throws IOException { JSONObject result = executeQuery("select mod(3, 2)"); diff --git a/integ-test/src/test/resources/correctness/expressions/functions.txt b/integ-test/src/test/resources/correctness/expressions/functions.txt index 9e7090f444..0fe9cc4697 100644 --- a/integ-test/src/test/resources/correctness/expressions/functions.txt +++ b/integ-test/src/test/resources/correctness/expressions/functions.txt @@ -10,19 +10,6 @@ ceil(-1) ceil(0.0) ceil(0.4999) ceil(abs(1)) -power(2, 2) -power(2, -2) -power(2.1, 2) -power(2, -2.1) -power(abs(2), 2) -sign(0) -sign(-1) -sign(1) -sign(abs(1)) -sqrt(0) -sqrt(1) -sqrt(1.1) -sqrt(abs(1)) exp(0) exp(1) exp(-1) @@ -38,3 +25,17 @@ log(log(2)) log10(2) log10(2.1) log10(log10(2)) +pi() +power(2, 2) +power(2, -2) +power(2.1, 2) +power(2, -2.1) +power(abs(2), 2) +sign(0) +sign(-1) +sign(1) +sign(abs(1)) +sqrt(0) +sqrt(1) +sqrt(1.1) +sqrt(abs(1)) \ No newline at end of file diff --git a/ppl/src/main/antlr/OpenDistroPPLLexer.g4 b/ppl/src/main/antlr/OpenDistroPPLLexer.g4 index 2d34e9c49f..f7a68559e3 100644 --- a/ppl/src/main/antlr/OpenDistroPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenDistroPPLLexer.g4 @@ -139,6 +139,7 @@ CEIL: 'CEIL'; CEILING: 'CEILING'; CONV: 'CONV'; CRC32: 'CRC32'; +E: 'E'; EXP: 'EXP'; FLOOR: 'FLOOR'; LN: 'LN'; @@ -146,8 +147,10 @@ LOG: 'LOG'; LOG10: 'LOG10'; LOG2: 'LOG2'; MOD: 'MOD'; +PI: 'PI'; POW: 'POW'; POWER: 'POWER'; +RAND: 'RAND'; ROUND: 'ROUND'; SIGN: 'SIGN'; SQRT: 'SQRT'; diff --git a/ppl/src/main/antlr/OpenDistroPPLParser.g4 b/ppl/src/main/antlr/OpenDistroPPLParser.g4 index 7b393e9a37..325fec1b86 100644 --- a/ppl/src/main/antlr/OpenDistroPPLParser.g4 +++ b/ppl/src/main/antlr/OpenDistroPPLParser.g4 @@ -195,7 +195,7 @@ evalFunctionName ; functionArgs - : functionArg (COMMA functionArg)* + : (functionArg (COMMA functionArg)*)? ; functionArg @@ -203,8 +203,8 @@ functionArg ; mathematicalFunctionBase - : ABS | CEIL | CEILING | CONV | CRC32 | EXP | FLOOR | LN | LOG | LOG10 | LOG2 | MOD | POW | POWER - | ROUND | SIGN | SQRT | TRUNCATE + : ABS | CEIL | CEILING | CONV | CRC32 | E | EXP | FLOOR | LN | LOG | LOG10 | LOG2 | MOD | PI |POW | POWER + | RAND | ROUND | SIGN | SQRT | TRUNCATE ; dateAndTimeFunctionBase diff --git a/sql/src/main/antlr/OpenDistroSQLParser.g4 b/sql/src/main/antlr/OpenDistroSQLParser.g4 index a6517c5572..ce72d9a670 100644 --- a/sql/src/main/antlr/OpenDistroSQLParser.g4 +++ b/sql/src/main/antlr/OpenDistroSQLParser.g4 @@ -170,8 +170,8 @@ scalarFunctionName ; mathematicalFunctionName - : ABS | CEIL | CEILING | CONV | CRC32 | EXP | FLOOR | LN | LOG | LOG10 | LOG2 | MOD | POW | POWER - | ROUND | SIGN | SQRT | TRUNCATE + : ABS | CEIL | CEILING | CONV | CRC32 | E | EXP | FLOOR | LN | LOG | LOG10 | LOG2 | MOD | PI | POW | POWER + | RAND | ROUND | SIGN | SQRT | TRUNCATE ; dateTimeFunctionName @@ -179,7 +179,7 @@ dateTimeFunctionName ; functionArgs - : functionArg (COMMA functionArg)* + : (functionArg (COMMA functionArg)*)? ; functionArg