diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/AbstractExprNumberValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/AbstractExprNumberValue.java index 8a2ff1d523..6b16e00868 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/AbstractExprNumberValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/AbstractExprNumberValue.java @@ -32,6 +32,11 @@ public boolean isNumber() { return true; } + @Override + public Byte byteValue() { + return value.byteValue(); + } + @Override public Short shortValue() { return value.shortValue(); diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprByteValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprByteValue.java new file mode 100644 index 0000000000..4022cebdc8 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprByteValue.java @@ -0,0 +1,54 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; + +/** + * Expression Byte Value. + */ +public class ExprByteValue extends AbstractExprNumberValue { + + public ExprByteValue(Number value) { + super(value); + } + + @Override + public int compare(ExprValue other) { + return Byte.compare(byteValue(), other.byteValue()); + } + + @Override + public boolean equal(ExprValue other) { + return byteValue().equals(other.byteValue()); + } + + @Override + public Object value() { + return byteValue(); + } + + @Override + public ExprType type() { + return ExprCoreType.BYTE; + } + + @Override + public String toString() { + return value().toString(); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValue.java index b477738201..7b98c0b4cf 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValue.java @@ -77,6 +77,14 @@ default BindingTuple bindingTuples() { return BindingTuple.EMPTY; } + /** + * Get byte value. + */ + default Byte byteValue() { + throw new ExpressionEvaluationException( + "invalid to get byteValue from value of type " + type()); + } + /** * Get short value. */ diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java index 3fddbdde7f..976f2de8bb 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java @@ -44,6 +44,14 @@ public static ExprValue booleanValue(Boolean value) { return value ? LITERAL_TRUE : LITERAL_FALSE; } + public static ExprValue byteValue(Byte value) { + return new ExprByteValue(value); + } + + public static ExprValue shortValue(Short value) { + return new ExprShortValue(value); + } + public static ExprValue integerValue(Integer value) { return new ExprIntegerValue(value); } @@ -105,6 +113,10 @@ public static ExprValue fromObjectValue(Object o) { return tupleValue((Map) o); } else if (o instanceof List) { return collectionValue(((List) o)); + } else if (o instanceof Byte) { + return byteValue((Byte) o); + } else if (o instanceof Short) { + return shortValue((Short) o); } else if (o instanceof Integer) { return integerValue((Integer) o); } else if (o instanceof Long) { diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java index 9eaf21ebcd..fe12d2735c 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java @@ -34,7 +34,8 @@ public enum ExprCoreType implements ExprType { /** * Numbers. */ - SHORT, + BYTE, + SHORT(BYTE), INTEGER(SHORT), LONG(INTEGER), FLOAT(LONG), diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java index c562bc3ff3..9125e48412 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java @@ -32,6 +32,10 @@ public class DSL { private final BuiltinFunctionRepository repository; + public static LiteralExpression literal(Byte value) { + return new LiteralExpression(ExprValueUtils.byteValue(value)); + } + public static LiteralExpression literal(Short value) { return new LiteralExpression(new ExprShortValue(value)); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/ArithmeticFunction.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/ArithmeticFunction.java index 87f89ff228..178c4458f2 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/ArithmeticFunction.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/ArithmeticFunction.java @@ -15,12 +15,14 @@ package com.amazon.opendistroforelasticsearch.sql.expression.operator.arthmetic; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BYTE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.SHORT; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDoubleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprFloatValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; @@ -58,6 +60,10 @@ public static void register(BuiltinFunctionRepository repository) { private static FunctionResolver add() { return FunctionDSL.define(BuiltinFunctionName.ADD.getName(), + FunctionDSL.impl( + FunctionDSL.nullMissingHandling( + (v1, v2) -> new ExprByteValue(v1.byteValue() + v2.byteValue())), + BYTE, BYTE, BYTE), FunctionDSL.impl( FunctionDSL.nullMissingHandling( (v1, v2) -> new ExprShortValue(v1.shortValue() + v2.shortValue())), @@ -84,6 +90,10 @@ private static FunctionResolver add() { private static FunctionResolver subtract() { return FunctionDSL.define(BuiltinFunctionName.SUBTRACT.getName(), + FunctionDSL.impl( + FunctionDSL.nullMissingHandling( + (v1, v2) -> new ExprByteValue(v1.byteValue() - v2.byteValue())), + BYTE, BYTE, BYTE), FunctionDSL.impl( FunctionDSL.nullMissingHandling( (v1, v2) -> new ExprShortValue(v1.shortValue() - v2.shortValue())), @@ -110,6 +120,10 @@ private static FunctionResolver subtract() { private static FunctionResolver multiply() { return FunctionDSL.define(BuiltinFunctionName.MULTIPLY.getName(), + FunctionDSL.impl( + FunctionDSL.nullMissingHandling( + (v1, v2) -> new ExprByteValue(v1.byteValue() * v2.byteValue())), + BYTE, BYTE, BYTE), FunctionDSL.impl( FunctionDSL.nullMissingHandling( (v1, v2) -> new ExprShortValue(v1.shortValue() * v2.shortValue())), @@ -136,6 +150,10 @@ private static FunctionResolver multiply() { private static FunctionResolver divide() { return FunctionDSL.define(BuiltinFunctionName.DIVIDE.getName(), + FunctionDSL.impl( + FunctionDSL.nullMissingHandling( + (v1, v2) -> new ExprByteValue(v1.byteValue() / v2.byteValue())), + BYTE, BYTE, BYTE), FunctionDSL.impl( FunctionDSL.nullMissingHandling( (v1, v2) -> v2.shortValue() == 0 ? ExprNullValue.of() : @@ -167,6 +185,10 @@ private static FunctionResolver divide() { private static FunctionResolver modules() { return FunctionDSL.define(BuiltinFunctionName.MODULES.getName(), + FunctionDSL.impl( + FunctionDSL.nullMissingHandling( + (v1, v2) -> new ExprByteValue(v1.byteValue() % v2.byteValue())), + BYTE, BYTE, BYTE), FunctionDSL.impl( FunctionDSL.nullMissingHandling( (v1, v2) -> v2.shortValue() == 0 ? ExprNullValue.of() : diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunction.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunction.java index 2d8b854c9e..72308c7c0a 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunction.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunction.java @@ -15,6 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.expression.operator.arthmetic; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BYTE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; @@ -22,6 +23,7 @@ import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.SHORT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDoubleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprFloatValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; @@ -97,6 +99,9 @@ public static void register(BuiltinFunctionRepository repository) { */ private static FunctionResolver abs() { return FunctionDSL.define(BuiltinFunctionName.ABS.getName(), + FunctionDSL.impl( + FunctionDSL.nullMissingHandling(v -> new ExprByteValue(Math.abs(v.byteValue()))), + BYTE, BYTE), FunctionDSL.impl( FunctionDSL.nullMissingHandling(v -> new ExprShortValue(Math.abs(v.shortValue()))), SHORT, SHORT), @@ -286,6 +291,11 @@ private static FunctionResolver log2() { */ private static FunctionResolver mod() { return FunctionDSL.define(BuiltinFunctionName.MOD.getName(), + FunctionDSL.impl( + FunctionDSL.nullMissingHandling( + (v1, v2) -> v2.byteValue() == 0 ? ExprNullValue.of() : + new ExprByteValue(v1.byteValue() % v2.byteValue())), + BYTE, BYTE, BYTE), FunctionDSL.impl( FunctionDSL.nullMissingHandling( (v1, v2) -> v2.shortValue() == 0 ? ExprNullValue.of() : diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java index 6400e80153..4c97632958 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java @@ -68,7 +68,7 @@ public class ExprValueUtilsTest { testTuple.put("1", new ExprIntegerValue(1)); } - private static List numberValues = Stream.of(1, 1L, 1f, 1D) + private static List numberValues = Stream.of((byte) 1, (short) 1, 1, 1L, 1f, 1D) .map(ExprValueUtils::fromObjectValue).collect(Collectors.toList()); private static List nonNumberValues = Arrays.asList( @@ -86,6 +86,8 @@ public class ExprValueUtilsTest { Lists.newArrayList(Iterables.concat(numberValues, nonNumberValues)); private static List> numberValueExtractor = Arrays.asList( + ExprValue::byteValue, + ExprValue::shortValue, ExprValueUtils::getIntegerValue, ExprValueUtils::getLongValue, ExprValueUtils::getFloatValue, @@ -106,8 +108,8 @@ public class ExprValueUtilsTest { Iterables.concat(numberValueExtractor, nonNumberValueExtractor, dateAndTimeValueExtractor)); private static List numberTypes = - Arrays.asList(ExprCoreType.INTEGER, ExprCoreType.LONG, ExprCoreType.FLOAT, - ExprCoreType.DOUBLE); + Arrays.asList(ExprCoreType.BYTE, ExprCoreType.SHORT, ExprCoreType.INTEGER, ExprCoreType.LONG, + ExprCoreType.FLOAT, ExprCoreType.DOUBLE); private static List nonNumberTypes = Arrays.asList(STRING, BOOLEAN, ARRAY, STRUCT); private static List dateAndTimeTypes = @@ -116,7 +118,7 @@ public class ExprValueUtilsTest { Lists.newArrayList(Iterables.concat(numberTypes, nonNumberTypes, dateAndTimeTypes)); private static Stream getValueTestArgumentStream() { - List expectedValues = Arrays.asList(1, 1L, 1f, 1D, "1", true, + List expectedValues = Arrays.asList((byte) 1, (short) 1, 1, 1L, 1f, 1D, "1", true, Arrays.asList(integerValue(1)), ImmutableMap.of("1", integerValue(1)), LocalDate.parse("2012-08-07"), @@ -248,6 +250,8 @@ public void constructDateAndTimeValue() { @Test public void hashCodeTest() { + assertEquals(new ExprByteValue(1).hashCode(), new ExprByteValue(1).hashCode()); + assertEquals(new ExprShortValue(1).hashCode(), new ExprShortValue(1).hashCode()); assertEquals(new ExprIntegerValue(1).hashCode(), new ExprIntegerValue(1).hashCode()); assertEquals(new ExprStringValue("1").hashCode(), new ExprStringValue("1").hashCode()); assertEquals(new ExprCollectionValue(ImmutableList.of(new ExprIntegerValue(1))).hashCode(), diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprValueOrderingTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprValueOrderingTest.java index e80668ee57..6601d27a84 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprValueOrderingTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprValueOrderingTest.java @@ -19,6 +19,7 @@ import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.byteValue; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.collectionValue; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.doubleValue; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.floatValue; @@ -143,6 +144,14 @@ public void natural_order_long_value() { assertEquals(-1, ordering.compare(longValue(4L), longValue(5L))); } + @Test + public void natural_order_byte_value() { + ExprValueOrdering ordering = ExprValueOrdering.natural(); + assertEquals(1, ordering.compare(byteValue((byte) 5), byteValue((byte) 4))); + assertEquals(0, ordering.compare(byteValue((byte) 5), byteValue((byte) 5))); + assertEquals(-1, ordering.compare(byteValue((byte) 4), byteValue((byte) 5))); + } + @Test public void natural_order_boolean_value() { ExprValueOrdering ordering = ExprValueOrdering.natural(); diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/WideningTypeRuleTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/WideningTypeRuleTest.java index 8b4f023ab3..0e614a4df5 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/WideningTypeRuleTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/function/WideningTypeRuleTest.java @@ -15,6 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.expression.function; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BYTE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; @@ -44,6 +45,11 @@ class WideningTypeRuleTest { private static Table numberWidenRule = new ImmutableTable.Builder() + .put(BYTE, SHORT, 1) + .put(BYTE, INTEGER, 2) + .put(BYTE, LONG, 3) + .put(BYTE, FLOAT, 4) + .put(BYTE, DOUBLE, 5) .put(SHORT, INTEGER, 1) .put(SHORT, LONG, 2) .put(SHORT, FLOAT, 3) diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java index cc55190635..11ec551b01 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/ArithmeticFunctionTest.java @@ -27,6 +27,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDoubleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprFloatValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; @@ -56,10 +57,12 @@ class ArithmeticFunctionTest extends ExpressionTestBase { private static Stream arithmeticFunctionArguments() { - List numberOp1 = Arrays.asList(new ExprShortValue(3), new ExprIntegerValue(3), - new ExprLongValue(3L), new ExprFloatValue(3f), new ExprDoubleValue(3D)); + List numberOp1 = Arrays.asList(new ExprByteValue(3), new ExprShortValue(3), + new ExprIntegerValue(3), new ExprLongValue(3L), new ExprFloatValue(3f), + new ExprDoubleValue(3D)); List numberOp2 = - Arrays.asList(new ExprShortValue(2), new ExprIntegerValue(2), new ExprLongValue(3L), + Arrays.asList(new ExprByteValue(2), new ExprShortValue(2), new ExprIntegerValue(2), + new ExprLongValue(3L), new ExprFloatValue(2f), new ExprDoubleValue(2D)); return Lists.cartesianProduct(numberOp1, numberOp2).stream() .map(list -> Arguments.of(list.get(0), list.get(1))); @@ -209,6 +212,29 @@ protected void assertValueEqual(BuiltinFunctionName builtinFunctionName, ExprTyp ExprValue op2, ExprValue actual) { switch ((ExprCoreType) type) { + case BYTE: + Byte vb1 = op1.byteValue(); + Byte vb2 = op2.byteValue(); + Integer vbActual = actual.integerValue(); + switch (builtinFunctionName) { + case ADD: + assertEquals(vb1 + vb2, vbActual); + return; + case SUBTRACT: + assertEquals(vb1 - vb2, vbActual); + return; + case DIVIDE: + assertEquals(vb1 / vb2, vbActual); + return; + case MULTIPLY: + assertEquals(vb1 * vb2, vbActual); + return; + case MODULES: + assertEquals(vb1 % vb2, vbActual); + return; + default: + throw new IllegalStateException("illegal function name: " + builtinFunctionName); + } case SHORT: Short vs1 = op1.shortValue(); Short vs2 = op2.shortValue(); diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java index 586b95b4bf..7beb733454 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java @@ -23,6 +23,7 @@ import static com.amazon.opendistroforelasticsearch.sql.config.TestConfig.STRING_TYPE_NULL_VALUE_FILED; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getDoubleValue; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getFloatValue; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BYTE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; @@ -38,6 +39,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprShortValue; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionTestBase; @@ -58,6 +60,11 @@ @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) public class MathematicalFunctionTest extends ExpressionTestBase { + private static Stream testLogByteArguments() { + Stream.Builder builder = Stream.builder(); + return builder.add(Arguments.of((byte) 2, (byte) 2)).build(); + } + private static Stream testLogShortArguments() { Stream.Builder builder = Stream.builder(); return builder.add(Arguments.of((short) 2, (short) 2)).build(); @@ -97,6 +104,17 @@ private static Stream trigonometricDoubleArguments() { .add(Arguments.of(1D, 2D)).build(); } + /** + * Test abs with byte value. + */ + @ParameterizedTest(name = "abs({0})") + @ValueSource(bytes = {-2, 2}) + public void abs_byte_value(Byte value) { + FunctionExpression abs = dsl.abs(DSL.literal(value)); + assertThat(abs.valueOf(valueEnv()), allOf(hasType(BYTE), hasValue(((byte) Math.abs(value))))); + assertEquals(String.format("abs(%s)", value.toString()), abs.toString()); + } + /** * Test abs with integer value. */ @@ -1045,21 +1063,39 @@ public void log2_missing_value() { assertTrue(log.valueOf(valueEnv()).isMissing()); } + /** + * Test mod with byte value. + */ + @ParameterizedTest(name = "mod({0}, {1})") + @MethodSource("testLogByteArguments") + public void mod_byte_value(Byte v1, Byte v2) { + FunctionExpression mod = dsl.mod(DSL.literal(v1), DSL.literal(v2)); + + assertThat( + mod.valueOf(valueEnv()), + allOf(hasType(BYTE), hasValue(Integer.valueOf(v1 % v2).byteValue()))); + assertEquals(String.format("mod(%s, %s)", v1, v2), mod.toString()); + + mod = dsl.mod(DSL.literal(v1), DSL.literal(new ExprByteValue(0))); + assertEquals(BYTE, mod.type()); + assertTrue(mod.valueOf(valueEnv()).isNull()); + } + /** * Test mod with short value. */ @ParameterizedTest(name = "mod({0}, {1})") - @MethodSource("testLogIntegerArguments") - public void mod_short_value(Integer v1, Integer v2) { - FunctionExpression mod = dsl.mod(DSL.literal(v1.shortValue()), DSL.literal(v2.shortValue())); + @MethodSource("testLogShortArguments") + public void mod_short_value(Short v1, Short v2) { + FunctionExpression mod = dsl.mod(DSL.literal(v1), DSL.literal(v2)); assertThat( mod.valueOf(valueEnv()), allOf(hasType(SHORT), - hasValue(Integer.valueOf(v1.shortValue() % v2.shortValue()).shortValue()))); + hasValue(Integer.valueOf(v1 % v2).shortValue()))); assertEquals(String.format("mod(%s, %s)", v1, v2), mod.toString()); - mod = dsl.mod(DSL.literal(v1.shortValue()), DSL.literal(new ExprShortValue(0))); + mod = dsl.mod(DSL.literal(v1), DSL.literal(new ExprShortValue(0))); assertEquals(SHORT, mod.type()); assertTrue(mod.valueOf(valueEnv()).isNull()); } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java index 3c7dec7f15..aa7402142c 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java @@ -38,6 +38,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprCollectionValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDoubleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprFloatValue; @@ -55,6 +56,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import com.sun.org.apache.xpath.internal.Arg; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.ObjectInputStream; @@ -100,6 +102,7 @@ private static Stream binaryPredicateArguments() { private static Stream testEqualArguments() { Stream.Builder builder = Stream.builder(); + builder.add(Arguments.of(new ExprByteValue(1), new ExprByteValue(1))); builder.add(Arguments.of(new ExprShortValue(1), new ExprShortValue(1))); builder.add(Arguments.of(new ExprIntegerValue(1), new ExprIntegerValue(1))); builder.add(Arguments.of(new ExprLongValue(1L), new ExprLongValue(1L))); @@ -117,7 +120,8 @@ private static Stream testEqualArguments() { private static Stream testNotEqualArguments() { List> arguments = Arrays.asList( - Arrays.asList(1, 2), Arrays.asList(1L, 2L), Arrays.asList(1F, 2F), Arrays.asList(1D, 2D), + Arrays.asList((byte) 1, (byte) 2), Arrays.asList(1, 2), Arrays.asList(1L, 2L), + Arrays.asList(1F, 2F), Arrays.asList(1D, 2D), Arrays.asList("str0", "str1"), Arrays.asList(true, false), Arrays.asList(ImmutableList.of(1), ImmutableList.of(2)), Arrays.asList(ImmutableMap.of("str", 1), ImmutableMap.of("str", 2)) diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java index 66197ef530..6913ce6a50 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/utils/ComparisonUtil.java @@ -21,6 +21,7 @@ import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getLongValue; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getStringValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDoubleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprFloatValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; @@ -42,7 +43,9 @@ public static int compare(ExprValue v1, ExprValue v2) { throw new ExpressionEvaluationException("invalid to call compare operation on null value"); } - if (v1 instanceof ExprShortValue) { + if (v1 instanceof ExprByteValue) { + return v1.byteValue().compareTo(v2.byteValue()); + } else if (v1 instanceof ExprShortValue) { return v1.shortValue().compareTo(v2.shortValue()); } else if (v1 instanceof ExprIntegerValue) { return getIntegerValue(v1).compareTo(getIntegerValue(v2)); diff --git a/docs/user/general/datatypes.rst b/docs/user/general/datatypes.rst index 70debe4e93..312db30464 100644 --- a/docs/user/general/datatypes.rst +++ b/docs/user/general/datatypes.rst @@ -23,6 +23,10 @@ The ODFE SQL Engine support the following data types. +===============+ | boolean | +---------------+ +| byte | ++---------------+ +| short | ++---------------+ | integer | +---------------+ | long | @@ -49,6 +53,8 @@ The ODFE SQL Engine support the following data types. +---------------+ | geo_point | +---------------+ +| binary | ++---------------+ | struct | +---------------+ | array | @@ -64,14 +70,20 @@ The table below list the mapping between Elasticsearch Data Type, ODFE SQL Data +====================+===============+===========+ | boolean | boolean | BOOLEAN | +--------------------+---------------+-----------+ +| byte | byte | TINYINT | ++--------------------+---------------+-----------+ +| short | byte | SMALLINT | ++--------------------+---------------+-----------+ | integer | integer | INTEGER | +--------------------+---------------+-----------+ -| long | long | LONG | +| long | long | BIGINT | +--------------------+---------------+-----------+ -| float | float | FLOAT | +| float | float | REAL | +--------------------+---------------+-----------+ | half_float | float | FLOAT | +--------------------+---------------+-----------+ +| scaled_float | float | DOUBLE | ++--------------------+---------------+-----------+ | double | double | DOUBLE | +--------------------+---------------+-----------+ | keyword | string | VARCHAR | @@ -80,13 +92,15 @@ The table below list the mapping between Elasticsearch Data Type, ODFE SQL Data +--------------------+---------------+-----------+ | date | timestamp | TIMESTAMP | +--------------------+---------------+-----------+ -| ip | ip | IP | +| ip | ip | VARCHAR | +--------------------+---------------+-----------+ | date | timestamp | TIMESTAMP | +--------------------+---------------+-----------+ +| binary | binary | VARBINARY | ++--------------------+---------------+-----------+ | object | struct | STRUCT | +--------------------+---------------+-----------+ -| nested | array | TBD | +| nested | array | STRUCT | +--------------------+---------------+-----------+ Notes: Not all the ODFE SQL Type has correspond Elasticsearch Type. e.g. data and time. To use function which required such data type, user should explict convert the data type. diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataType.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataType.java index 761b937ab6..e52e338dfe 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataType.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataType.java @@ -46,7 +46,9 @@ public enum ElasticsearchDataType implements ExprType { ES_IP(Arrays.asList(UNKNOWN), "ip"), - ES_GEO_POINT(Arrays.asList(UNKNOWN), "geo_point"); + ES_GEO_POINT(Arrays.asList(UNKNOWN), "geo_point"), + + ES_BINARY(Arrays.asList(UNKNOWN), "binary"); /** * Parent of current type. diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprBinaryValue.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprBinaryValue.java new file mode 100644 index 0000000000..35800bb649 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprBinaryValue.java @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value; + +import com.amazon.opendistroforelasticsearch.sql.data.model.AbstractExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType; +import lombok.EqualsAndHashCode; + +/** + * Elasticsearch BinaryValue. + * Todo, add this to avoid the unknown value type exception, the implementation will be changed. + */ +@EqualsAndHashCode(callSuper = false) +public class ElasticsearchExprBinaryValue extends AbstractExprValue { + private final String encodedString; + + public ElasticsearchExprBinaryValue(String encodedString) { + this.encodedString = encodedString; + } + + @Override + public int compare(ExprValue other) { + return encodedString.compareTo((String) other.value()); + } + + @Override + public boolean equal(ExprValue other) { + return encodedString.equals(other.value()); + } + + @Override + public Object value() { + return encodedString; + } + + @Override + public ExprType type() { + return ElasticsearchDataType.ES_BINARY; + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java index 252b864acc..89dccc0b50 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java @@ -20,13 +20,16 @@ import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.nullValue; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BYTE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.SHORT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_BINARY; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_GEO_POINT; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_IP; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT; @@ -35,11 +38,13 @@ import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchDateFormatters.STRICT_DATE_OPTIONAL_TIME_FORMATTER; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprByteValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprCollectionValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprDoubleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprFloatValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprLongValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprShortValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTimestampValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; @@ -103,6 +108,10 @@ private ExprValue construct(String field, JsonNode value) { return constructInteger(value.intValue()); } else if (type.equals(LONG)) { return constructLong(value.longValue()); + } else if (type.equals(SHORT)) { + return constructShort(value.shortValue()); + } else if (type.equals(BYTE)) { + return constructByte(((byte)value.shortValue())); } else if (type.equals(FLOAT)) { return constructFloat(value.floatValue()); } else if (type.equals(DOUBLE)) { @@ -130,6 +139,8 @@ private ExprValue construct(String field, JsonNode value) { } else if (type.equals(ES_GEO_POINT)) { return new ElasticsearchExprGeoPointValue(value.get("lat").doubleValue(), value.get("lon").doubleValue()); + } else if (type.equals(ES_BINARY)) { + return new ElasticsearchExprBinaryValue(value.textValue()); } else { throw new IllegalStateException( String.format( @@ -211,6 +222,14 @@ private ExprLongValue constructLong(Long value) { return new ExprLongValue(value); } + private ExprShortValue constructShort(Short value) { + return new ExprShortValue(value); + } + + private ExprByteValue constructByte(Byte value) { + return new ExprByteValue(value); + } + private ExprFloatValue constructFloat(Float value) { return new ExprFloatValue(value); } diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java index 5b8ed19357..49c72089ab 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java @@ -54,10 +54,13 @@ public class ElasticsearchIndex implements Table { .put("text", ElasticsearchDataType.ES_TEXT) .put("text_keyword", ElasticsearchDataType.ES_TEXT_KEYWORD) .put("keyword", ExprCoreType.STRING) + .put("byte", ExprCoreType.BYTE) + .put("short", ExprCoreType.SHORT) .put("integer", ExprCoreType.INTEGER) .put("long", ExprCoreType.LONG) .put("float", ExprCoreType.FLOAT) .put("half_float", ExprCoreType.FLOAT) + .put("scaled_float", ExprCoreType.DOUBLE) .put("double", ExprCoreType.DOUBLE) .put("boolean", ExprCoreType.BOOLEAN) .put("nested", ExprCoreType.ARRAY) @@ -65,6 +68,7 @@ public class ElasticsearchIndex implements Table { .put("date", ExprCoreType.TIMESTAMP) .put("ip", ElasticsearchDataType.ES_IP) .put("geo_point", ElasticsearchDataType.ES_GEO_POINT) + .put("binary", ElasticsearchDataType.ES_BINARY) .build(); /** Elasticsearch client connection. */ diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprBinaryValueTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprBinaryValueTest.java new file mode 100644 index 0000000000..ad5ca8a097 --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprBinaryValueTest.java @@ -0,0 +1,55 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value; + +import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_BINARY; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +public class ElasticsearchExprBinaryValueTest { + + @Test + public void compare() { + assertEquals( + 0, + new ElasticsearchExprBinaryValue("U29tZSBiaW5hcnkgYmxvYg==") + .compare(new ElasticsearchExprBinaryValue("U29tZSBiaW5hcnkgYmxvYg=="))); + } + + @Test + public void equal() { + ElasticsearchExprBinaryValue value = + new ElasticsearchExprBinaryValue("U29tZSBiaW5hcnkgYmxvYg=="); + assertTrue(value.equal(new ElasticsearchExprBinaryValue("U29tZSBiaW5hcnkgYmxvYg=="))); + } + + @Test + public void value() { + ElasticsearchExprBinaryValue value = + new ElasticsearchExprBinaryValue("U29tZSBiaW5hcnkgYmxvYg=="); + assertEquals("U29tZSBiaW5hcnkgYmxvYg==", value.value()); + } + + @Test + public void type() { + ElasticsearchExprBinaryValue value = + new ElasticsearchExprBinaryValue("U29tZSBiaW5hcnkgYmxvYg=="); + assertEquals(ES_BINARY, value.type()); + } + +} diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java index ac9b302442..1a4cb83a6d 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java @@ -18,21 +18,26 @@ package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.booleanValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.byteValue; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.doubleValue; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.floatValue; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.longValue; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.nullValue; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.shortValue; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.stringValue; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BYTE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.SHORT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_BINARY; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_GEO_POINT; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_IP; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT; @@ -58,6 +63,8 @@ class ElasticsearchExprValueFactoryTest { private static final Map MAPPING = new ImmutableMap.Builder() + .put("byteV", BYTE) + .put("shortV", SHORT) .put("intV", INTEGER) .put("longV", LONG) .put("floatV", FLOAT) @@ -75,6 +82,7 @@ class ElasticsearchExprValueFactoryTest { .put("textKeywordV", ES_TEXT_KEYWORD) .put("ipV", ES_IP) .put("geoV", ES_GEO_POINT) + .put("binaryV", ES_BINARY) .build(); private ElasticsearchExprValueFactory exprValueFactory = new ElasticsearchExprValueFactory(MAPPING); @@ -85,6 +93,16 @@ public void constructNullValue() { assertEquals(nullValue(), constructFromObject("intV", null)); } + @Test + public void constructByte() { + assertEquals(byteValue((byte) 1), tupleValue("{\"byteV\":1}").get("byteV")); + } + + @Test + public void constructShort() { + assertEquals(shortValue((short) 1), tupleValue("{\"shortV\":1}").get("shortV")); + } + @Test public void constructInteger() { assertEquals(integerValue(1), tupleValue("{\"intV\":1}").get("intV")); @@ -217,6 +235,12 @@ public void constructGeoPoint() { tupleValue("{\"geoV\":{\"lat\":42.60355556,\"lon\":-97.25263889}}").get("geoV")); } + @Test + public void constructBinary() { + assertEquals(new ElasticsearchExprBinaryValue("U29tZSBiaW5hcnkgYmxvYg=="), + tupleValue("{\"binaryV\":\"U29tZSBiaW5hcnkgYmxvYg==\"}").get("binaryV")); + } + @Test public void constructFromInvalidJsonThrowException() { IllegalStateException exception = diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java index bec7d9e8ce..0ca1aeb8b3 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java @@ -107,6 +107,9 @@ void getFieldTypes() { .put("family", "nested") .put("employer", "object") .put("birthday", "date") + .put("id1", "byte") + .put("id2", "short") + .put("blob", "binary") .build()))); Table index = new ElasticsearchIndex(client, settings, "test"); @@ -114,17 +117,21 @@ void getFieldTypes() { assertThat( fieldTypes, allOf( - aMapWithSize(10), - hasEntry("name", (ExprType) ExprCoreType.STRING), + aMapWithSize(13), + hasEntry("name", ExprCoreType.STRING), hasEntry("address", (ExprType) ElasticsearchDataType.ES_TEXT), - hasEntry("age", (ExprType) ExprCoreType.INTEGER), + hasEntry("age", ExprCoreType.INTEGER), hasEntry("account_number", ExprCoreType.LONG), - hasEntry("balance1", (ExprType) ExprCoreType.FLOAT), - hasEntry("balance2", (ExprType) ExprCoreType.DOUBLE), - hasEntry("gender", (ExprType) ExprCoreType.BOOLEAN), - hasEntry("family", (ExprType) ExprCoreType.ARRAY), - hasEntry("employer", (ExprType) ExprCoreType.STRUCT), - hasEntry("birthday", (ExprType) ExprCoreType.TIMESTAMP))); + hasEntry("balance1", ExprCoreType.FLOAT), + hasEntry("balance2", ExprCoreType.DOUBLE), + hasEntry("gender", ExprCoreType.BOOLEAN), + hasEntry("family", ExprCoreType.ARRAY), + hasEntry("employer", ExprCoreType.STRUCT), + hasEntry("birthday", ExprCoreType.TIMESTAMP), + hasEntry("id1", ExprCoreType.BYTE), + hasEntry("id2", ExprCoreType.SHORT), + hasEntry("blob", (ExprType) ElasticsearchDataType.ES_BINARY) + )); } @Test diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java index 4dc4cc5b8c..1548895c27 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/SQLIntegTestCase.java @@ -20,6 +20,8 @@ import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getAccountIndexMapping; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getBankIndexMapping; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getBankWithNullValuesIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getDataTypeNonnumericIndexMapping; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getDataTypeNumericIndexMapping; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getDateIndexMapping; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getDateTimeIndexMapping; import static com.amazon.opendistroforelasticsearch.sql.legacy.TestUtils.getDeepNestedIndexMapping; @@ -519,7 +521,15 @@ public enum Index { DEEP_NESTED(TestsConstants.TEST_INDEX_DEEP_NESTED, "_doc", getDeepNestedIndexMapping(), - "src/test/resources/deep_nested_index_data.json"); + "src/test/resources/deep_nested_index_data.json"), + DATA_TYPE_NUMERIC(TestsConstants.TEST_INDEX_DATATYPE_NUMERIC, + "_doc", + getDataTypeNumericIndexMapping(), + "src/test/resources/datatypes_numeric.json"), + DATA_TYPE_NONNUMERIC(TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC, + "_doc", + getDataTypeNonnumericIndexMapping(), + "src/test/resources/datatypes.json"); private final String name; private final String type; diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java index 6219b90a5c..1bb43bfb8d 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestUtils.java @@ -243,6 +243,16 @@ public static String getDeepNestedIndexMapping() { return getMappingFile(mappingFile); } + public static String getDataTypeNumericIndexMapping() { + String mappingFile = "datatypes_numeric_index_mapping.json"; + return getMappingFile(mappingFile); + } + + public static String getDataTypeNonnumericIndexMapping() { + String mappingFile = "datatypes_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static void loadBulk(Client client, String jsonPath, String defaultIndex) throws Exception { System.out.println(String.format("Loading file %s into elasticsearch cluster", jsonPath)); diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java index f742db269d..3c26df77d1 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/TestsConstants.java @@ -56,7 +56,8 @@ public class TestsConstants { public final static String TEST_INDEX_DATE_TIME = TEST_INDEX + "_datetime"; public final static String TEST_INDEX_DEEP_NESTED = TEST_INDEX + "_deep_nested"; public final static String TEST_INDEX_STRINGS = TEST_INDEX + "_strings"; - + public final static String TEST_INDEX_DATATYPE_NUMERIC = TEST_INDEX + "_datatypes_numeric"; + public final static String TEST_INDEX_DATATYPE_NONNUMERIC = TEST_INDEX + "_datatypes_nonnumeric"; public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; public final static String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/DataTypeIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/DataTypeIT.java new file mode 100644 index 0000000000..e609bd93e3 --- /dev/null +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/DataTypeIT.java @@ -0,0 +1,67 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.ppl; + +import static com.amazon.opendistroforelasticsearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NONNUMERIC; +import static com.amazon.opendistroforelasticsearch.sql.legacy.SQLIntegTestCase.Index.DATA_TYPE_NUMERIC; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; +import static com.amazon.opendistroforelasticsearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.schema; +import static com.amazon.opendistroforelasticsearch.sql.util.MatcherUtils.verifySchema; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.Test; + +public class DataTypeIT extends PPLIntegTestCase { + + @Override + public void init() throws IOException { + loadIndex(DATA_TYPE_NUMERIC); + loadIndex(DATA_TYPE_NONNUMERIC); + } + + @Test + public void test_numeric_data_types() throws IOException { + JSONObject result = executeQuery( + String.format("source=%s", TEST_INDEX_DATATYPE_NUMERIC)); + verifySchema(result, + schema("long_number", "long"), + schema("integer_number", "integer"), + schema("short_number", "short"), + schema("byte_number", "byte"), + schema("double_number", "double"), + schema("float_number", "float"), + schema("half_float_number", "float"), + schema("scaled_float_number", "double")); + } + + @Test + public void test_nonnumeric_data_types() throws IOException { + JSONObject result = executeQuery( + String.format("source=%s", TEST_INDEX_DATATYPE_NONNUMERIC)); + verifySchema(result, + schema("boolean_value", "boolean"), + schema("keyword_value", "string"), + schema("text_value", "string"), + schema("binary_value", "binary"), + schema("date_value", "timestamp"), + schema("ip_value", "ip"), + schema("object_value", "struct"), + schema("nested_value", "array")); + } + +} diff --git a/integ-test/src/test/resources/datatypes.json b/integ-test/src/test/resources/datatypes.json new file mode 100644 index 0000000000..92ad732e07 --- /dev/null +++ b/integ-test/src/test/resources/datatypes.json @@ -0,0 +1,2 @@ +{"index":{"_id":"1"}} +{"boolean_value": true, "keyword_value": "keyword", "text_value": "text", "binary_value": "U29tZSBiaW5hcnkgYmxvYg==", "date_value": "2020-10-13 13:00:00", "ip_value": "127.0.0.0.1", "object_value": {"first": "Dale", "last": "Dale"}, "nested_value": [{"first" : "John", "last" : "Smith"}, {"first" : "Alice", "last" : "White"}} diff --git a/integ-test/src/test/resources/datatypes_numeric.json b/integ-test/src/test/resources/datatypes_numeric.json new file mode 100644 index 0000000000..5aa3b00511 --- /dev/null +++ b/integ-test/src/test/resources/datatypes_numeric.json @@ -0,0 +1,2 @@ +{"index":{"_id":"1"}} +{"long_number": 1, "integer_number": 2, "short_number": 3, "byte_number": 4, "double_number": 5.1, "float_number": 6.2, "half_float_number": 7.3, "scaled_float_number": 8.4} diff --git a/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json new file mode 100644 index 0000000000..48007ee4f0 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/datatypes_index_mapping.json @@ -0,0 +1,37 @@ +{ + "mappings": { + "properties": { + "boolean_value": { + "type": "boolean" + }, + "keyword_value": { + "type": "keyword" + }, + "text_value": { + "type": "text" + }, + "binary_value": { + "type": "binary" + }, + "date_value": { + "type": "date" + }, + "ip_value": { + "type": "ip" + }, + "object_value": { + "properties": { + "first": { + "type": "text" + }, + "last": { + "type": "text" + } + } + }, + "nested_value": { + "type": "nested" + } + } + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/indexDefinitions/datatypes_numeric_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/datatypes_numeric_index_mapping.json new file mode 100644 index 0000000000..c15503c271 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/datatypes_numeric_index_mapping.json @@ -0,0 +1,31 @@ +{ + "mappings": { + "properties": { + "long_number":{ + "type": "long" + }, + "integer_number": { + "type": "integer" + }, + "short_number": { + "type": "short" + }, + "byte_number": { + "type": "byte" + }, + "double_number": { + "type": "double" + }, + "float_number": { + "type": "float" + }, + "half_float_number": { + "type": "half_float" + }, + "scaled_float_number": { + "type": "scaled_float", + "scaling_factor": 100 + } + } + } +} \ No newline at end of file diff --git a/sql-jdbc/src/main/java/com/amazon/opendistroforelasticsearch/jdbc/types/BinaryType.java b/sql-jdbc/src/main/java/com/amazon/opendistroforelasticsearch/jdbc/types/BinaryType.java new file mode 100644 index 0000000000..024beb7e79 --- /dev/null +++ b/sql-jdbc/src/main/java/com/amazon/opendistroforelasticsearch/jdbc/types/BinaryType.java @@ -0,0 +1,41 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.jdbc.types; + +import java.sql.SQLException; +import java.util.Map; + +public class BinaryType implements TypeHelper { + + public static final BinaryType INSTANCE = new BinaryType(); + + private BinaryType() { + + } + + @Override + public String fromValue(Object value, Map conversionParams) throws SQLException { + if (value == null) + return null; + else + return String.valueOf(value); + } + + @Override + public String getTypeName() { + return "Binary"; + } +} diff --git a/sql-jdbc/src/main/java/com/amazon/opendistroforelasticsearch/jdbc/types/ElasticsearchType.java b/sql-jdbc/src/main/java/com/amazon/opendistroforelasticsearch/jdbc/types/ElasticsearchType.java index 09656c1008..8c71bf8103 100644 --- a/sql-jdbc/src/main/java/com/amazon/opendistroforelasticsearch/jdbc/types/ElasticsearchType.java +++ b/sql-jdbc/src/main/java/com/amazon/opendistroforelasticsearch/jdbc/types/ElasticsearchType.java @@ -75,6 +75,7 @@ public enum ElasticsearchType { DATE(JDBCType.DATE, Date.class, 24, 24, false), TIME(JDBCType.TIME, Time.class, 24, 24, false), TIMESTAMP(JDBCType.TIMESTAMP, Timestamp.class, 24, 24, false), + BINARY(JDBCType.VARBINARY, String.class, Integer.MAX_VALUE, 0, false), NULL(JDBCType.NULL, null, 0, 0, false), UNSUPPORTED(JDBCType.OTHER, null, 0, 0, false); @@ -96,6 +97,7 @@ public enum ElasticsearchType { jdbcTypeToESTypeMap.put(JDBCType.TIMESTAMP, TIMESTAMP); jdbcTypeToESTypeMap.put(JDBCType.TIME, TIME); jdbcTypeToESTypeMap.put(JDBCType.DATE, DATE); + jdbcTypeToESTypeMap.put(JDBCType.VARBINARY, BINARY); } /** diff --git a/sql-jdbc/src/main/java/com/amazon/opendistroforelasticsearch/jdbc/types/TypeConverters.java b/sql-jdbc/src/main/java/com/amazon/opendistroforelasticsearch/jdbc/types/TypeConverters.java index 71162e585a..1b11cdfb1e 100644 --- a/sql-jdbc/src/main/java/com/amazon/opendistroforelasticsearch/jdbc/types/TypeConverters.java +++ b/sql-jdbc/src/main/java/com/amazon/opendistroforelasticsearch/jdbc/types/TypeConverters.java @@ -21,6 +21,7 @@ import java.sql.Time; import java.sql.Timestamp; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -60,6 +61,8 @@ public class TypeConverters { tcMap.put(JDBCType.SMALLINT, new SmallIntTypeConverter()); tcMap.put(JDBCType.INTEGER, new IntegerTypeConverter()); tcMap.put(JDBCType.BIGINT, new BigIntTypeConverter()); + + tcMap.put(JDBCType.BINARY, new BinaryTypeConverter()); } public static TypeConverter getInstance(JDBCType jdbcType) { @@ -222,6 +225,24 @@ public Set getSupportedJavaClasses() { } } + public static class BinaryTypeConverter extends BaseTypeConverter { + + private static final Set supportedJavaClasses = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList( + String.class + ))); + + @Override + public Class getDefaultJavaClass() { + return String.class; + } + + @Override + public Set getSupportedJavaClasses() { + return supportedJavaClasses; + } + } + public static class IntegerTypeConverter extends BaseTypeConverter { private static final Set supportedJavaClasses = Collections.unmodifiableSet( diff --git a/sql-jdbc/src/test/java/com/amazon/opendistroforelasticsearch/jdbc/types/BinaryTypeTests.java b/sql-jdbc/src/test/java/com/amazon/opendistroforelasticsearch/jdbc/types/BinaryTypeTests.java new file mode 100644 index 0000000000..1b58e03409 --- /dev/null +++ b/sql-jdbc/src/test/java/com/amazon/opendistroforelasticsearch/jdbc/types/BinaryTypeTests.java @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.jdbc.types; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +public class BinaryTypeTests { + + @ParameterizedTest + @CsvSource(value = { + "U29tZSBiaW5hcnkgYmxvYg==, U29tZSBiaW5hcnkgYmxvYg==", + "TWFuIGlzIGRpc3Rpbmd1aXN, TWFuIGlzIGRpc3Rpbmd1aXN", + "YW55IGNhcm5hbCBwbGVhc3VyZS4=, YW55IGNhcm5hbCBwbGVhc3VyZS4=" + }) + void testTimeFromString(String inputString, String result) { + String binary = Assertions.assertDoesNotThrow( + () -> BinaryType.INSTANCE.fromValue(inputString, null)); + assertEquals(result, binary); + } + +}