From ee0fdee3a45a7ee3b0c2963ce10e1bc46c6f93ed Mon Sep 17 00:00:00 2001 From: penghuo Date: Mon, 27 Jul 2020 09:46:15 -0700 Subject: [PATCH 1/3] Add Text and Keyword Data Type --- .../data/model/AbstractExprNumberValue.java | 54 ++++++ .../sql/data/model/AbstractExprValue.java | 80 +++++++++ .../sql/data/model/ExprBooleanValue.java | 30 +++- .../sql/data/model/ExprCollectionValue.java | 50 +++++- .../sql/data/model/ExprDateValue.java | 30 +++- .../sql/data/model/ExprDoubleValue.java | 31 +++- .../sql/data/model/ExprFloatValue.java | 31 +++- .../sql/data/model/ExprIntegerValue.java | 31 +++- .../sql/data/model/ExprLongValue.java | 32 +++- .../sql/data/model/ExprMissingValue.java | 35 +++- .../sql/data/model/ExprNullValue.java | 40 ++++- .../sql/data/model/ExprStringValue.java | 30 +++- .../sql/data/model/ExprTimeValue.java | 29 ++- .../sql/data/model/ExprTimestampValue.java | 29 ++- .../sql/data/model/ExprTupleValue.java | 37 +++- .../sql/data/model/ExprValue.java | 100 ++++++++++- .../sql/data/model/ExprValueUtils.java | 54 +----- .../sql/data/type/ExprCoreType.java | 9 + .../data/utils/NaturalExprValueOrdering.java | 45 +---- .../sql/expression/Expression.java | 3 +- .../expression/datetime/DateTimeFunction.java | 84 +-------- .../sql/expression/function/FunctionDSL.java | 160 +++++++++++++++++ .../sql/expression/function/FunctionName.java | 3 +- .../function/SerializableBiFunction.java | 27 +++ .../function/SerializableFunction.java | 24 +++ .../expression/operator/OperatorUtils.java | 20 --- .../predicate/BinaryPredicateOperator.java | 168 ++---------------- .../sql/data/model/DateTimeValueTest.java | 29 ++- .../data/model/ExprCollectionValueTest.java | 30 ++++ .../sql/data/model/ExprTupleValueTest.java | 2 +- .../sql/data/model/ExprValueCompareTest.java | 83 +++++++++ .../sql/data/model/ExprValueUtilsTest.java | 79 +++++--- .../sql/data/utils/ExprValueOrderingTest.java | 39 ---- .../BinaryPredicateOperatorTest.java | 42 +++++ docs/experiment/ppl/cmd/stats.rst | 4 +- docs/user/general/datatype.rst | 74 ++++++++ docs/user/index.rst | 1 + .../data/type/ElasticsearchDataType.java | 63 +++++++ .../value/ElasticsearchExprTextValue.java | 37 ++++ .../value/ElasticsearchExprValueFactory.java | 3 + .../storage/ElasticsearchIndex.java | 11 +- .../data/type/ElasticsearchDataTypeTest.java | 44 +++++ .../value/ElasticsearchExprTextValueTest.java | 30 ++++ .../ElasticsearchExprValueFactoryTest.java | 8 + .../storage/ElasticsearchIndexTest.java | 19 +- .../sql/protocol/response/QueryResult.java | 2 +- 46 files changed, 1355 insertions(+), 511 deletions(-) create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/AbstractExprNumberValue.java create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/AbstractExprValue.java create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionDSL.java create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/SerializableBiFunction.java create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/SerializableFunction.java create mode 100644 core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueCompareTest.java create mode 100644 docs/user/general/datatype.rst create mode 100644 elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataType.java create mode 100644 elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprTextValue.java create mode 100644 elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataTypeTest.java create mode 100644 elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprTextValueTest.java diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/AbstractExprNumberValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/AbstractExprNumberValue.java new file mode 100644 index 0000000000..e5214aa8a5 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/AbstractExprNumberValue.java @@ -0,0 +1,54 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.google.common.base.Objects; +import lombok.RequiredArgsConstructor; + +/** + * Expression Number Value. + */ +@RequiredArgsConstructor +public abstract class AbstractExprNumberValue extends AbstractExprValue { + private final Number value; + + @Override + public Integer integerValue() { + return value.intValue(); + } + + @Override + public Long longValue() { + return value.longValue(); + } + + @Override + public Float floatValue() { + return value.floatValue(); + } + + @Override + public Double doubleValue() { + return value.doubleValue(); + } + + @Override + public int hashCode() { + return Objects.hashCode(value); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/AbstractExprValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/AbstractExprValue.java new file mode 100644 index 0000000000..1e6e51a336 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/AbstractExprValue.java @@ -0,0 +1,80 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; + +/** + * Abstract ExprValue. + */ +public abstract class AbstractExprValue implements ExprValue { + /** + * The customize compareTo logic. + */ + @Override + public int compareTo(ExprValue other) { + if (this.isNull() || this.isMissing()) { + return this.compare(other); + } else if (other.isNull() || other.isMissing()) { + return -other.compareTo(this); + } + if (!this.type().equals(other.type())) { + throw new ExpressionEvaluationException( + String.format( + "compare expected value have same type, but with [%s, %s]", + this.type(), other.type())); + } + return compare(other); + } + + /** + * The customize equals logic. + * The table below list the NULL and MISSING handling logic. + * A B A == B + * NULL NULL TRUE + * NULL MISSING FALSE + * MISSING NULL FALSE + * MISSING MISSING TRUE + */ + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } else if (!(o instanceof ExprValue)) { + return false; + } + ExprValue other = (ExprValue) o; + if (this.isNull() || this.isMissing()) { + return equal(other); + } else if (other.isNull() || other.isMissing()) { + return other.equals(this); + } else { + return equal(other); + } + } + + /** + * The expression value compare. + */ + public abstract int compare(ExprValue other); + + /** + * The expression value equal. + */ + public abstract boolean equal(ExprValue other); +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprBooleanValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprBooleanValue.java index eb14416f75..dd0e0805fc 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprBooleanValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprBooleanValue.java @@ -16,10 +16,14 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.google.common.base.Objects; import lombok.EqualsAndHashCode; -@EqualsAndHashCode -public class ExprBooleanValue implements ExprValue { +/** + * Expression Boolean Value. + */ +public class ExprBooleanValue extends AbstractExprValue { private static final ExprBooleanValue TRUE = new ExprBooleanValue(true); private static final ExprBooleanValue FALSE = new ExprBooleanValue(false); @@ -39,12 +43,32 @@ public Object value() { } @Override - public ExprCoreType type() { + public ExprType type() { return ExprCoreType.BOOLEAN; } + @Override + public Boolean booleanValue() { + return value; + } + @Override public String toString() { return value.toString(); } + + @Override + public int compare(ExprValue other) { + return Boolean.compare(value, other.booleanValue()); + } + + @Override + public boolean equal(ExprValue other) { + return value.equals(other.booleanValue()); + } + + @Override + public int hashCode() { + return Objects.hashCode(value); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprCollectionValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprCollectionValue.java index aa4053cef5..4d71e260e2 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprCollectionValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprCollectionValue.java @@ -16,14 +16,20 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.google.common.base.Objects; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; -@EqualsAndHashCode +/** + * Expression Collection Value. + */ @RequiredArgsConstructor -public class ExprCollectionValue implements ExprValue { +public class ExprCollectionValue extends AbstractExprValue { private final List valueList; @Override @@ -32,14 +38,52 @@ public Object value() { } @Override - public ExprCoreType type() { + public ExprType type() { return ExprCoreType.ARRAY; } + @Override + public List collectionValue() { + return valueList; + } + @Override public String toString() { return valueList.stream() .map(Object::toString) .collect(Collectors.joining(",", "[", "]")); } + + @Override + public boolean equal(ExprValue o) { + if (!(o instanceof ExprCollectionValue)) { + return false; + } else { + ExprCollectionValue other = (ExprCollectionValue) o; + Iterator thisIterator = this.valueList.iterator(); + Iterator otherIterator = other.valueList.iterator(); + + while (thisIterator.hasNext() && otherIterator.hasNext()) { + ExprValue thisEntry = thisIterator.next(); + ExprValue otherEntry = otherIterator.next(); + if (!thisEntry.equals(otherEntry)) { + return false; + } + } + return !(thisIterator.hasNext() || otherIterator.hasNext()); + } + } + + /** + * Only compare the size of the list. + */ + @Override + public int compare(ExprValue other) { + return Integer.compare(valueList.size(), other.collectionValue().size()); + } + + @Override + public int hashCode() { + return Objects.hashCode(valueList); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDateValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDateValue.java index dc06b2cfb3..52394dcd72 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDateValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDateValue.java @@ -18,22 +18,22 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import com.google.common.base.Objects; import java.time.Instant; import java.time.LocalDate; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; -import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; /** - * Date Value. + * Expression Date Value. */ -@EqualsAndHashCode @RequiredArgsConstructor -public class ExprDateValue implements ExprValue { +public class ExprDateValue extends AbstractExprValue { /** * todo. only support UTC now. */ @@ -59,16 +59,32 @@ public String value() { } @Override - public ExprCoreType type() { + public ExprType type() { return ExprCoreType.DATE; } + @Override + public ZonedDateTime dateValue() { + return date.atZone(ZONE); + } + @Override public String toString() { return String.format("DATE '%s'", value()); } - public ZonedDateTime getDate() { - return date.atZone(ZONE); + @Override + public int compare(ExprValue other) { + return date.compareTo(other.dateValue().toInstant()); + } + + @Override + public boolean equal(ExprValue other) { + return date.atZone(ZONE).equals(other.dateValue()); + } + + @Override + public int hashCode() { + return Objects.hashCode(date); } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDoubleValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDoubleValue.java index c464cbcfe3..a6e4347123 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDoubleValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDoubleValue.java @@ -16,26 +16,39 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; -import lombok.EqualsAndHashCode; -import lombok.RequiredArgsConstructor; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; -@EqualsAndHashCode -@RequiredArgsConstructor -public class ExprDoubleValue implements ExprValue { - private final Double value; +/** + * Expression Double Value. + */ +public class ExprDoubleValue extends AbstractExprNumberValue { + + public ExprDoubleValue(Number value) { + super(value); + } @Override public Object value() { - return value; + return doubleValue(); } @Override - public ExprCoreType type() { + public ExprType type() { return ExprCoreType.DOUBLE; } @Override public String toString() { - return value.toString(); + return doubleValue().toString(); + } + + @Override + public int compare(ExprValue other) { + return Double.compare(doubleValue(), other.doubleValue()); + } + + @Override + public boolean equal(ExprValue other) { + return doubleValue().equals(other.doubleValue()); } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprFloatValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprFloatValue.java index 37c44058f0..7d364fe6e0 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprFloatValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprFloatValue.java @@ -16,26 +16,39 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; -import lombok.EqualsAndHashCode; -import lombok.RequiredArgsConstructor; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; -@EqualsAndHashCode -@RequiredArgsConstructor -public class ExprFloatValue implements ExprValue { - private final Float value; +/** + * Expression Float Value. + */ +public class ExprFloatValue extends AbstractExprNumberValue { + + public ExprFloatValue(Number value) { + super(value); + } @Override public Object value() { - return value; + return floatValue(); } @Override - public ExprCoreType type() { + public ExprType type() { return ExprCoreType.FLOAT; } @Override public String toString() { - return value.toString(); + return floatValue().toString(); + } + + @Override + public int compare(ExprValue other) { + return Float.compare(floatValue(), other.floatValue()); + } + + @Override + public boolean equal(ExprValue other) { + return floatValue().equals(other.floatValue()); } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprIntegerValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprIntegerValue.java index 19d5462e78..e600196af2 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprIntegerValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprIntegerValue.java @@ -16,26 +16,39 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; -import lombok.EqualsAndHashCode; -import lombok.RequiredArgsConstructor; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; -@EqualsAndHashCode -@RequiredArgsConstructor -public class ExprIntegerValue implements ExprValue { - private final Integer value; +/** + * Expression Integer Value. + */ +public class ExprIntegerValue extends AbstractExprNumberValue { + + public ExprIntegerValue(Number value) { + super(value); + } @Override public Object value() { - return value; + return integerValue(); } @Override - public ExprCoreType type() { + public ExprType type() { return ExprCoreType.INTEGER; } @Override public String toString() { - return value.toString(); + return integerValue().toString(); + } + + @Override + public int compare(ExprValue other) { + return Integer.compare(integerValue(), other.integerValue()); + } + + @Override + public boolean equal(ExprValue other) { + return integerValue().equals(other.integerValue()); } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprLongValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprLongValue.java index 730a7c406e..0191376a10 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprLongValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprLongValue.java @@ -16,26 +16,40 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; -import lombok.EqualsAndHashCode; -import lombok.RequiredArgsConstructor; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.google.common.base.Objects; -@EqualsAndHashCode -@RequiredArgsConstructor -public class ExprLongValue implements ExprValue { - private final Long value; +/** + * Expression Long Value. + */ +public class ExprLongValue extends AbstractExprNumberValue { + + public ExprLongValue(Number value) { + super(value); + } @Override public Object value() { - return value; + return longValue(); } @Override - public ExprCoreType type() { + public ExprType type() { return ExprCoreType.LONG; } @Override public String toString() { - return value.toString(); + return longValue().toString(); + } + + @Override + public int compare(ExprValue other) { + return Long.compare(longValue(), other.longValue()); + } + + @Override + public boolean equal(ExprValue other) { + return longValue().equals(other.longValue()); } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprMissingValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprMissingValue.java index 3efe25f06c..a09611e479 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprMissingValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprMissingValue.java @@ -15,15 +15,15 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; -import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; -import lombok.EqualsAndHashCode; +import java.util.Objects; /** - * The definition of the expression missing value. + * Expression Missing Value. + * Missing value only equal to missing value, and is smaller than any other value. */ -@EqualsAndHashCode -public class ExprMissingValue implements ExprValue { +public class ExprMissingValue extends AbstractExprValue { private static final ExprValue instance = new ExprMissingValue(); private ExprMissingValue() { @@ -39,7 +39,7 @@ public Object value() { } @Override - public ExprCoreType type() { + public ExprType type() { throw new ExpressionEvaluationException("invalid to call type operation on missing value"); } @@ -47,4 +47,27 @@ public ExprCoreType type() { public boolean isMissing() { return true; } + + /** + * When MISSING value compare to other expression value. + * 1) NULL is equal to MISSING. + * 2) NULL is less than all other expression values. + */ + @Override + public int compare(ExprValue other) { + return other.isMissing() ? 0 : -1; + } + + /** + * Missing value is equal to Missing value. + */ + @Override + public boolean equal(ExprValue other) { + return other.isMissing(); + } + + @Override + public int hashCode() { + return Objects.hashCode("MISSING"); + } } \ No newline at end of file diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprNullValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprNullValue.java index b0b509c9e2..64d035e82f 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprNullValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprNullValue.java @@ -15,20 +15,28 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; -import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; -import lombok.EqualsAndHashCode; +import java.util.Objects; /** - * The definition of the expression null value. + * Expression Null Value. + * Null value + *
  • equal to null value. + *
  • large than missing value. + *
  • less than any other value. */ -@EqualsAndHashCode -public class ExprNullValue implements ExprValue { +public class ExprNullValue extends AbstractExprValue { private static final ExprValue instance = new ExprNullValue(); private ExprNullValue() { } + @Override + public int hashCode() { + return Objects.hashCode("NULL"); + } + public static ExprValue of() { return instance; } @@ -39,7 +47,7 @@ public Object value() { } @Override - public ExprCoreType type() { + public ExprType type() { throw new ExpressionEvaluationException("invalid to call type operation on null value"); } @@ -47,4 +55,24 @@ public ExprCoreType type() { public boolean isNull() { return true; } + + /** + * When NULL value compare to other expression value. + * 1) NULL is equal to NULL. + * 2) NULL is large than MISSING. + * 3) NULL is less than all other expression values. + */ + @Override + public int compare(ExprValue other) { + return other.isNull() ? 0 : other.isMissing() ? 1 : -1; + } + + /** + * NULL value is equal to NULL value. + */ + @Override + public boolean equal(ExprValue other) { + return other.isNull(); + } + } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java index a06af303b5..8ea3e0f60d 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java @@ -16,12 +16,16 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import java.util.Objects; import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; -@EqualsAndHashCode +/** + * Expression String Value. + */ @RequiredArgsConstructor -public class ExprStringValue implements ExprValue { +public class ExprStringValue extends AbstractExprValue { private final String value; @Override @@ -30,12 +34,32 @@ public Object value() { } @Override - public ExprCoreType type() { + public ExprType type() { return ExprCoreType.STRING; } + @Override + public String stringValue() { + return value; + } + @Override public String toString() { return String.format("\"%s\"", value); } + + @Override + public int compare(ExprValue other) { + return value.compareTo(other.stringValue()); + } + + @Override + public boolean equal(ExprValue other) { + return value.equals(other.stringValue()); + } + + @Override + public int hashCode() { + return Objects.hashCode(value); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimeValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimeValue.java index 82fd72a4f1..22917a80eb 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimeValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimeValue.java @@ -18,20 +18,21 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; import java.time.LocalTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; +import java.util.Objects; import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; /** - * Time Value. + * Expression Time Value. */ -@EqualsAndHashCode @RequiredArgsConstructor -public class ExprTimeValue implements ExprValue { +public class ExprTimeValue extends AbstractExprValue { /** * todo. only support UTC now. */ @@ -56,12 +57,32 @@ public String value() { } @Override - public ExprCoreType type() { + public ExprType type() { return ExprCoreType.TIME; } + @Override + public LocalTime timeValue() { + return time; + } + @Override public String toString() { return String.format("TIME '%s'", value()); } + + @Override + public int compare(ExprValue other) { + return time.compareTo(other.timeValue()); + } + + @Override + public boolean equal(ExprValue other) { + return time.equals(other.timeValue()); + } + + @Override + public int hashCode() { + return Objects.hashCode(time); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java index c6865bf981..2904457598 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java @@ -18,6 +18,7 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; import java.time.Instant; import java.time.LocalDateTime; @@ -25,15 +26,15 @@ import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.time.temporal.ChronoUnit; +import java.util.Objects; import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; /** - * Timestamp Value. + * Expression Timestamp Value. */ -@EqualsAndHashCode @RequiredArgsConstructor -public class ExprTimestampValue implements ExprValue { +public class ExprTimestampValue extends AbstractExprValue { /** * todo. only support UTC now. */ @@ -64,12 +65,32 @@ public String value() { } @Override - public ExprCoreType type() { + public ExprType type() { return ExprCoreType.TIMESTAMP; } + @Override + public Instant timestampValue() { + return timestamp; + } + @Override public String toString() { return String.format("TIMESTAMP '%s'", value()); } + + @Override + public int compare(ExprValue other) { + return timestamp.compareTo(other.timestampValue()); + } + + @Override + public boolean equal(ExprValue other) { + return timestamp.equals(other.timestampValue()); + } + + @Override + public int hashCode() { + return Objects.hashCode(timestamp); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValue.java index 92df972f66..41c702a255 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValue.java @@ -16,17 +16,22 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.LazyBindingTuple; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; +/** + * Expression Tuple Value. + */ @RequiredArgsConstructor -public class ExprTupleValue implements ExprValue { +public class ExprTupleValue extends AbstractExprValue { private final LinkedHashMap valueMap; @@ -41,7 +46,7 @@ public Object value() { } @Override - public ExprCoreType type() { + public ExprType type() { return ExprCoreType.STRUCT; } @@ -59,21 +64,27 @@ public BindingTuple bindingTuples() { bindingName -> valueMap.getOrDefault(bindingName, ExprMissingValue.of())); } + @Override + public Map tupleValue() { + return valueMap; + } + /** * Override the equals method. * @return true for equal, otherwise false. */ - public boolean equals(Object o) { - if (o == this) { - return true; - } else if (!(o instanceof ExprTupleValue)) { + public boolean equal(ExprValue o) { + if (!(o instanceof ExprTupleValue)) { return false; } else { ExprTupleValue other = (ExprTupleValue) o; Iterator> thisIterator = this.valueMap.entrySet().iterator(); Iterator> otherIterator = other.valueMap.entrySet().iterator(); while (thisIterator.hasNext() && otherIterator.hasNext()) { - if (!thisIterator.next().equals(otherIterator.next())) { + Entry thisEntry = thisIterator.next(); + Entry otherEntry = otherIterator.next(); + if (!(thisEntry.getKey().equals(otherEntry.getKey()) + && thisEntry.getValue().equals(otherEntry.getValue()))) { return false; } } @@ -81,4 +92,16 @@ public boolean equals(Object o) { } } + /** + * Only compare the size of the map. + */ + @Override + public int compare(ExprValue other) { + return Integer.compare(valueMap.size(), other.tupleValue().size()); + } + + @Override + public int hashCode() { + return Objects.hashCode(valueMap); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValue.java index 904b1aea09..fc005a3301 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValue.java @@ -16,12 +16,20 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; +import java.io.Serializable; +import java.time.Instant; +import java.time.LocalTime; +import java.time.ZonedDateTime; +import java.util.List; +import java.util.Map; /** * The definition of the Expression Value. */ -public interface ExprValue { +public interface ExprValue extends Serializable, Comparable { /** * Get the Object value of the Expression Value. */ @@ -30,7 +38,7 @@ public interface ExprValue { /** * Get the {@link ExprCoreType} of the Expression Value. */ - ExprCoreType type(); + ExprType type(); /** * Is null value. @@ -56,4 +64,92 @@ default boolean isMissing() { default BindingTuple bindingTuples() { return BindingTuple.EMPTY; } + + /** + * Get integer value. + */ + default Integer integerValue() { + throw new ExpressionEvaluationException( + "invalid to get integerValue from value of type " + type()); + } + + /** + * Get long value. + */ + default Long longValue() { + throw new ExpressionEvaluationException( + "invalid to get longValue from value of type " + type()); + } + + /** + * Get float value. + */ + default Float floatValue() { + throw new ExpressionEvaluationException( + "invalid to get floatValue from value of type " + type()); + } + + /** + * Get float value. + */ + default Double doubleValue() { + throw new ExpressionEvaluationException( + "invalid to get doubleValue from value of type " + type()); + } + + /** + * Get string value. + */ + default String stringValue() { + throw new ExpressionEvaluationException( + "invalid to get stringValue from value of type " + type()); + } + + /** + * Get boolean value. + */ + default Boolean booleanValue() { + throw new ExpressionEvaluationException( + "invalid to get booleanValue from value of type " + type()); + } + + /** + * Get timestamp value. + */ + default Instant timestampValue() { + throw new ExpressionEvaluationException( + "invalid to get timestampValue from value of type " + type()); + } + + /** + * Get time value. + */ + default LocalTime timeValue() { + throw new ExpressionEvaluationException( + "invalid to get timeValue from value of type " + type()); + } + + /** + * Get date value. + */ + default ZonedDateTime dateValue() { + throw new ExpressionEvaluationException( + "invalid to get dateValue from value of type " + type()); + } + + /** + * Get map value. + */ + default Map tupleValue() { + throw new ExpressionEvaluationException( + "invalid to get tupleValue from value of type " + type()); + } + + /** + * Get collection value. + */ + default List collectionValue() { + throw new ExpressionEvaluationException( + "invalid to get collectionValue from value of type " + type()); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java index c1c30a4164..c8f9e30d4f 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtils.java @@ -135,77 +135,41 @@ public static ExprValue fromObjectValue(Object o, ExprCoreType type) { } public static Integer getIntegerValue(ExprValue exprValue) { - return getNumberValue(exprValue).intValue(); + return exprValue.integerValue(); } public static Double getDoubleValue(ExprValue exprValue) { - return getNumberValue(exprValue).doubleValue(); + return exprValue.doubleValue(); } public static Long getLongValue(ExprValue exprValue) { - return getNumberValue(exprValue).longValue(); + return exprValue.longValue(); } public static Float getFloatValue(ExprValue exprValue) { - return getNumberValue(exprValue).floatValue(); + return exprValue.floatValue(); } public static String getStringValue(ExprValue exprValue) { - return convert(exprValue, STRING); + return exprValue.stringValue(); } public static List getCollectionValue(ExprValue exprValue) { - return convert(exprValue, ARRAY); + return exprValue.collectionValue(); } public static Map getTupleValue(ExprValue exprValue) { - return convert(exprValue, STRUCT); + return exprValue.tupleValue(); } public static Boolean getBooleanValue(ExprValue exprValue) { - return convert(exprValue, BOOLEAN); + return exprValue.booleanValue(); } /** * Get {@link ZonedDateTime} from ExprValue of Date type. */ public static ZonedDateTime getDateValue(ExprValue exprValue) { - if (ExprCoreType.DATE == exprValue.type()) { - return ((ExprDateValue) exprValue).getDate(); - } else { - throw new ExpressionEvaluationException( - String.format("invalid to convert expression with type:%s to type:%s", exprValue.type(), - ExprCoreType.DATE)); - } - } - - /** - * Get Number Value from {@link ExprValue}. - */ - @VisibleForTesting - public static Number getNumberValue(ExprValue exprValue) { - switch (exprValue.type()) { - case INTEGER: - case DOUBLE: - case LONG: - case FLOAT: - return (Number) exprValue.value(); - default: - break; - } - throw new ExpressionEvaluationException( - String - .format("invalid to getNumberValue with expression has type of %s", exprValue.type())); - } - - @SuppressWarnings("unchecked") - private static T convert(ExprValue exprValue, ExprCoreType toType) { - if (exprValue.type() == toType) { - return (T) exprValue.value(); - } else { - throw new ExpressionEvaluationException( - String.format("invalid to convert expression with type:%s to type:%s", exprValue.type(), - toType)); - } + return exprValue.dateValue(); } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java index 1e0d67149e..0b8312afb5 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java @@ -19,6 +19,7 @@ import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; /** * Expression Type. @@ -87,4 +88,12 @@ public List getParent() { public String typeName() { return this.name(); } + + /** + * Retrun all the valid ExprCoreType. + */ + public static List coreTypes() { + return Arrays.stream(ExprCoreType.values()).filter(type -> type != UNKNOWN) + .collect(Collectors.toList()); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NaturalExprValueOrdering.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NaturalExprValueOrdering.java index 0aeb2562d2..83534f5013 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NaturalExprValueOrdering.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/NaturalExprValueOrdering.java @@ -15,21 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.data.utils; -import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getBooleanValue; -import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getCollectionValue; -import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getDoubleValue; -import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getFloatValue; -import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getIntegerValue; -import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getLongValue; -import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getStringValue; -import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.getTupleValue; -import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.COMPARE_WITH_NULL_OR_MISSING; -import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.LIST_COMPARATOR; -import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.MAP_COMPARATOR; -import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.STRING_COMPARATOR; - import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; -import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; import com.google.common.collect.Ordering; /** @@ -45,36 +31,7 @@ public class NaturalExprValueOrdering extends ExprValueOrdering { @Override public int compare(ExprValue left, ExprValue right) { - if (COMPARE_WITH_NULL_OR_MISSING.test(left, right)) { - throw new ExpressionEvaluationException("compare with null or missing value is invalid"); - } - if (!left.type().equals(right.type())) { - throw new ExpressionEvaluationException( - String.format( - "compare expected value have same type, but with [%s, %s]", - left.type(), right.type())); - } - switch (left.type()) { - case DOUBLE: - return Double.compare(getDoubleValue(left), getDoubleValue(right)); - case FLOAT: - return Float.compare(getFloatValue(left), getFloatValue(right)); - case LONG: - return Long.compare(getLongValue(left), getLongValue(right)); - case INTEGER: - return Integer.compare(getIntegerValue(left), getIntegerValue(right)); - case BOOLEAN: - return Boolean.compare(getBooleanValue(left), getBooleanValue(right)); - case STRING: - return STRING_COMPARATOR.apply(getStringValue(left), getStringValue(right)); - case STRUCT: - return MAP_COMPARATOR.apply(getTupleValue(left), getTupleValue(right)); - case ARRAY: - return LIST_COMPARATOR.apply(getCollectionValue(left), getCollectionValue(right)); - default: - throw new ExpressionEvaluationException( - String.format("compare doesn't support type [%s]", left.type())); - } + return left.compareTo(right); } @Override diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/Expression.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/Expression.java index 8f233cf224..c4348ddfcf 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/Expression.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/Expression.java @@ -18,11 +18,12 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import java.io.Serializable; /** * The definition of the resolved expression. */ -public interface Expression { +public interface Expression extends Serializable { /** * Evaluate the value of expression in the value environment. diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java index 1257a70f1c..785b187041 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java @@ -24,25 +24,13 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprIntegerValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; -import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; -import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; -import com.amazon.opendistroforelasticsearch.sql.expression.Expression; -import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; -import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; -import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder; -import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionDSL; import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver; -import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature; -import java.util.Collections; -import java.util.function.Function; -import java.util.stream.Collectors; import lombok.experimental.UtilityClass; -import org.apache.commons.lang3.tuple.Pair; /** * The definition of date and time functions. - * todo, keep define and unaryImpl internally for now. there are two purpose for doing this * 1) have the clear interface for function define. * 2) the implementation should rely on ExprValue. */ @@ -56,77 +44,11 @@ public void register(BuiltinFunctionRepository repository) { * DAYOFMONTH(DATE). return the day of the month (1-31). */ private FunctionResolver dayOfMonth() { - return define(DAYOFMONTH.getName(), - unaryImpl(DateTimeFunction::exprDayOfMonth, INTEGER, DATE) + return FunctionDSL.define(DAYOFMONTH.getName(), + FunctionDSL.unaryImpl(DateTimeFunction::exprDayOfMonth, INTEGER, DATE) ); } - /** - * Define overloaded function with implementation. - * @param functionName function name. - * @param functions a list of function implementation. - * @return FunctionResolver. - */ - private FunctionResolver define(FunctionName functionName, - Function>... functions) { - - FunctionResolver.FunctionResolverBuilder builder = FunctionResolver.builder(); - builder.functionName(functionName); - for (Function> func : functions) { - Pair functionBuilder = func.apply(functionName); - builder.functionBundle(functionBuilder.getKey(), functionBuilder.getValue()); - } - return builder.build(); - } - - /** - * Unary Function Implementation. - * @param function {@link ExprValue} based unary function. - * @param returnType return type. - * @param argsType argument type. - * - * @return Unary Function Implementation. - */ - private Function> unaryImpl( - Function function, - ExprType returnType, - ExprType argsType) { - - return functionName -> { - FunctionSignature functionSignature = - new FunctionSignature(functionName, Collections.singletonList(argsType)); - FunctionBuilder functionBuilder = - arguments -> new FunctionExpression(functionName, arguments) { - @Override - public ExprValue valueOf(Environment valueEnv) { - ExprValue value = arguments.get(0).valueOf(valueEnv); - if (value.isMissing()) { - return ExprValueUtils.missingValue(); - } else if (value.isNull()) { - return ExprValueUtils.nullValue(); - } else { - return function.apply(value); - } - } - - @Override - public ExprType type() { - return returnType; - } - - @Override - public String toString() { - return String.format("%s(%s)", functionName, - arguments.stream() - .map(Object::toString) - .collect(Collectors.joining(", "))); - } - }; - return Pair.of(functionSignature, functionBuilder); - }; - } - /** * Day of Month implementation for ExprValue. * @param date ExprValue of Date type. diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionDSL.java new file mode 100644 index 0000000000..827707eea0 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionDSL.java @@ -0,0 +1,160 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import lombok.experimental.UtilityClass; +import org.apache.commons.lang3.tuple.Pair; + +/** + * Function Define Utility. + */ +@UtilityClass +public class FunctionDSL { + /** + * Define overloaded function with implementation. + * + * @param functionName function name. + * @param functions a list of function implementation. + * @return FunctionResolver. + */ + public FunctionResolver define(FunctionName functionName, + Function>... functions) { + return define(functionName, Arrays.asList(functions)); + } + + /** + * Define overloaded function with implementation. + * + * @param functionName function name. + * @param functions a list of function implementation. + * @return FunctionResolver. + */ + public FunctionResolver define(FunctionName functionName, + List>> functions) { + + FunctionResolver.FunctionResolverBuilder builder = FunctionResolver.builder(); + builder.functionName(functionName); + for (Function> func : functions) { + Pair functionBuilder = func.apply(functionName); + builder.functionBundle(functionBuilder.getKey(), functionBuilder.getValue()); + } + return builder.build(); + } + + /** + * Unary Function Implementation. + * + * @param function {@link ExprValue} based unary function. + * @param returnType return type. + * @param argsType argument type. + * @return Unary Function Implementation. + */ + public SerializableFunction> unaryImpl( + SerializableFunction function, + ExprType returnType, + ExprType argsType) { + + return functionName -> { + FunctionSignature functionSignature = + new FunctionSignature(functionName, Collections.singletonList(argsType)); + FunctionBuilder functionBuilder = + arguments -> new FunctionExpression(functionName, arguments) { + @Override + public ExprValue valueOf(Environment valueEnv) { + ExprValue value = arguments.get(0).valueOf(valueEnv); + if (value.isMissing()) { + return ExprValueUtils.missingValue(); + } else if (value.isNull()) { + return ExprValueUtils.nullValue(); + } else { + return function.apply(value); + } + } + + @Override + public ExprType type() { + return returnType; + } + + @Override + public String toString() { + return String.format("%s(%s)", functionName, + arguments.stream() + .map(Object::toString) + .collect(Collectors.joining(", "))); + } + }; + return Pair.of(functionSignature, functionBuilder); + }; + } + + /** + * Binary Function Implementation. + * + * @param function {@link ExprValue} based unary function. + * @param returnType return type. + * @param args1Type argument type. + * @param args2Type argument type. + * @return Unary Function Implementation. + */ + public Function> binaryImpl( + SerializableBiFunction function, + ExprType returnType, + ExprType args1Type, + ExprType args2Type) { + + return functionName -> { + FunctionSignature functionSignature = + new FunctionSignature(functionName, Arrays.asList(args1Type, args2Type)); + FunctionBuilder functionBuilder = + arguments -> new FunctionExpression(functionName, arguments) { + @Override + public ExprValue valueOf(Environment valueEnv) { + ExprValue arg1 = arguments.get(0).valueOf(valueEnv); + ExprValue arg2 = arguments.get(1).valueOf(valueEnv); + return function.apply(arg1, arg2); + } + + @Override + public ExprType type() { + return returnType; + } + + @Override + public String toString() { + return String.format("%s %s %s", arguments.get(0).toString(), functionName, arguments + .get(1).toString()); + } + }; + return Pair.of(functionSignature, functionBuilder); + }; + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionName.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionName.java index 25c865dbb7..d0fe205413 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionName.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionName.java @@ -15,6 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.expression.function; +import java.io.Serializable; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -24,7 +25,7 @@ */ @EqualsAndHashCode @RequiredArgsConstructor -public class FunctionName { +public class FunctionName implements Serializable { @Getter private final String functionName; diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/SerializableBiFunction.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/SerializableBiFunction.java new file mode 100644 index 0000000000..7fb1b4bd9e --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/SerializableBiFunction.java @@ -0,0 +1,27 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import java.io.Serializable; +import java.util.function.BiFunction; + +/** + * Serializable BiFunction. + */ +public interface SerializableBiFunction extends BiFunction, Serializable { +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/SerializableFunction.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/SerializableFunction.java new file mode 100644 index 0000000000..547c22ae6e --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/SerializableFunction.java @@ -0,0 +1,24 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.expression.function; + +import java.io.Serializable; +import java.util.function.Function; + +public interface SerializableFunction extends Function, Serializable { +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/OperatorUtils.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/OperatorUtils.java index a60a4fc48e..39d4db9c09 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/OperatorUtils.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/OperatorUtils.java @@ -288,26 +288,6 @@ public String toString() { }; } - /** - * String comparator. - */ - public static final BiFunction STRING_COMPARATOR = String::compareTo; - /** - * List comparator. - */ - public static final BiFunction LIST_COMPARATOR = - (left, right) -> Integer.compare(left.size(), right.size()); - /** - * Map comparator. - */ - public static final BiFunction MAP_COMPARATOR = - (left, right) -> Integer.compare(left.size(), right.size()); - /** - * Predicate NULL or MISSING. - */ - public static final BiPredicate COMPARE_WITH_NULL_OR_MISSING = - (left, right) -> left.isMissing() || right.isMissing() || left.isNull() || right.isNull(); - public interface TriFunction { R apply(T t, U u, V v); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java index d843fdc200..0d6f493905 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java @@ -19,17 +19,16 @@ import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_NULL; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.LITERAL_TRUE; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.ARRAY; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.FLOAT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.LONG; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; -import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; import static com.amazon.opendistroforelasticsearch.sql.expression.operator.OperatorUtils.binaryOperator; import static com.amazon.opendistroforelasticsearch.sql.utils.OperatorUtils.matches; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; @@ -40,6 +39,7 @@ import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName; import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository; import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder; +import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionDSL; import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName; import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver; import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature; @@ -47,10 +47,9 @@ import com.google.common.collect.ImmutableTable; import com.google.common.collect.Table; import java.util.Arrays; -import java.util.List; import java.util.Map; import java.util.function.BiFunction; -import java.util.function.Function; +import java.util.stream.Collectors; import lombok.experimental.UtilityClass; /** @@ -164,38 +163,6 @@ public static void register(BuiltinFunctionRepository repository) { .put(LITERAL_MISSING, LITERAL_MISSING, LITERAL_MISSING) .build(); - /** - * The equalTo logic. - * A B A == B - * NULL NULL TRUE - * NULL MISSING FALSE - * MISSING NULL FALSE - * MISSING MISSING TRUE - */ - private static Table equalTable = - new ImmutableTable.Builder() - .put(LITERAL_NULL, LITERAL_NULL, LITERAL_TRUE) - .put(LITERAL_NULL, LITERAL_MISSING, LITERAL_FALSE) - .put(LITERAL_MISSING, LITERAL_NULL, LITERAL_FALSE) - .put(LITERAL_MISSING, LITERAL_MISSING, LITERAL_TRUE) - .build(); - - /** - * The notEqualTo logic. - * A B A != B - * NULL NULL FALSE - * NULL MISSING TRUE - * MISSING NULL TRUE - * MISSING MISSING FALSE - */ - private static Table notEqualTable = - new ImmutableTable.Builder() - .put(LITERAL_NULL, LITERAL_NULL, LITERAL_FALSE) - .put(LITERAL_NULL, LITERAL_MISSING, LITERAL_TRUE) - .put(LITERAL_MISSING, LITERAL_NULL, LITERAL_TRUE) - .put(LITERAL_MISSING, LITERAL_MISSING, LITERAL_FALSE) - .build(); - private static FunctionResolver and() { FunctionName functionName = BuiltinFunctionName.AND.getName(); return FunctionResolver.builder() @@ -227,41 +194,21 @@ private static FunctionResolver xor() { } private static FunctionResolver equal() { - return new FunctionResolver( - BuiltinFunctionName.EQUAL.getName(), - predicate( - BuiltinFunctionName.EQUAL.getName(), - equalTable, - LITERAL_FALSE, - Integer::equals, - Long::equals, - Float::equals, - Double::equals, - String::equals, - Boolean::equals, - List::equals, - Map::equals - ) - ); + return FunctionDSL.define(BuiltinFunctionName.EQUAL.getName(), + ExprCoreType.coreTypes().stream() + .map(type -> FunctionDSL.binaryImpl((v1, v2) -> ExprBooleanValue.of(v1.equals(v2)), + BOOLEAN, type, type)) + .collect( + Collectors.toList())); } private static FunctionResolver notEqual() { - return new FunctionResolver( - BuiltinFunctionName.NOTEQUAL.getName(), - predicate( - BuiltinFunctionName.NOTEQUAL.getName(), - notEqualTable, - LITERAL_TRUE, - (v1, v2) -> ! v1.equals(v2), - (v1, v2) -> ! v1.equals(v2), - (v1, v2) -> ! v1.equals(v2), - (v1, v2) -> ! v1.equals(v2), - (v1, v2) -> ! v1.equals(v2), - (v1, v2) -> ! v1.equals(v2), - (v1, v2) -> ! v1.equals(v2), - (v1, v2) -> ! v1.equals(v2) - ) - ); + return FunctionDSL + .define(BuiltinFunctionName.NOTEQUAL.getName(), ExprCoreType.coreTypes().stream() + .map(type -> FunctionDSL + .binaryImpl((v1, v2) -> ExprBooleanValue.of(!v1.equals(v2)), BOOLEAN, type, type)) + .collect( + Collectors.toList())); } private static FunctionResolver less() { @@ -330,52 +277,6 @@ private static FunctionResolver like() { ); } - /** - * Util method to generate EQUAL/NOT EQUAL operation bundles. - * Applicable for integer, long, float, double, string types of operands - * {@param defaultValue} Default value for one missing/null operand - */ - private static Map predicate( - FunctionName functionName, - Table table, - ExprValue defaultValue, - BiFunction integerFunc, - BiFunction longFunc, - BiFunction floatFunc, - BiFunction doubleFunc, - BiFunction stringFunc, - BiFunction booleanFunc, - BiFunction listFunc, - BiFunction mapFunc) { - ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); - return builder - .put(new FunctionSignature(functionName, Arrays.asList(INTEGER, INTEGER)), - equalPredicate(functionName, table, integerFunc, ExprValueUtils::getIntegerValue, - defaultValue, BOOLEAN)) - .put(new FunctionSignature(functionName, Arrays.asList(LONG, LONG)), - equalPredicate(functionName, table, longFunc, ExprValueUtils::getLongValue, - defaultValue, BOOLEAN)) - .put(new FunctionSignature(functionName, Arrays.asList(FLOAT, FLOAT)), - equalPredicate(functionName, table, floatFunc, ExprValueUtils::getFloatValue, - defaultValue, BOOLEAN)) - .put(new FunctionSignature(functionName, Arrays.asList(DOUBLE, DOUBLE)), - equalPredicate(functionName, table, doubleFunc, ExprValueUtils::getDoubleValue, - defaultValue, BOOLEAN)) - .put(new FunctionSignature(functionName, Arrays.asList(STRING, STRING)), - equalPredicate(functionName, table, stringFunc, ExprValueUtils::getStringValue, - defaultValue, BOOLEAN)) - .put(new FunctionSignature(functionName, Arrays.asList(BOOLEAN, BOOLEAN)), - equalPredicate(functionName, table, booleanFunc, ExprValueUtils::getBooleanValue, - defaultValue, BOOLEAN)) - .put(new FunctionSignature(functionName, Arrays.asList(ARRAY, ARRAY)), - equalPredicate(functionName, table, listFunc, ExprValueUtils::getCollectionValue, - defaultValue, BOOLEAN)) - .put(new FunctionSignature(functionName, Arrays.asList(STRUCT, STRUCT)), - equalPredicate(functionName, table, mapFunc, ExprValueUtils::getTupleValue, - defaultValue, BOOLEAN)) - .build(); - } - /** * Util method to generate binary predicate bundles. * Applicable for integer, long, float, double, string types of operands @@ -461,43 +362,4 @@ public String toString() { } }; } - - /** - * Building method for equalTo and notEqualTo operators. - * - * @param defaultValue the return value when expr value is missing/null - */ - private static FunctionBuilder equalPredicate(FunctionName functionName, - Table table, - BiFunction function, - Function observer, - ExprValue defaultValue, - ExprType returnType) { - return arguments -> new FunctionExpression(functionName, arguments) { - @Override - public ExprValue valueOf(Environment env) { - ExprValue arg1 = arguments.get(0).valueOf(env); - ExprValue arg2 = arguments.get(1).valueOf(env); - if (table.contains(arg1, arg2)) { - return table.get(arg1, arg2); - } else if (arg1.isMissing() || arg1.isNull() || arg2.isMissing() || arg2.isNull()) { - return defaultValue; - } else { - return ExprValueUtils.fromObjectValue( - function.apply(observer.apply(arg1), observer.apply(arg2))); - } - } - - @Override - public ExprType type() { - return returnType; - } - - @Override - public String toString() { - return String.format("%s %s %s", arguments.get(0).toString(), functionName, arguments - .get(1).toString()); - } - }; - } } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java index 08599e9b7a..be4ad7f7fc 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/DateTimeValueTest.java @@ -17,34 +17,55 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIME; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; import org.junit.jupiter.api.Test; public class DateTimeValueTest { @Test - public void timestampValueInterfaceTest() { - ExprTimeValue timeValue = new ExprTimeValue("01:01:01"); + public void timeValueInterfaceTest() { + ExprValue timeValue = new ExprTimeValue("01:01:01"); assertEquals(TIME, timeValue.type()); + assertEquals(LocalTime.parse("01:01:01"), timeValue.timeValue()); assertEquals("01:01:01", timeValue.value()); assertEquals("TIME '01:01:01'", timeValue.toString()); } @Test - public void timeValueInterfaceTest() { - ExprTimestampValue timestampValue = new ExprTimestampValue("2020-07-07 01:01:01"); + public void timestampValueInterfaceTest() { + ExprValue timestampValue = new ExprTimestampValue("2020-07-07 01:01:01"); assertEquals(TIMESTAMP, timestampValue.type()); + assertEquals(Instant.ofEpochSecond(1594083661), timestampValue.timestampValue()); assertEquals("2020-07-07 01:01:01", timestampValue.value()); assertEquals("TIMESTAMP '2020-07-07 01:01:01'", timestampValue.toString()); } + @Test + public void dateValueInterfaceTest() { + ExprValue dateValue = new ExprDateValue("2012-07-07"); + + assertEquals(LocalDate.parse("2012-07-07").atStartOfDay(ZoneId.of("UTC")), + dateValue.dateValue()); + ExpressionEvaluationException exception = + assertThrows(ExpressionEvaluationException.class, + () -> ExprValueUtils.getDateValue(integerValue(1))); + assertEquals("invalid to get dateValue from value of type INTEGER", + exception.getMessage()); + } + @Test public void dateInUnsupportedFormat() { SemanticCheckException exception = diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprCollectionValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprCollectionValueTest.java index 64e932a0fe..83085ac1d5 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprCollectionValueTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprCollectionValueTest.java @@ -17,13 +17,43 @@ import static com.amazon.opendistroforelasticsearch.sql.utils.ComparisonUtil.compare; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.util.Arrays; import org.junit.jupiter.api.Test; public class ExprCollectionValueTest { + @Test + public void equal_to_itself() { + ExprValue value = ExprValueUtils.collectionValue(ImmutableList.of(1)); + assertTrue(value.equals(value)); + } + + @Test + public void collection_compare_int() { + ExprValue intValue = ExprValueUtils.integerValue(10); + ExprValue value = ExprValueUtils.collectionValue(ImmutableList.of(1)); + assertFalse(value.equals(intValue)); + } + + @Test + public void compare_collection_with_different_size() { + ExprValue value1 = ExprValueUtils.collectionValue(ImmutableList.of(1)); + ExprValue value2 = ExprValueUtils.collectionValue(ImmutableList.of(1, 2)); + assertFalse(value1.equals(value2)); + assertFalse(value2.equals(value1)); + } + + @Test + public void compare_collection_with_int_object() { + ExprValue value = ExprValueUtils.collectionValue(ImmutableList.of(1)); + assertFalse(value.equals(1)); + } @Test public void comparabilityTest() { diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValueTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValueTest.java index fcfb072b90..cd79cc13d5 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValueTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTupleValueTest.java @@ -40,7 +40,7 @@ public void tuple_compare_int() { } @Test - public void compre_tuple_with_different_size() { + public void compare_tuple_with_different_size() { ExprValue tupleValue1 = ExprValueUtils.tupleValue(ImmutableMap.of("integer_value", 2)); ExprValue tupleValue2 = ExprValueUtils.tupleValue(ImmutableMap.of("integer_value", 2, "float_value", 1f)); diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueCompareTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueCompareTest.java new file mode 100644 index 0000000000..e5d6d877b3 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueCompareTest.java @@ -0,0 +1,83 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.data.model; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +public class ExprValueCompareTest { + + @Test + public void timeValueCompare() { + assertEquals(0, new ExprTimeValue("18:00:00").compareTo(new ExprTimeValue("18:00:00"))); + assertEquals(1, new ExprTimeValue("19:00:00").compareTo(new ExprTimeValue("18:00:00"))); + assertEquals(-1, new ExprTimeValue("18:00:00").compareTo(new ExprTimeValue("19:00:00"))); + } + + @Test + public void dateValueCompare() { + assertEquals(0, new ExprDateValue("2012-08-07").compareTo(new ExprDateValue("2012-08-07"))); + assertEquals(1, new ExprDateValue("2012-08-08").compareTo(new ExprDateValue("2012-08-07"))); + assertEquals(-1, new ExprDateValue("2012-08-07").compareTo(new ExprDateValue("2012-08-08"))); + } + + @Test + public void timestampValueCompare() { + assertEquals(0, + new ExprTimestampValue("2012-08-07 18:00:00") + .compareTo(new ExprTimestampValue("2012-08-07 18:00:00"))); + assertEquals(1, + new ExprTimestampValue("2012-08-07 19:00:00") + .compareTo(new ExprTimestampValue("2012-08-07 18:00:00"))); + assertEquals(-1, + new ExprTimestampValue("2012-08-07 18:00:00") + .compareTo(new ExprTimestampValue("2012-08-07 19:00:00"))); + } + + @Test + public void nullValueEqualToNullValue() { + assertEquals(0, ExprNullValue.of().compareTo(ExprNullValue.of())); + } + + @Test + public void nullValueLessThanNotNullValue() { + assertEquals(-1, ExprNullValue.of().compareTo(ExprBooleanValue.of(true))); + assertEquals(1, ExprBooleanValue.of(true).compareTo(ExprNullValue.of())); + } + + @Test + public void missingValueEqualToMissingValue() { + assertEquals(0, ExprMissingValue.of().compareTo(ExprMissingValue.of())); + } + + @Test + public void missingValueLessThanNotMissingValue() { + assertEquals(-1, ExprMissingValue.of().compareTo(ExprBooleanValue.of(true))); + assertEquals(1, ExprBooleanValue.of(true).compareTo(ExprMissingValue.of())); + + assertEquals(-1, ExprMissingValue.of().compareTo(ExprNullValue.of())); + assertEquals(1, ExprNullValue.of().compareTo(ExprMissingValue.of())); + } + + @Test + public void missingValueLessThanNullValue() { + assertEquals(-1, ExprMissingValue.of().compareTo(ExprNullValue.of())); + assertEquals(1, ExprNullValue.of().compareTo(ExprMissingValue.of())); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java index f9bd0ff36d..cc9334602e 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprValueUtilsTest.java @@ -16,26 +16,36 @@ package com.amazon.opendistroforelasticsearch.sql.data.model; import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DATE; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIME; +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException; import com.amazon.opendistroforelasticsearch.sql.storage.bindingtuple.BindingTuple; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import java.time.Instant; import java.time.LocalDate; +import java.time.LocalTime; import java.time.ZoneId; import java.util.AbstractMap; import java.util.ArrayList; import java.util.Arrays; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.hamcrest.Matchers; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -44,10 +54,25 @@ @DisplayName("Test Expression Value Utils") public class ExprValueUtilsTest { + private static LinkedHashMap testTuple = new LinkedHashMap<>(); + + static { + testTuple.put("1", new ExprIntegerValue(1)); + } + private static List numberValues = Stream.of(1, 1L, 1f, 1D) .map(ExprValueUtils::fromObjectValue).collect(Collectors.toList()); - private static List nonNumberValues = Stream.of("1", true, Arrays.asList(1), - ImmutableMap.of("1", 1)).map(ExprValueUtils::fromObjectValue).collect(Collectors.toList()); + + private static List nonNumberValues = Arrays.asList( + new ExprStringValue("1"), + ExprBooleanValue.of(true), + new ExprCollectionValue(ImmutableList.of(new ExprIntegerValue(1))), + new ExprTupleValue(testTuple), + new ExprDateValue("2012-08-07"), + new ExprTimeValue("18:00:00"), + new ExprTimestampValue("2012-08-07 18:00:00") + ); + private static List allValues = Lists.newArrayList(Iterables.concat(numberValues, nonNumberValues)); @@ -60,7 +85,10 @@ public class ExprValueUtilsTest { ExprValueUtils::getStringValue, ExprValueUtils::getBooleanValue, ExprValueUtils::getCollectionValue, - ExprValueUtils::getTupleValue); + ExprValueUtils::getTupleValue, + ExprValue::dateValue, + ExprValue::timeValue, + ExprValue::timestampValue); private static List> allValueExtractor = Lists.newArrayList( Iterables.concat(numberValueExtractor, nonNumberValueExtractor)); @@ -69,14 +97,18 @@ public class ExprValueUtilsTest { ExprCoreType.DOUBLE); private static List nonNumberTypes = Arrays.asList(ExprCoreType.STRING, ExprCoreType.BOOLEAN, ExprCoreType.ARRAY, - ExprCoreType.STRUCT); + ExprCoreType.STRUCT, DATE, TIME, TIMESTAMP); private static List allTypes = Lists.newArrayList(Iterables.concat(numberTypes, nonNumberTypes)); private static Stream getValueTestArgumentStream() { List expectedValues = Arrays.asList(1, 1L, 1f, 1D, "1", true, Arrays.asList(integerValue(1)), - ImmutableMap.of("1", integerValue(1))); + ImmutableMap.of("1", integerValue(1)), + LocalDate.parse("2012-08-07").atStartOfDay(ZoneId.of("UTC")), + LocalTime.parse("18:00:00"), + Instant.ofEpochSecond(1344362400) + ); Stream.Builder builder = Stream.builder(); for (int i = 0; i < expectedValues.size(); i++) { builder.add(Arguments.of( @@ -150,9 +182,7 @@ public void getType(ExprValue value, ExprCoreType expectType) { public void invalidGetNumberValue(ExprValue value, Function extractor) { Exception exception = assertThrows(ExpressionEvaluationException.class, () -> extractor.apply(value)); - assertEquals( - String.format("invalid to getNumberValue with expression has type of %s", value.type()), - exception.getMessage()); + assertThat(exception.getMessage(), Matchers.containsString("invalid")); } /** @@ -164,9 +194,7 @@ public void invalidConvertExprValue(ExprValue value, Function ExprCoreType toType) { Exception exception = assertThrows(ExpressionEvaluationException.class, () -> extractor.apply(value)); - assertEquals(String - .format("invalid to convert expression with type:%s to type:%s", value.type(), toType), - exception.getMessage()); + assertThat(exception.getMessage(), Matchers.containsString("invalid")); } @Test @@ -193,23 +221,26 @@ public void bindingTuples() { @Test public void constructDateAndTimeValue() { assertEquals(new ExprDateValue("2012-07-07"), - ExprValueUtils.fromObjectValue("2012-07-07", ExprCoreType.DATE)); + ExprValueUtils.fromObjectValue("2012-07-07", DATE)); assertEquals(new ExprTimeValue("01:01:01"), - ExprValueUtils.fromObjectValue("01:01:01", ExprCoreType.TIME)); + ExprValueUtils.fromObjectValue("01:01:01", TIME)); assertEquals(new ExprTimestampValue("2012-07-07 01:01:01"), - ExprValueUtils.fromObjectValue("2012-07-07 01:01:01", ExprCoreType.TIMESTAMP)); + ExprValueUtils.fromObjectValue("2012-07-07 01:01:01", TIMESTAMP)); } @Test - public void getDateFromValue() { - assertEquals(LocalDate.parse("2012-07-07").atStartOfDay(ZoneId.of("UTC")), - ExprValueUtils.getDateValue(new ExprDateValue( - "2012-07-07"))); - - ExpressionEvaluationException exception = - assertThrows(ExpressionEvaluationException.class, - () -> ExprValueUtils.getDateValue(integerValue(1))); - assertEquals("invalid to convert expression with type:INTEGER to type:DATE", - exception.getMessage()); + public void hashCodeTest() { + assertEquals(new ExprIntegerValue(1).hashCode(), new ExprIntegerValue(1).hashCode()); + assertEquals(new ExprStringValue("1").hashCode(), new ExprStringValue("1").hashCode()); + assertEquals(new ExprCollectionValue(ImmutableList.of(new ExprIntegerValue(1))).hashCode(), + new ExprCollectionValue(ImmutableList.of(new ExprIntegerValue(1))).hashCode()); + assertEquals(new ExprTupleValue(testTuple).hashCode(), + new ExprTupleValue(testTuple).hashCode()); + assertEquals(new ExprDateValue("2012-08-07").hashCode(), + new ExprDateValue("2012-08-07").hashCode()); + assertEquals(new ExprTimeValue("18:00:00").hashCode(), + new ExprTimeValue("18:00:00").hashCode()); + assertEquals(new ExprTimestampValue("2012-08-07 18:00:00").hashCode(), + new ExprTimestampValue("2012-08-07 18:00:00").hashCode()); } } \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprValueOrderingTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprValueOrderingTest.java index d2ac42afc4..755501f86c 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprValueOrderingTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprValueOrderingTest.java @@ -205,43 +205,4 @@ public void order_compare_value_with_different_type() { "compare expected value have same type, but with [INTEGER, DOUBLE]", exception.getMessage()); } - - @Test - public void order_compare_value_with_null_value() { - ExprValueOrdering ordering = ExprValueOrdering.natural(); - ExpressionEvaluationException exception = - assertThrows( - ExpressionEvaluationException.class, - () -> ordering.compare(integerValue(1), LITERAL_NULL)); - assertEquals("compare with null or missing value is invalid", exception.getMessage()); - - exception = - assertThrows( - ExpressionEvaluationException.class, - () -> ordering.compare(integerValue(1), LITERAL_MISSING)); - assertEquals("compare with null or missing value is invalid", exception.getMessage()); - - exception = - assertThrows( - ExpressionEvaluationException.class, - () -> ordering.compare(LITERAL_NULL, integerValue(1))); - assertEquals("compare with null or missing value is invalid", exception.getMessage()); - - exception = - assertThrows( - ExpressionEvaluationException.class, - () -> ordering.compare(LITERAL_MISSING, integerValue(1))); - assertEquals("compare with null or missing value is invalid", exception.getMessage()); - } - - @Test - public void order_compare_unknown_type() { - when(left.type()).thenReturn(ExprCoreType.UNKNOWN); - when(right.type()).thenReturn(ExprCoreType.UNKNOWN); - - ExprValueOrdering ordering = ExprValueOrdering.natural(); - ExpressionEvaluationException exception = - assertThrows(ExpressionEvaluationException.class, () -> ordering.compare(left, right)); - assertEquals("compare doesn't support type [UNKNOWN]", exception.getMessage()); - } } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java index 8da8967f30..5194d70278 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java @@ -30,19 +30,27 @@ import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.BOOLEAN; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; + import static com.amazon.opendistroforelasticsearch.sql.utils.ComparisonUtil.compare; import static com.amazon.opendistroforelasticsearch.sql.utils.OperatorUtils.matches; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionTestBase; import com.amazon.opendistroforelasticsearch.sql.expression.FunctionExpression; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.util.Arrays; +import java.util.Base64; import java.util.List; import java.util.stream.Stream; import org.junit.jupiter.api.Test; @@ -719,4 +727,38 @@ public void test_null_like_missing() { assertEquals(BOOLEAN, like.type()); assertEquals(LITERAL_MISSING, like.valueOf(valueEnv())); } + + /** + * Todo. remove this test cases after script serilization implemented. + */ + @Test + public void serializationTest() throws Exception { + Expression expression = dsl.equal(DSL.literal("v1"), DSL.literal("v2")); + // serialization + ByteArrayOutputStream output = new ByteArrayOutputStream(); + ObjectOutputStream objectOutput = new ObjectOutputStream(output); + objectOutput.writeObject(expression); + objectOutput.flush(); + String source = Base64.getEncoder().encodeToString(output.toByteArray()); + + // deserialization + ByteArrayInputStream input = new ByteArrayInputStream(Base64.getDecoder().decode(source)); + ObjectInputStream objectInput = new ObjectInputStream(input); + Expression e = (Expression) objectInput.readObject(); + ExprValue exprValue = e.valueOf(valueEnv()); + + assertEquals(LITERAL_FALSE, exprValue); + } + + @Test + public void compareNumberValueWithDifferentType() { + FunctionExpression equal = dsl.equal(DSL.literal(1), DSL.literal(1L)); + assertTrue(equal.valueOf(valueEnv()).booleanValue()); + } + + @Test + public void compare_int_long() { + FunctionExpression equal = dsl.equal(DSL.literal(1), DSL.literal(1L)); + assertTrue(equal.valueOf(valueEnv()).booleanValue()); + } } \ No newline at end of file diff --git a/docs/experiment/ppl/cmd/stats.rst b/docs/experiment/ppl/cmd/stats.rst index 2871951949..5d0ce101c6 100644 --- a/docs/experiment/ppl/cmd/stats.rst +++ b/docs/experiment/ppl/cmd/stats.rst @@ -51,8 +51,8 @@ PPL query:: +----------+--------------------+ | gender | avg(age) | |----------+--------------------| - | M | 33.666666666666664 | | F | 28 | + | M | 33.666666666666664 | +----------+--------------------+ @@ -68,7 +68,7 @@ PPL query:: +----------+--------------------+------------+ | gender | avg(age) | sum(age) | |----------+--------------------+------------| - | M | 33.666666666666664 | 101 | | F | 28 | 28 | + | M | 33.666666666666664 | 101 | +----------+--------------------+------------+ diff --git a/docs/user/general/datatype.rst b/docs/user/general/datatype.rst new file mode 100644 index 0000000000..4ebdf6f56a --- /dev/null +++ b/docs/user/general/datatype.rst @@ -0,0 +1,74 @@ +========== +Data Types +========== + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +ODFE SQL Data Types +=================== + +The ODFE SQL Engine support the following data types. + ++---------------+ +| ODFE SQL Type | ++===============+ +| boolean | ++---------------+ +| integer | ++---------------+ +| long | ++---------------+ +| float | ++---------------+ +| double | ++---------------+ +| string | ++---------------+ +| text | ++---------------+ +| timestamp | ++---------------+ +| date | ++---------------+ +| time | ++---------------+ +| struct | ++---------------+ +| array | ++---------------+ + +Data Types Mapping +================== + +The table below list the mapping between Elasticsearch Data Type, ODFE SQL Data Type and SQL Type. + ++--------------------+---------------+-----------+ +| Elasticsearch Type | ODFE SQL Type | SQL Type | ++====================+===============+===========+ +| boolean | boolean | BOOLEAN | ++--------------------+---------------+-----------+ +| integer | integer | INTEGER | ++--------------------+---------------+-----------+ +| long | long | LONG | ++--------------------+---------------+-----------+ +| float | float | FLOAT | ++--------------------+---------------+-----------+ +| double | double | DOUBLE | ++--------------------+---------------+-----------+ +| keyword | string | VARCHAR | ++--------------------+---------------+-----------+ +| text | text | VARCHAR | ++--------------------+---------------+-----------+ +| date | timestamp | TIMESTAMP | ++--------------------+---------------+-----------+ +| object | struct | STRUCT | ++--------------------+---------------+-----------+ +| nested | array | TBD | ++--------------------+---------------+-----------+ + +Notes: Not all the ODFE SQL Type has correspond Elasticsearch Type. e.g. data and time. To use function which required such data type, user should explict convert the data type. diff --git a/docs/user/index.rst b/docs/user/index.rst index f20e9922f2..d5f668e9f3 100644 --- a/docs/user/index.rst +++ b/docs/user/index.rst @@ -20,6 +20,7 @@ Open Distro for Elasticsearch SQL enables you to extract insights out of Elastic * **Language Structure** - `Identifiers `_ + - `Data Types `_ * **Data Query Language** diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataType.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataType.java new file mode 100644 index 0000000000..cb2a277356 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataType.java @@ -0,0 +1,63 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; + +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import lombok.RequiredArgsConstructor; + +/** + * The extension of ExprType in Elasticsearch. + */ +@RequiredArgsConstructor +public enum ElasticsearchDataType implements ExprType { + /** + * Elasticsearch Text. + * Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/text.html + */ + ES_TEXT(Collections.singletonList(STRING), "string"), + + /** + * Elasticsearch multi-fields which has text and keyword. + * Ref: https://www.elastic.co/guide/en/elasticsearch/reference/current/multi-fields.html + */ + ES_TEXT_KEYWORD(Arrays.asList(STRING, ES_TEXT), "string"); + + /** + * Parent of current type. + */ + private final List parents; + /** + * JDBC type name. + */ + private final String jdbcType; + + @Override + public List getParent() { + return parents; + } + + @Override + public String typeName() { + return jdbcType; + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprTextValue.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprTextValue.java new file mode 100644 index 0000000000..927b62bbd2 --- /dev/null +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprTextValue.java @@ -0,0 +1,37 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value; + +import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; + +/** + * Expression Text Value, it is a extension of the ExprValue by Elasticsearch. + */ +public class ElasticsearchExprTextValue extends ExprStringValue { + public ElasticsearchExprTextValue(String value) { + super(value); + } + + @Override + public ExprType type() { + return ES_TEXT; + } +} diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java index a9dc7d31af..82832ac034 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactory.java @@ -27,6 +27,7 @@ import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchDateFormatters.SQL_LITERAL_DATE_TIME_FORMAT; import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchDateFormatters.STRICT_DATE_OPTIONAL_TIME_FORMATTER; @@ -111,6 +112,8 @@ private ExprValue construct(String field, JsonNode value) { return constructArray(value, field); } else if (type.equals(TIMESTAMP)) { return constructTimestamp(value); + } else if (type.equals(ES_TEXT)) { + return new ElasticsearchExprTextValue(value.asText()); } else { throw new IllegalStateException( String.format( diff --git a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java index c87dfc47e6..accc3bf52d 100644 --- a/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java +++ b/elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndex.java @@ -19,6 +19,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchExprValueFactory; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; import com.amazon.opendistroforelasticsearch.sql.planner.DefaultImplementor; @@ -37,11 +38,11 @@ public class ElasticsearchIndex implements Table { /** * Type mapping from Elasticsearch data type to expression type in our type system in query - * engine. TODO: date, geo, ip etc. + * engine. TODO: geo, ip etc. */ - private static final Map ES_TYPE_TO_EXPR_TYPE_MAPPING = - ImmutableMap.builder() - .put("text", ExprCoreType.STRING) + private static final Map ES_TYPE_TO_EXPR_TYPE_MAPPING = + ImmutableMap.builder() + .put("text", ElasticsearchDataType.ES_TEXT) .put("keyword", ExprCoreType.STRING) .put("integer", ExprCoreType.INTEGER) .put("long", ExprCoreType.LONG) @@ -94,7 +95,7 @@ public PhysicalPlan visitRelation(LogicalRelation node, ElasticsearchIndexScan c indexScan); } - private ExprCoreType transformESTypeToExprType(String esType) { + private ExprType transformESTypeToExprType(String esType) { return ES_TYPE_TO_EXPR_TYPE_MAPPING.getOrDefault(esType, ExprCoreType.UNKNOWN); } } diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataTypeTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataTypeTest.java new file mode 100644 index 0000000000..7b04afbe56 --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/type/ElasticsearchDataTypeTest.java @@ -0,0 +1,44 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type; + +import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; +import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT; +import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT_KEYWORD; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class ElasticsearchDataTypeTest { + @Test + public void testIsCompatible() { + assertTrue(STRING.isCompatible(ES_TEXT)); + assertFalse(ES_TEXT.isCompatible(STRING)); + + assertTrue(STRING.isCompatible(ES_TEXT_KEYWORD)); + assertTrue(ES_TEXT.isCompatible(ES_TEXT_KEYWORD)); + } + + @Test + public void testTypeName() { + assertEquals("string", ES_TEXT.typeName()); + assertEquals("string", ES_TEXT_KEYWORD.typeName()); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprTextValueTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprTextValueTest.java new file mode 100644 index 0000000000..2336cc9671 --- /dev/null +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprTextValueTest.java @@ -0,0 +1,30 @@ +/* + * + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value; + +import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class ElasticsearchExprTextValueTest { + @Test + public void typeOfExprTextValue() { + assertEquals(ES_TEXT, new ElasticsearchExprTextValue("A").type()); + } +} \ No newline at end of file diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java index b0c7064423..f83b323d3f 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchExprValueFactoryTest.java @@ -33,6 +33,7 @@ import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRUCT; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -67,6 +68,7 @@ class ElasticsearchExprValueFactoryTest { .put("arrayV", ARRAY) .put("arrayV.info", STRING) .put("arrayV.author", STRING) + .put("textV", ES_TEXT) .build(); private ElasticsearchExprValueFactory exprValueFactory = new ElasticsearchExprValueFactory(MAPPING); @@ -106,6 +108,12 @@ public void constructBoolean() { assertEquals(booleanValue(true), tupleValue("{\"boolV\":true}").get("boolV")); } + @Test + public void constructText() { + assertEquals(new ElasticsearchExprTextValue("text"), tupleValue("{\"textV\":\"text\"}").get( + "textV")); + } + @Test public void constructDate() { assertEquals( diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java index 0cd9e881c9..458c2f2ecd 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java @@ -41,6 +41,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.client.ElasticsearchClient; +import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchExprValueFactory; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; @@ -97,16 +98,16 @@ void getFieldTypes() { fieldTypes, allOf( aMapWithSize(10), - hasEntry("name", ExprCoreType.STRING), - hasEntry("address", ExprCoreType.STRING), - hasEntry("age", ExprCoreType.INTEGER), + hasEntry("name", (ExprType) ExprCoreType.STRING), + hasEntry("address", (ExprType) ElasticsearchDataType.ES_TEXT), + hasEntry("age", (ExprType) ExprCoreType.INTEGER), hasEntry("account_number", ExprCoreType.LONG), - hasEntry("balance1", ExprCoreType.FLOAT), - hasEntry("balance2", ExprCoreType.DOUBLE), - hasEntry("gender", ExprCoreType.BOOLEAN), - hasEntry("family", ExprCoreType.ARRAY), - hasEntry("employer", ExprCoreType.STRUCT), - hasEntry("birthday", ExprCoreType.TIMESTAMP))); + hasEntry("balance1", (ExprType) ExprCoreType.FLOAT), + hasEntry("balance2", (ExprType) ExprCoreType.DOUBLE), + hasEntry("gender", (ExprType) ExprCoreType.BOOLEAN), + hasEntry("family", (ExprType) ExprCoreType.ARRAY), + hasEntry("employer", (ExprType) ExprCoreType.STRUCT), + hasEntry("birthday", (ExprType) ExprCoreType.TIMESTAMP))); } @Test diff --git a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java index 45dfd5bf88..389b259861 100644 --- a/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java +++ b/protocol/src/main/java/com/amazon/opendistroforelasticsearch/sql/protocol/response/QueryResult.java @@ -93,7 +93,7 @@ private Object[] convertExprValuesToValues(Collection exprValues) { } private String getTypeString(ExprValue exprValue) { - return exprValue.type().name().toLowerCase(); + return exprValue.type().typeName().toLowerCase(); } } From b44d420d0b836ffff636ba39d1a1d90c50439ffb Mon Sep 17 00:00:00 2001 From: penghuo Date: Mon, 27 Jul 2020 09:53:51 -0700 Subject: [PATCH 2/3] update --- .../sql/expression/datetime/DateTimeFunction.java | 2 +- .../sql/expression/function/FunctionDSL.java | 4 ++-- .../operator/predicate/BinaryPredicateOperator.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java index 785b187041..62208b5692 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java @@ -45,7 +45,7 @@ public void register(BuiltinFunctionRepository repository) { */ private FunctionResolver dayOfMonth() { return FunctionDSL.define(DAYOFMONTH.getName(), - FunctionDSL.unaryImpl(DateTimeFunction::exprDayOfMonth, INTEGER, DATE) + FunctionDSL.impl(DateTimeFunction::exprDayOfMonth, INTEGER, DATE) ); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionDSL.java index 827707eea0..de129b4d51 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionDSL.java @@ -77,7 +77,7 @@ public FunctionResolver define(FunctionName functionName, * @param argsType argument type. * @return Unary Function Implementation. */ - public SerializableFunction> unaryImpl( + public SerializableFunction> impl( SerializableFunction function, ExprType returnType, ExprType argsType) { @@ -125,7 +125,7 @@ public String toString() { * @param args2Type argument type. * @return Unary Function Implementation. */ - public Function> binaryImpl( + public Function> impl( SerializableBiFunction function, ExprType returnType, ExprType args1Type, diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java index 0d6f493905..edfd60c153 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java @@ -196,7 +196,7 @@ private static FunctionResolver xor() { private static FunctionResolver equal() { return FunctionDSL.define(BuiltinFunctionName.EQUAL.getName(), ExprCoreType.coreTypes().stream() - .map(type -> FunctionDSL.binaryImpl((v1, v2) -> ExprBooleanValue.of(v1.equals(v2)), + .map(type -> FunctionDSL.impl((v1, v2) -> ExprBooleanValue.of(v1.equals(v2)), BOOLEAN, type, type)) .collect( Collectors.toList())); @@ -206,7 +206,7 @@ private static FunctionResolver notEqual() { return FunctionDSL .define(BuiltinFunctionName.NOTEQUAL.getName(), ExprCoreType.coreTypes().stream() .map(type -> FunctionDSL - .binaryImpl((v1, v2) -> ExprBooleanValue.of(!v1.equals(v2)), BOOLEAN, type, type)) + .impl((v1, v2) -> ExprBooleanValue.of(!v1.equals(v2)), BOOLEAN, type, type)) .collect( Collectors.toList())); } From e088f0a3c046063224ab7bc62c85295f72b4f161 Mon Sep 17 00:00:00 2001 From: penghuo Date: Wed, 29 Jul 2020 08:35:08 -0700 Subject: [PATCH 3/3] address comments --- .../sql/data/model/ExprMissingValue.java | 4 +-- .../expression/datetime/DateTimeFunction.java | 3 ++- .../sql/expression/function/FunctionDSL.java | 26 +++++++++++++------ 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprMissingValue.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprMissingValue.java index a09611e479..7eb29ab8f0 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprMissingValue.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprMissingValue.java @@ -50,8 +50,8 @@ public boolean isMissing() { /** * When MISSING value compare to other expression value. - * 1) NULL is equal to MISSING. - * 2) NULL is less than all other expression values. + * 1) MISSING is equal to MISSING. + * 2) MISSING is less than all other expression values. */ @Override public int compare(ExprValue other) { diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java index 62208b5692..4b96eb80a6 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/datetime/DateTimeFunction.java @@ -45,7 +45,8 @@ public void register(BuiltinFunctionRepository repository) { */ private FunctionResolver dayOfMonth() { return FunctionDSL.define(DAYOFMONTH.getName(), - FunctionDSL.impl(DateTimeFunction::exprDayOfMonth, INTEGER, DATE) + FunctionDSL.impl(FunctionDSL.nullMissingHandling(DateTimeFunction::exprDayOfMonth), + INTEGER, DATE) ); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionDSL.java index de129b4d51..841027656f 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/function/FunctionDSL.java @@ -90,13 +90,7 @@ public SerializableFunction valueEnv) { ExprValue value = arguments.get(0).valueOf(valueEnv); - if (value.isMissing()) { - return ExprValueUtils.missingValue(); - } else if (value.isNull()) { - return ExprValueUtils.nullValue(); - } else { - return function.apply(value); - } + return function.apply(value); } @Override @@ -125,7 +119,7 @@ public String toString() { * @param args2Type argument type. * @return Unary Function Implementation. */ - public Function> impl( + public SerializableFunction> impl( SerializableBiFunction function, ExprType returnType, ExprType args1Type, @@ -157,4 +151,20 @@ public String toString() { return Pair.of(functionSignature, functionBuilder); }; } + + /** + * Wrapper the unary ExprValue function with default NULL and MISSING handling. + */ + public SerializableFunction nullMissingHandling( + SerializableFunction function) { + return value -> { + if (value.isMissing()) { + return ExprValueUtils.missingValue(); + } else if (value.isNull()) { + return ExprValueUtils.nullValue(); + } else { + return function.apply(value); + } + }; + } }