From dee4aec709a6b73817685b91b7d9619d3cf581bf Mon Sep 17 00:00:00 2001 From: Chen Dai <46505291+dai-chen@users.noreply.github.com> Date: Fri, 31 Jul 2020 12:38:11 -0700 Subject: [PATCH] Support select fields and alias in new query engine (#636) * Change grammar * Add UT * Change grammar and add UT for field alias * Support alias by named expression * Add UT * Add javadoc * Make query field required after test * Change java doc * Fix comparison test data type issue * Fix broken UT * Fix jacoco coverage * Fix broken UTs * Fix PPL ast builder * Add doctest * Fix broken legacy IT * Fix jacoco * Fix all IT and preserve original name with alias * Fix remove command failure * Fix remove command failure * Fix doctest * Fix broken UT * Fix broken UT * Revert to make Alias optional for PPL * Prepare PR * Prepare PR * Prepare PR * Dont remove single quotes for column and alias name --- .../sql/common/utils/StringUtils.java | 20 ++++- .../sql/analysis/Analyzer.java | 7 +- .../sql/analysis/ExpressionAnalyzer.java | 34 ++++++++ .../sql/ast/AbstractNodeVisitor.java | 5 ++ .../sql/ast/dsl/AstDSL.java | 10 ++- .../sql/ast/expression/Alias.java | 58 +++++++++++++ .../sql/expression/DSL.java | 24 ++++++ .../sql/expression/NamedExpression.java | 72 +++++++++++++++++ .../sql/planner/logical/LogicalPlanDSL.java | 3 +- .../sql/planner/logical/LogicalProject.java | 4 +- .../sql/planner/physical/PhysicalPlanDSL.java | 3 +- .../sql/planner/physical/ProjectOperator.java | 8 +- .../sql/analysis/AnalyzerTest.java | 22 ++--- .../sql/analysis/ExpressionAnalyzerTest.java | 60 +++++++++++++- .../sql/expression/NamedExpressionTest.java | 45 +++++++++++ .../sql/planner/DefaultImplementorTest.java | 4 +- .../sql/planner/PlannerTest.java | 12 +-- .../logical/LogicalPlanNodeVisitorTest.java | 3 +- .../physical/PhysicalPlanNodeVisitorTest.java | 5 +- .../planner/physical/ProjectOperatorTest.java | 10 ++- docs/user/dql/expressions.rst | 25 +++--- docs/user/dql/functions.rst | 44 +++++----- .../ElasticsearchExecutionProtectorTest.java | 4 +- .../storage/ElasticsearchIndexTest.java | 4 +- .../correctness/runner/ComparisonTest.java | 6 +- .../runner/connection/DBConnection.java | 2 +- .../runner/connection/ESConnection.java | 6 +- .../runner/connection/JDBCConnection.java | 7 +- .../correctness/tests/TestDataSetTest.java | 66 ++++++++++++--- .../sql/correctness/testset/TestDataSet.java | 58 ++++++++++++- .../sql/legacy/PrettyFormatResponseIT.java | 1 + .../sql/legacy/QueryIT.java | 2 + .../sql/sql/IdentifierIT.java | 28 +++++++ .../sql/sql/MathematicalFunctionIT.java | 2 +- .../resources/correctness/queries/select.txt | 8 +- .../sql/legacy/plugin/RestSQLQueryAction.java | 13 ++- .../sql/legacy/plugin/RestSqlAction.java | 8 +- .../sql/ppl/parser/AstExpressionBuilder.java | 7 +- .../sql/ppl/utils/ArgumentFactory.java | 4 +- .../antlr/OpenDistroSQLIdentifierParser.g4 | 8 ++ sql/src/main/antlr/OpenDistroSQLLexer.g4 | 2 +- sql/src/main/antlr/OpenDistroSQLParser.g4 | 5 +- .../sql/sql/SQLService.java | 12 ++- .../sql/sql/parser/AstBuilder.java | 40 ++++++++- .../sql/sql/parser/AstExpressionBuilder.java | 40 +++++---- .../sql/sql/SQLServiceTest.java | 20 ++--- .../sql/sql/antlr/SQLSyntaxParserTest.java | 20 +++++ .../sql/sql/parser/AstBuilderTest.java | 81 +++++++++++++++---- .../parser/AstQualifiedNameBuilderTest.java | 2 +- 49 files changed, 759 insertions(+), 175 deletions(-) create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Alias.java create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/NamedExpression.java create mode 100644 core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/NamedExpressionTest.java diff --git a/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/utils/StringUtils.java b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/utils/StringUtils.java index 1df4b26758..51a757506e 100644 --- a/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/utils/StringUtils.java +++ b/common/src/main/java/com/amazon/opendistroforelasticsearch/sql/common/utils/StringUtils.java @@ -19,13 +19,13 @@ public class StringUtils { /** - * Unquote Identifier with mark. + * Unquote any string with mark specified. * @param text string * @param mark quotation mark * @return An unquoted string whose outer pair of (single/double/back-tick) quotes have been * removed */ - public static String unquoteIdentifier(String text, String mark) { + public static String unquote(String text, String mark) { if (isQuoted(text, mark)) { return text.substring(mark.length(), text.length() - mark.length()); } @@ -38,7 +38,7 @@ public static String unquoteIdentifier(String text, String mark) { * @return An unquoted string whose outer pair of (single/double/back-tick) quotes have been * removed */ - public static String unquoteIdentifier(String text) { + public static String unquoteText(String text) { if (isQuoted(text, "\"") || isQuoted(text, "'") || isQuoted(text, "`")) { return text.substring(1, text.length() - 1); } else { @@ -46,6 +46,20 @@ public static String unquoteIdentifier(String text) { } } + /** + * Unquote Identifier which has " or ` as mark. + * @param identifier identifier that possibly enclosed by double quotes or back ticks + * @return An unquoted string whose outer pair of (double/back-tick) quotes have been + * removed + */ + public static String unquoteIdentifier(String identifier) { + if (isQuoted(identifier, "\"") || isQuoted(identifier, "`")) { + return identifier.substring(1, identifier.length() - 1); + } else { + return identifier; + } + } + private static boolean isQuoted(String text, String mark) { return !Strings.isNullOrEmpty(text) && text.startsWith(mark) && text.endsWith(mark); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java index 03202b34ff..628720efec 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java @@ -15,6 +15,8 @@ package com.amazon.opendistroforelasticsearch.sql.analysis; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.named; + import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.Namespace; import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.Symbol; import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; @@ -40,6 +42,7 @@ import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression; import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalAggregation; @@ -166,8 +169,8 @@ public LogicalPlan visitProject(Project node, AnalysisContext context) { } } - List expressions = node.getProjectList().stream() - .map(expr -> expressionAnalyzer.analyze(expr, context)) + List expressions = node.getProjectList().stream() + .map(expr -> named(expressionAnalyzer.analyze(expr, context))) .collect(Collectors.toList()); return new LogicalProject(child, expressions); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java index 2f9710b7b6..db9a782d94 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzer.java @@ -19,6 +19,7 @@ import com.amazon.opendistroforelasticsearch.sql.analysis.symbol.Symbol; import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; import com.amazon.opendistroforelasticsearch.sql.ast.expression.AggregateFunction; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Alias; import com.amazon.opendistroforelasticsearch.sql.ast.expression.And; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Compare; import com.amazon.opendistroforelasticsearch.sql.ast.expression.EqualTo; @@ -27,10 +28,13 @@ import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Not; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Or; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.QualifiedName; import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedAttribute; import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Xor; +import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; @@ -149,10 +153,40 @@ public Expression visitField(Field node, AnalysisContext context) { return visitIdentifier(attr, context); } + @Override + public Expression visitQualifiedName(QualifiedName node, AnalysisContext context) { + // Name with qualifier (index.field, index_alias.field, object/nested.inner_field + // text.keyword) is not supported for now + if (node.getParts().size() > 1) { + throw new SyntaxCheckException(String.format( + "Qualified name [%s] is not supported yet", node)); + } + return visitIdentifier(node.toString(), context); + } + + @Override + public Expression visitAlias(Alias node, AnalysisContext context) { + return DSL.named(node.getName(), + node.getDelegated().accept(this, context), + node.getAlias()); + } + private Expression visitIdentifier(String ident, AnalysisContext context) { TypeEnvironment typeEnv = context.peek(); ReferenceExpression ref = DSL.ref(ident, typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident))); + + // Fall back to old engine too if type is not supported semantically + if (isTypeNotSupported(ref.type())) { + throw new SyntaxCheckException(String.format( + "Identifier [%s] of type [%s] is not supported yet", ident, ref.type())); + } return ref; } + + private boolean isTypeNotSupported(ExprType type) { + return "struct".equalsIgnoreCase(type.typeName()) + || "array".equalsIgnoreCase(type.typeName()); + } + } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/AbstractNodeVisitor.java index cb962c241a..f91212023a 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/AbstractNodeVisitor.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.sql.ast; import com.amazon.opendistroforelasticsearch.sql.ast.expression.AggregateFunction; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Alias; import com.amazon.opendistroforelasticsearch.sql.ast.expression.And; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; import com.amazon.opendistroforelasticsearch.sql.ast.expression.AttributeList; @@ -178,4 +179,8 @@ public T visitDedupe(Dedupe node, C context) { public T visitValues(Values node, C context) { return visitChildren(node, context); } + + public T visitAlias(Alias node, C context) { + return visitChildren(node, context); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java index 4c62c119d4..cb9578a6bf 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.sql.ast.dsl; import com.amazon.opendistroforelasticsearch.sql.ast.expression.AggregateFunction; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Alias; import com.amazon.opendistroforelasticsearch.sql.ast.expression.And; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Compare; @@ -43,7 +44,6 @@ import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort; import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Values; -import com.google.common.collect.ImmutableList; import java.util.Arrays; import java.util.List; import lombok.experimental.UtilityClass; @@ -226,6 +226,14 @@ public static Field field(String field, List fieldArgs) { return new Field(field, fieldArgs); } + public Alias alias(String name, UnresolvedExpression expr) { + return new Alias(name, expr); + } + + public Alias alias(String name, UnresolvedExpression expr, String alias) { + return new Alias(name, expr, alias); + } + public static List exprList(UnresolvedExpression... exprList) { return Arrays.asList(exprList); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Alias.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Alias.java new file mode 100644 index 0000000000..bcdac6e607 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Alias.java @@ -0,0 +1,58 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.ast.expression; + +import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Alias abstraction that associate an unnamed expression with a name and an optional alias. + * The name and alias information preserved is useful for semantic analysis and response + * formatting eventually. This can avoid restoring the info in toString() method which is + * inaccurate because original info is already lost. + */ +@AllArgsConstructor +@EqualsAndHashCode(callSuper = false) +@Getter +@RequiredArgsConstructor +@ToString +public class Alias extends UnresolvedExpression { + + /** + * Original field name. + */ + private final String name; + + /** + * Expression aliased. + */ + private final UnresolvedExpression delegated; + + /** + * Optional field alias. + */ + private String alias; + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitAlias(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java index f0d38ae143..76a0087557 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/DSL.java @@ -75,6 +75,30 @@ public static ReferenceExpression ref(String ref, ExprType type) { return new ReferenceExpression(ref, type); } + /** + * Wrap a named expression if not yet. The intent is that different languages may use + * Alias or not when building AST. This caused either named or unnamed expression + * is resolved by analyzer. To make unnamed expression acceptable for logical project, + * it is required to wrap it by named expression here before passing to logical project. + * + * @param expression expression + * @return expression if named already or expression wrapped by named expression. + */ + public static NamedExpression named(Expression expression) { + if (expression instanceof NamedExpression) { + return (NamedExpression) expression; + } + return named(expression.toString(), expression); + } + + public static NamedExpression named(String name, Expression expression) { + return new NamedExpression(name, expression); + } + + public static NamedExpression named(String name, Expression expression, String alias) { + return new NamedExpression(name, expression, alias); + } + public FunctionExpression abs(Expression... expressions) { return function(BuiltinFunctionName.ABS, expressions); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/NamedExpression.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/NamedExpression.java new file mode 100644 index 0000000000..6f3889fa6c --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/NamedExpression.java @@ -0,0 +1,72 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.expression; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType; +import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment; +import com.google.common.base.Strings; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Named expression that represents expression with name. + * Please see more details in associated unresolved expression operator + * {@link com.amazon.opendistroforelasticsearch.sql.ast.expression.Alias}. + */ +@AllArgsConstructor +@EqualsAndHashCode +@RequiredArgsConstructor +@ToString +public class NamedExpression implements Expression { + + /** + * Expression name. + */ + private final String name; + + /** + * Expression that being named. + */ + private final Expression delegated; + + /** + * Optional alias. + */ + private String alias; + + @Override + public ExprValue valueOf(Environment valueEnv) { + return delegated.valueOf(valueEnv); + } + + @Override + public ExprType type() { + return delegated.type(); + } + + /** + * Get expression name using name or its alias (if it's present). + * @return expression name + */ + public String getName() { + return Strings.isNullOrEmpty(alias) ? name : alias; + } + +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java index 2f828ad9e6..1066b279fc 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java @@ -18,6 +18,7 @@ import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression; import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; import com.google.common.collect.ImmutableSet; @@ -50,7 +51,7 @@ public static LogicalPlan rename( return new LogicalRename(input, renameMap); } - public static LogicalPlan project(LogicalPlan input, Expression... fields) { + public static LogicalPlan project(LogicalPlan input, NamedExpression... fields) { return new LogicalProject(input, Arrays.asList(fields)); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java index edf179903c..a68b176d60 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java @@ -15,7 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.planner.logical; -import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression; import java.util.Arrays; import java.util.List; import lombok.EqualsAndHashCode; @@ -32,7 +32,7 @@ public class LogicalProject extends LogicalPlan { private final LogicalPlan child; @Getter - private final List projectList; + private final List projectList; @Override public List getChild() { diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java index 6dedd39e04..40a8348976 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java @@ -18,6 +18,7 @@ import com.amazon.opendistroforelasticsearch.sql.ast.tree.Sort.SortOption; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.amazon.opendistroforelasticsearch.sql.expression.LiteralExpression; +import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression; import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; import com.google.common.collect.ImmutableSet; @@ -47,7 +48,7 @@ public static RenameOperator rename( return new RenameOperator(input, renameMap); } - public static ProjectOperator project(PhysicalPlan input, Expression... fields) { + public static ProjectOperator project(PhysicalPlan input, NamedExpression... fields) { return new ProjectOperator(input, Arrays.asList(fields)); } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java index 04aa049e57..28c1de18dc 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java @@ -17,7 +17,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; -import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableMap.Builder; import java.util.Collections; @@ -37,7 +37,7 @@ public class ProjectOperator extends PhysicalPlan { @Getter private final PhysicalPlan input; @Getter - private final List projectList; + private final List projectList; @Override public R accept(PhysicalPlanNodeVisitor visitor, C context) { @@ -58,11 +58,11 @@ public boolean hasNext() { public ExprValue next() { ExprValue inputValue = input.next(); ImmutableMap.Builder mapBuilder = new Builder<>(); - for (Expression expr : projectList) { + for (NamedExpression expr : projectList) { ExprValue exprValue = expr.valueOf(inputValue.bindingTuples()); // missing value is ignored. if (!exprValue.isMissing()) { - mapBuilder.put(expr.toString(), exprValue); + mapBuilder.put(expr.getName(), exprValue); } } return ExprTupleValue.fromExprValueMap(mapBuilder.build()); diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java index b7ffc28387..d705854661 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java @@ -131,13 +131,15 @@ public void rename_to_invalid_expression() { public void project_source() { assertAnalyzeEqual( LogicalPlanDSL.project( - LogicalPlanDSL.relation("schema"), DSL.ref("integer_value", INTEGER), DSL.ref( - "double_value", DOUBLE)), + LogicalPlanDSL.relation("schema"), + DSL.named("integer_value", DSL.ref("integer_value", INTEGER)), + DSL.named("double_value", DSL.ref("double_value", DOUBLE)) + ), AstDSL.projectWithArg( AstDSL.relation("schema"), AstDSL.defaultFieldsArgs(), - AstDSL.field("integer_value"), - AstDSL.field("double_value"))); + AstDSL.field("integer_value"), // Field not wrapped by Alias + AstDSL.alias("double_value", AstDSL.field("double_value")))); } @Test @@ -176,15 +178,15 @@ public void project_values() { assertAnalyzeEqual( LogicalPlanDSL.project( LogicalPlanDSL.values(ImmutableList.of(DSL.literal(123))), - DSL.literal(123), - DSL.literal("hello"), - DSL.literal(false) + DSL.named("123", DSL.literal(123)), + DSL.named("hello", DSL.literal("hello")), + DSL.named("false", DSL.literal(false)) ), AstDSL.project( AstDSL.values(ImmutableList.of(AstDSL.intLiteral(123))), - AstDSL.intLiteral(123), - AstDSL.stringLiteral("hello"), - AstDSL.booleanLiteral(false) + AstDSL.alias("123", AstDSL.intLiteral(123)), + AstDSL.alias("hello", AstDSL.stringLiteral("hello")), + AstDSL.alias("false", AstDSL.booleanLiteral(false)) ) ); } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzerTest.java index 843007edac..9d7535253c 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzerTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/ExpressionAnalyzerTest.java @@ -25,7 +25,7 @@ import com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL; import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; -import com.amazon.opendistroforelasticsearch.sql.data.model.ExprNullValue; +import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; @@ -74,6 +74,64 @@ public void not() { ); } + @Test + public void qualified_name() { + assertAnalyzeEqual( + DSL.ref("integer_value", INTEGER), + AstDSL.qualifiedName("integer_value") + ); + } + + @Test + public void named_expression() { + assertAnalyzeEqual( + DSL.named("int", DSL.ref("integer_value", INTEGER)), + AstDSL.alias("int", AstDSL.qualifiedName("integer_value")) + ); + } + + @Test + public void named_expression_with_alias() { + assertAnalyzeEqual( + DSL.named("integer", DSL.ref("integer_value", INTEGER), "int"), + AstDSL.alias("integer", AstDSL.qualifiedName("integer_value"), "int") + ); + } + + @Test + public void skip_identifier_with_qualifier() { + SyntaxCheckException exception = + assertThrows(SyntaxCheckException.class, + () -> analyze(AstDSL.qualifiedName("index_alias", "integer_value"))); + + assertEquals( + "Qualified name [index_alias.integer_value] is not supported yet", + exception.getMessage() + ); + } + + @Test + public void skip_struct_data_type() { + SyntaxCheckException exception = + assertThrows(SyntaxCheckException.class, + () -> analyze(AstDSL.qualifiedName("struct_value"))); + assertEquals( + "Identifier [struct_value] of type [STRUCT] is not supported yet", + exception.getMessage() + ); + } + + @Test + public void skip_array_data_type() { + SyntaxCheckException exception = + assertThrows(SyntaxCheckException.class, + () -> analyze(AstDSL.qualifiedName("array_value"))); + assertEquals( + "Identifier [array_value] of type [ARRAY] is not supported yet", + exception.getMessage() + ); + } + @Test public void undefined_var_semantic_check_failed() { SemanticCheckException exception = assertThrows(SemanticCheckException.class, diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/NamedExpressionTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/NamedExpressionTest.java new file mode 100644 index 0000000000..77196099ff --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/expression/NamedExpressionTest.java @@ -0,0 +1,45 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + * + */ + +package com.amazon.opendistroforelasticsearch.sql.expression; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; +import org.junit.jupiter.api.Test; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +class NamedExpressionTest extends ExpressionTestBase { + + @Test + void name_an_expression() { + LiteralExpression delegated = DSL.literal(10); + NamedExpression namedExpression = DSL.named("10", delegated); + + assertEquals("10", namedExpression.getName()); + assertEquals(delegated.type(), namedExpression.type()); + assertEquals(delegated.valueOf(valueEnv()), namedExpression.valueOf(valueEnv())); + } + + @Test + void name_an_expression_with_alias() { + LiteralExpression delegated = DSL.literal(10); + NamedExpression namedExpression = DSL.named("10", delegated, "ten"); + assertEquals("ten", namedExpression.getName()); + } + +} \ No newline at end of file diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java index 9931304c8e..427883c1d9 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/DefaultImplementorTest.java @@ -19,6 +19,7 @@ import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.literal; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.named; import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.ref; import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.aggregation; import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.eval; @@ -36,6 +37,7 @@ import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue; import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression; import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.AvgAggregator; @@ -59,7 +61,7 @@ class DefaultImplementorTest { @Test public void visitShouldReturnDefaultPhysicalOperator() { String indexName = "test"; - ReferenceExpression include = ref("age", INTEGER); + NamedExpression include = named("age", ref("age", INTEGER)); ReferenceExpression exclude = ref("name", STRING); ReferenceExpression dedupeField = ref("name", STRING); Expression filterExpr = literal(ExprBooleanValue.of(true)); diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java index eddd35493e..33e14cf902 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/PlannerTest.java @@ -98,15 +98,15 @@ public void plan_a_query_without_relation_involved() { assertPhysicalPlan( PhysicalPlanDSL.project( PhysicalPlanDSL.values(emptyList()), - DSL.literal(123), - DSL.literal("hello"), - DSL.literal(false) + DSL.named("123", DSL.literal(123)), + DSL.named("hello", DSL.literal("hello")), + DSL.named("false", DSL.literal(false)) ), LogicalPlanDSL.project( LogicalPlanDSL.values(emptyList()), - DSL.literal(123), - DSL.literal("hello"), - DSL.literal(false) + DSL.named("123", DSL.literal(123)), + DSL.named("hello", DSL.literal("hello")), + DSL.named("false", DSL.literal(false)) ) ); } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java index 4f9dcb2107..fd360850c6 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java @@ -15,6 +15,7 @@ package com.amazon.opendistroforelasticsearch.sql.planner.logical; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.named; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -77,7 +78,7 @@ public void testAbstractPlanNodeVisitorShouldReturnNull() { assertNull(rename.accept(new LogicalPlanNodeVisitor() { }, null)); - LogicalPlan project = LogicalPlanDSL.project(relation, ref); + LogicalPlan project = LogicalPlanDSL.project(relation, named("ref", ref)); assertNull(project.accept(new LogicalPlanNodeVisitor() { }, null)); diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java index 4a6de485cc..2feaf09f7f 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java @@ -17,6 +17,7 @@ import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.DOUBLE; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.named; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -56,7 +57,7 @@ public void print_physical_plan() { ImmutableList.of(dsl.avg(DSL.ref("response", INTEGER))), ImmutableList.of()), ImmutableMap.of(DSL.ref("ivalue", INTEGER), DSL.ref("avg(response)", DOUBLE))), - ref), + named("ref", ref)), ref); PhysicalPlanPrinter printer = new PhysicalPlanPrinter(); @@ -90,7 +91,7 @@ public void test_PhysicalPlanVisitor_should_return_null() { assertNull(rename.accept(new PhysicalPlanNodeVisitor() { }, null)); - PhysicalPlan project = PhysicalPlanDSL.project(plan, ref); + PhysicalPlan project = PhysicalPlanDSL.project(plan, named("ref", ref)); assertNull(project.accept(new PhysicalPlanNodeVisitor() { }, null)); diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperatorTest.java index 09d97b19d8..ec6af198d7 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperatorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperatorTest.java @@ -45,7 +45,7 @@ public void project_one_field() { when(inputPlan.hasNext()).thenReturn(true, false); when(inputPlan.next()) .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", 200))); - PhysicalPlan plan = project(inputPlan, DSL.ref("action", STRING)); + PhysicalPlan plan = project(inputPlan, DSL.named("action", DSL.ref("action", STRING))); List result = execute(plan); assertThat( @@ -60,7 +60,9 @@ public void project_two_field_follow_the_project_order() { when(inputPlan.hasNext()).thenReturn(true, false); when(inputPlan.next()) .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", 200))); - PhysicalPlan plan = project(inputPlan, DSL.ref("response", INTEGER), DSL.ref("action", STRING)); + PhysicalPlan plan = project(inputPlan, + DSL.named("response", DSL.ref("response", INTEGER)), + DSL.named("action", DSL.ref("action", STRING))); List result = execute(plan); assertThat( @@ -77,7 +79,9 @@ public void project_ignore_missing_value() { when(inputPlan.next()) .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", 200))) .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "POST"))); - PhysicalPlan plan = project(inputPlan, DSL.ref("response", INTEGER), DSL.ref("action", STRING)); + PhysicalPlan plan = project(inputPlan, + DSL.named("response", DSL.ref("response", INTEGER)), + DSL.named("action", DSL.ref("action", STRING))); List result = execute(plan); assertThat( diff --git a/docs/user/dql/expressions.rst b/docs/user/dql/expressions.rst index 7c1af0853d..8b2fc88dce 100644 --- a/docs/user/dql/expressions.rst +++ b/docs/user/dql/expressions.rst @@ -37,7 +37,7 @@ Here is an example for different type of literals:: od> SELECT 123, 'hello', false, -4.567, DATE '2020-07-07', TIME '01:01:01', TIMESTAMP '2020-07-07 01:01:01'; fetched rows / total rows = 1/1 +-------+-----------+---------+----------+---------------------+-------------------+-----------------------------------+ - | 123 | "hello" | false | -4.567 | DATE '2020-07-07' | TIME '01:01:01' | TIMESTAMP '2020-07-07 01:01:01' | + | 123 | 'hello' | false | -4.567 | DATE '2020-07-07' | TIME '01:01:01' | TIMESTAMP '2020-07-07 01:01:01' | |-------+-----------+---------+----------+---------------------+-------------------+-----------------------------------| | 123 | hello | False | -4.567 | 2020-07-07 | 01:01:01 | 2020-07-07 01:01:01 | +-------+-----------+---------+----------+---------------------+-------------------+-----------------------------------+ @@ -86,11 +86,11 @@ Here is an example for different type of arithmetic expressions:: od> SELECT 1 + 2, (9 - 1) % 3, 2 * 4 / 3; fetched rows / total rows = 1/1 - +---------+-------------+-------------+ - | 1 + 2 | 9 - 1 % 3 | 2 * 4 / 3 | - |---------+-------------+-------------| - | 3 | 2 | 2 | - +---------+-------------+-------------+ + +---------+---------------+-------------+ + | 1 + 2 | (9 - 1) % 3 | 2 * 4 / 3 | + |---------+---------------+-------------| + | 3 | 2 | 2 | + +---------+---------------+-------------+ Comparison Operators ================================== @@ -151,7 +151,7 @@ expr LIKE pattern. The expr is string value, pattern is supports literal text, a od> SELECT 'axyzb' LIKE 'a%b', 'acb' LIKE 'a_b', 'axyzb' NOT LIKE 'a%b', 'acb' NOT LIKE 'a_b'; fetched rows / total rows = 1/1 +----------------------+--------------------+--------------------------+------------------------+ - | "axyzb" like "a%b" | "acb" like "a_b" | "axyzb" not like "a%b" | "acb" not like "a_b" | + | 'axyzb' LIKE 'a%b' | 'acb' LIKE 'a_b' | 'axyzb' NOT LIKE 'a%b' | 'acb' NOT LIKE 'a_b' | |----------------------+--------------------+--------------------------+------------------------| | True | True | False | False | +----------------------+--------------------+--------------------------+------------------------+ @@ -163,12 +163,11 @@ Here is an example for null value test:: od> SELECT 0 IS NULL, 0 IS NOT NULL, NULL IS NULL, NULL IS NOT NULL; fetched rows / total rows = 1/1 - +--------------+------------------+-----------------+---------------------+ - | is null(0) | is not null(0) | is null(NULL) | is not null(NULL) | - |--------------+------------------+-----------------+---------------------| - | False | True | True | False | - +--------------+------------------+-----------------+---------------------+ - + +-------------+-----------------+----------------+--------------------+ + | 0 IS NULL | 0 IS NOT NULL | NULL IS NULL | NULL IS NOT NULL | + |-------------+-----------------+----------------+--------------------| + | False | True | True | False | + +-------------+-----------------+----------------+--------------------+ Function Call ============= diff --git a/docs/user/dql/functions.rst b/docs/user/dql/functions.rst index a8d97fde7c..85ed109425 100644 --- a/docs/user/dql/functions.rst +++ b/docs/user/dql/functions.rst @@ -44,7 +44,7 @@ Example:: od> SELECT ACOS(0) fetched rows / total rows = 1/1 +--------------------+ - | acos(0) | + | ACOS(0) | |--------------------| | 1.5707963267948966 | +--------------------+ @@ -89,7 +89,7 @@ Example:: od> SELECT ASIN(0) fetched rows / total rows = 1/1 +-----------+ - | asin(0) | + | ASIN(0) | |-----------| | 0 | +-----------+ @@ -112,7 +112,7 @@ Example:: od> SELECT ATAN(2), ATAN(2, 3) fetched rows / total rows = 1/1 +--------------------+--------------------+ - | atan(2) | atan(2, 3) | + | ATAN(2) | ATAN(2, 3) | |--------------------+--------------------| | 1.1071487177940904 | 0.5880026035475675 | +--------------------+--------------------+ @@ -135,7 +135,7 @@ Example:: od> SELECT ATAN2(2, 3) fetched rows / total rows = 1/1 +--------------------+ - | atan2(2, 3) | + | ATAN2(2, 3) | |--------------------| | 0.5880026035475675 | +--------------------+ @@ -205,7 +205,7 @@ Example:: od> SELECT CONV('12', 10, 16), CONV('2C', 16, 10), CONV(12, 10, 2), CONV(1111, 2, 10) fetched rows / total rows = 1/1 +----------------------+----------------------+-------------------+---------------------+ - | conv("12", 10, 16) | conv("2C", 16, 10) | conv(12, 10, 2) | conv(1111, 2, 10) | + | CONV('12', 10, 16) | CONV('2C', 16, 10) | CONV(12, 10, 2) | CONV(1111, 2, 10) | |----------------------+----------------------+-------------------+---------------------| | c | 44 | 1100 | 15 | +----------------------+----------------------+-------------------+---------------------+ @@ -227,7 +227,7 @@ Example:: od> SELECT COS(0) fetched rows / total rows = 1/1 +----------+ - | cos(0) | + | COS(0) | |----------| | 1 | +----------+ @@ -261,7 +261,7 @@ Example:: od> SELECT COT(1) fetched rows / total rows = 1/1 +--------------------+ - | cot(1) | + | COT(1) | |--------------------| | 0.6420926159343306 | +--------------------+ @@ -284,7 +284,7 @@ Example:: od> SELECT CRC32('MySQL') fetched rows / total rows = 1/1 +------------------+ - | crc32("MySQL") | + | CRC32('MySQL') | |------------------| | 3259397556 | +------------------+ @@ -352,7 +352,7 @@ Example:: od> SELECT DEGREES(1.57) fetched rows / total rows = 1/1 +-------------------+ - | degrees(1.57) | + | DEGREES(1.57) | |-------------------| | 89.95437383553924 | +-------------------+ @@ -384,7 +384,7 @@ Example:: od> SELECT E() fetched rows / total rows = 1/1 +-------------------+ - | e() | + | E() | |-------------------| | 2.718281828459045 | +-------------------+ @@ -586,7 +586,7 @@ Example:: od> SELECT MOD(3, 2), MOD(3.1, 2) fetched rows / total rows = 1/1 +-------------+---------------+ - | mod(3, 2) | mod(3.1, 2) | + | MOD(3, 2) | MOD(3.1, 2) | |-------------+---------------| | 1 | 1.1 | +-------------+---------------+ @@ -651,7 +651,7 @@ Example:: od> SELECT PI() fetched rows / total rows = 1/1 +-------------------+ - | pi() | + | PI() | |-------------------| | 3.141592653589793 | +-------------------+ @@ -674,7 +674,7 @@ Example:: od> SELECT POW(3, 2), POW(-3, 2), POW(3, -2) fetched rows / total rows = 1/1 +-------------+--------------+--------------------+ - | pow(3, 2) | pow(-3, 2) | pow(3, -2) | + | POW(3, 2) | POW(-3, 2) | POW(3, -2) | |-------------+--------------+--------------------| | 9 | 9 | 0.1111111111111111 | +-------------+--------------+--------------------+ @@ -697,7 +697,7 @@ Example:: od> SELECT POWER(3, 2), POWER(-3, 2), POWER(3, -2) fetched rows / total rows = 1/1 +---------------+----------------+--------------------+ - | power(3, 2) | power(-3, 2) | power(3, -2) | + | POWER(3, 2) | POWER(-3, 2) | POWER(3, -2) | |---------------+----------------+--------------------| | 9 | 9 | 0.1111111111111111 | +---------------+----------------+--------------------+ @@ -720,7 +720,7 @@ Example:: od> SELECT RADIANS(90) fetched rows / total rows = 1/1 +--------------------+ - | radians(90) | + | RADIANS(90) | |--------------------| | 1.5707963267948966 | +--------------------+ @@ -743,7 +743,7 @@ Example:: od> SELECT RAND(3) fetched rows / total rows = 1/1 +------------+ - | rand(3) | + | RAND(3) | |------------| | 0.73105735 | +------------+ @@ -802,7 +802,7 @@ Example:: od> SELECT ROUND(12.34), ROUND(12.34, 1), ROUND(12.34, -1), ROUND(12, 1) fetched rows / total rows = 1/1 +----------------+-------------------+--------------------+----------------+ - | round(12.34) | round(12.34, 1) | round(12.34, -1) | round(12, 1) | + | ROUND(12.34) | ROUND(12.34, 1) | ROUND(12.34, -1) | ROUND(12, 1) | |----------------+-------------------+--------------------+----------------| | 12 | 12.3 | 10 | 12 | +----------------+-------------------+--------------------+----------------+ @@ -836,7 +836,7 @@ Example:: od> SELECT SIGN(1), SIGN(0), SIGN(-1.1) fetched rows / total rows = 1/1 +-----------+-----------+--------------+ - | sign(1) | sign(0) | sign(-1.1) | + | SIGN(1) | SIGN(0) | SIGN(-1.1) | |-----------+-----------+--------------| | 1 | 0 | -1 | +-----------+-----------+--------------+ @@ -870,7 +870,7 @@ Example:: od> SELECT SIN(0) fetched rows / total rows = 1/1 +----------+ - | sin(0) | + | SIN(0) | |----------| | 0 | +----------+ @@ -907,7 +907,7 @@ Example:: od> SELECT SQRT(4), SQRT(4.41) fetched rows / total rows = 1/1 +-----------+--------------+ - | sqrt(4) | sqrt(4.41) | + | SQRT(4) | SQRT(4.41) | |-----------+--------------| | 2 | 2.1 | +-----------+--------------+ @@ -952,7 +952,7 @@ Example:: od> SELECT TAN(0) fetched rows / total rows = 1/1 +----------+ - | tan(0) | + | TAN(0) | |----------| | 0 | +----------+ @@ -999,7 +999,7 @@ Example:: fetched rows / total rows = 1/1 +----------------------+-----------------------+-------------------+ - | truncate(56.78, 1) | truncate(56.78, -1) | truncate(56, 1) | + | TRUNCATE(56.78, 1) | TRUNCATE(56.78, -1) | TRUNCATE(56, 1) | |----------------------+-----------------------+-------------------| | 56.7 | 50 | 56 | +----------------------+-----------------------+-------------------+ diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionProtectorTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionProtectorTest.java index c5dcf97b55..e254274197 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionProtectorTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/executor/ElasticsearchExecutionProtectorTest.java @@ -21,6 +21,7 @@ import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.literal; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.named; import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.ref; import static com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL.filter; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -33,6 +34,7 @@ import com.amazon.opendistroforelasticsearch.sql.elasticsearch.executor.protector.ResourceMonitorPlan; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.storage.ElasticsearchIndexScan; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression; import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.AvgAggregator; @@ -72,7 +74,7 @@ public void setup() { @Test public void testProtectIndexScan() { String indexName = "test"; - ReferenceExpression include = ref("age", INTEGER); + NamedExpression include = named("age", ref("age", INTEGER)); ReferenceExpression exclude = ref("name", STRING); ReferenceExpression dedupeField = ref("name", STRING); Expression filterExpr = literal(ExprBooleanValue.of(true)); diff --git a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java index 458c2f2ecd..12eaf2c9fc 100644 --- a/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java +++ b/elasticsearch/src/test/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/storage/ElasticsearchIndexTest.java @@ -20,6 +20,7 @@ import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER; import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING; import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.literal; +import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.named; import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.ref; import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.aggregation; import static com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL.eval; @@ -45,6 +46,7 @@ import com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value.ElasticsearchExprValueFactory; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.mapping.IndexMapping; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.NamedExpression; import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.AvgAggregator; @@ -122,7 +124,7 @@ void implementRelationOperatorOnly() { @Test void implementOtherLogicalOperators() { String indexName = "test"; - ReferenceExpression include = ref("age", INTEGER); + NamedExpression include = named("age", ref("age", INTEGER)); ReferenceExpression exclude = ref("name", STRING); ReferenceExpression dedupeField = ref("name", STRING); Expression filterExpr = literal(ExprBooleanValue.of(true)); diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java index 333d864b06..53d040dd6b 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/ComparisonTest.java @@ -156,9 +156,9 @@ private int nextId() { return testCaseId++; } - private void insertTestDataInBatch(DBConnection conn, String tableName, List testData) { - Iterator iterator = testData.iterator(); - String[] fieldNames = iterator.next(); // first row is header of column names + private void insertTestDataInBatch(DBConnection conn, String tableName, List testData) { + Iterator iterator = testData.iterator(); + String[] fieldNames = (String[]) iterator.next(); // first row is header of column names Iterators.partition(iterator, 100). forEachRemaining(batch -> conn.insert(tableName, fieldNames, batch)); } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/DBConnection.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/DBConnection.java index 6779398be4..46eff96b6f 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/DBConnection.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/DBConnection.java @@ -48,7 +48,7 @@ public interface DBConnection { * @param columnNames column names * @param batch batch of rows */ - void insert(String tableName, String[] columnNames, List batch); + void insert(String tableName, String[] columnNames, List batch); /** * Fetch data from database. diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/ESConnection.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/ESConnection.java index 4ce4a7bde2..04dbecae0f 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/ESConnection.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/ESConnection.java @@ -66,7 +66,7 @@ public void drop(String tableName) { } @Override - public void insert(String tableName, String[] columnNames, List batch) { + public void insert(String tableName, String[] columnNames, List batch) { Request request = new Request("POST", "/" + tableName + "/_bulk?refresh=true"); request.setJsonEntity(buildBulkBody(columnNames, batch)); performRequest(request); @@ -96,9 +96,9 @@ private void performRequest(Request request) { } } - private String buildBulkBody(String[] columnNames, List batch) { + private String buildBulkBody(String[] columnNames, List batch) { StringBuilder body = new StringBuilder(); - for (String[] fieldValues : batch) { + for (Object[] fieldValues : batch) { JSONObject json = new JSONObject(); for (int i = 0; i < columnNames.length; i++) { json.put(columnNames[i], fieldValues[i]); diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java index ed731330b1..2aff6b982c 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/runner/connection/JDBCConnection.java @@ -93,10 +93,10 @@ public void drop(String tableName) { } @Override - public void insert(String tableName, String[] columnNames, List batch) { + public void insert(String tableName, String[] columnNames, List batch) { try (Statement stmt = connection.createStatement()) { String names = String.join(",", columnNames); - for (String[] fieldValues : batch) { + for (Object[] fieldValues : batch) { stmt.addBatch(StringUtils.format( "INSERT INTO %s(%s) VALUES (%s)", tableName, names, getValueList(fieldValues))); } @@ -139,8 +139,9 @@ private String parseColumnNameAndTypesInSchemaJson(String schema) { collect(joining(",")); } - private String getValueList(String[] fieldValues) { + private String getValueList(Object[] fieldValues) { return Arrays.stream(fieldValues). + map(String::valueOf). map(val -> val.replace(SINGLE_QUOTE, DOUBLE_QUOTE)). map(val -> SINGLE_QUOTE + val + SINGLE_QUOTE). collect(joining(",")); diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestDataSetTest.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestDataSetTest.java index 6ca49fa34c..6077e18f47 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestDataSetTest.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/tests/TestDataSetTest.java @@ -29,41 +29,81 @@ public class TestDataSetTest { @Test public void testDataSetWithSingleColumnData() { - TestDataSet dataSet = new TestDataSet("test", "mappings", "hello\nworld\n123"); + String mappings = + "{\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"field\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + TestDataSet dataSet = new TestDataSet("test", mappings, "field\nhello\nworld\n123"); assertEquals("test", dataSet.getTableName()); - assertEquals("mappings", dataSet.getSchema()); + assertEquals(mappings, dataSet.getSchema()); assertThat( dataSet.getDataRows(), contains( - new String[] {"hello"}, - new String[] {"world"}, - new String[] {"123"} + new Object[] {"field"}, + new Object[] {"hello"}, + new Object[] {"world"}, + new Object[] {"123"} ) ); } @Test public void testDataSetWithMultiColumnsData() { - TestDataSet dataSet = new TestDataSet("test", "mappings", "hello,world\n123"); + String mappings = + "{\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"field1\": {\n" + + " \"type\": \"text\"\n" + + " },\n" + + " \"field2\": {\n" + + " \"type\": \"integer\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + TestDataSet dataSet = new TestDataSet("test", mappings, + "field1,field2\nhello,123\nworld,456"); assertThat( dataSet.getDataRows(), contains( - new String[] {"hello", "world"}, - new String[] {"123"} + new Object[] {"field1", "field2"}, + new Object[] {"hello", 123}, + new Object[] {"world", 456} ) ); } @Test public void testDataSetWithEscapedComma() { - TestDataSet dataSet = new TestDataSet("test", "mappings", - "hello,\"hello,world,123\"\n123\n\"[abc,def,ghi]\",456"); + String mappings = + "{\n" + + " \"mappings\": {\n" + + " \"properties\": {\n" + + " \"field\": {\n" + + " \"type\": \"text\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + + TestDataSet dataSet = new TestDataSet("test", mappings, + "field\n\"hello,world,123\"\n123\n\"[abc,def,ghi]\""); assertThat( dataSet.getDataRows(), contains( - new String[] {"hello", "hello,world,123"}, - new String[] {"123"}, - new String[] {"[abc,def,ghi]", "456"} + new Object[] {"field"}, + new Object[] {"hello,world,123"}, + new Object[] {"123"}, + new Object[] {"[abc,def,ghi]"} ) ); } diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestDataSet.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestDataSet.java index ab5b77dc09..42036a2f2a 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestDataSet.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/correctness/testset/TestDataSet.java @@ -18,9 +18,11 @@ import static com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils.unquoteSingleField; import static java.util.stream.Collectors.joining; +import com.amazon.opendistroforelasticsearch.sql.legacy.utils.StringUtils; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.json.JSONObject; /** * Test data set @@ -29,12 +31,12 @@ public class TestDataSet { private final String tableName; private final String schema; - private final List dataRows; + private final List dataRows; public TestDataSet(String tableName, String schemaFileContent, String dataFileContent) { this.tableName = tableName; this.schema = schemaFileContent; - this.dataRows = splitColumns(dataFileContent, ','); + this.dataRows = convertStringDataToActualType(splitColumns(dataFileContent, ',')); } public String getTableName() { @@ -45,7 +47,7 @@ public String getSchema() { return schema; } - public List getDataRows() { + public List getDataRows() { return dataRows; } @@ -82,6 +84,56 @@ private List splitColumns(String content, char separator) { return result; } + /** + * Convert column string values (read from CSV file) to objects of its real type + * based on the type information in index mapping file. + */ + private List convertStringDataToActualType(List rows) { + JSONObject types = new JSONObject(schema); + String[] columnNames = rows.get(0); + + List result = new ArrayList<>(); + result.add(columnNames); + + rows.stream() + .skip(1) + .map(row -> convertStringArrayToObjectArray(types, columnNames, row)) + .forEach(result::add); + return result; + } + + private Object[] convertStringArrayToObjectArray(JSONObject types, String[] columnNames, String[] row) { + Object[] result = new Object[row.length]; + for (int i = 0; i < row.length; i++) { + String colName = columnNames[i]; + String colTypePath = "/mappings/properties/" + colName; + String colType = ((JSONObject) types.query(colTypePath)).getString("type"); + result[i] = convertStringToObject(colType, row[i]); + } + return result; + } + + private Object convertStringToObject(String type, String str) { + switch (type.toLowerCase()) { + case "text": + case "keyword": + case "date": + return str; + case "integer": + return Integer.valueOf(str); + case "float": + case "half_float": + return Float.valueOf(str); + case "double": + return Double.valueOf(str); + case "boolean": + return Boolean.valueOf(str); + default: + throw new IllegalStateException(StringUtils.format( + "Data type %s is not supported yet for value: %s", type, str)); + } + } + @Override public String toString() { int total = dataRows.size(); diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java index cb5be363dc..273f366171 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/PrettyFormatResponseIT.java @@ -433,6 +433,7 @@ public void aggregationFunctionInHaving() throws IOException { // public void nestedAggregationFunctionInSelect() { // String query = String.format(Locale.ROOT, "SELECT SUM(SQRT(age)) FROM age GROUP BY age", TEST_INDEX_ACCOUNT); // } + @Ignore("New engine returns string type") @Test public void fieldsWithAlias() throws IOException { JSONObject response = executeQuery( diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryIT.java index bdf5e09435..1f5c644f98 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/legacy/QueryIT.java @@ -1648,6 +1648,7 @@ public void fieldCollapsingTest() throws IOException { Assert.assertEquals(21, hits.length()); } + @Ignore("New engine doesn't have 'alias' field in schema in response") @Test public void backticksQuotedIndexNameTest() throws Exception { TestUtils.createIndexByRestClient(client(), "bank_unquote", null); @@ -1754,6 +1755,7 @@ public void functionInCaseFieldShouldThrowESExceptionDueToIllegalScriptInJdbc() "For more details, please send request for Json format"); } + @Ignore("This is already supported in our new query engine") @Test public void functionCallWithIllegalScriptShouldThrowESExceptionInJdbc() { String response = executeQuery("select log(balance + 2) from " + TEST_INDEX_BANK, diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/IdentifierIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/IdentifierIT.java index 21d9614e25..3b5dc26bc5 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/IdentifierIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/IdentifierIT.java @@ -50,6 +50,34 @@ public void testQuotedIndexNames() throws IOException { queryAndAssertTheDoc("SELECT * FROM \"logs.2020.01\""); } + @Test + public void testSpecialFieldName() throws IOException { + new Index("test") + .addDoc("{\"@timestamp\": 10, \"dimensions:major_version\": 30}"); + + assertEquals( + "{\n" + + " \"schema\": [\n" + + " {\n" + + " \"name\": \"@timestamp\",\n" + + " \"type\": \"long\"\n" + + " },\n" + + " {\n" + + " \"name\": \"dimensions:major_version\",\n" + + " \"type\": \"long\"\n" + + " }\n" + + " ],\n" + + " \"total\": 1,\n" + + " \"datarows\": [[\n" + + " 10,\n" + + " 30\n" + + " ]],\n" + + " \"size\": 1\n" + + "}\n", + executeQuery("SELECT @timestamp, `dimensions:major_version` FROM test", "jdbc") + ); + } + private void createIndexWithOneDoc(String... indexNames) throws IOException { for (String indexName : indexNames) { new Index(indexName).addDoc("{\"age\": 30}"); diff --git a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/MathematicalFunctionIT.java b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/MathematicalFunctionIT.java index 5bfadb3c6f..ffc33c1324 100644 --- a/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/MathematicalFunctionIT.java +++ b/integ-test/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/MathematicalFunctionIT.java @@ -54,7 +54,7 @@ public void testConv() throws IOException { @Test public void testCrc32() throws IOException { JSONObject result = executeQuery("select crc32('MySQL')"); - verifySchema(result, schema("crc32(\"MySQL\")", null, "long")); + verifySchema(result, schema("crc32('MySQL')", null, "long")); verifyDataRows(result, rows(3259397556L)); } diff --git a/integ-test/src/test/resources/correctness/queries/select.txt b/integ-test/src/test/resources/correctness/queries/select.txt index adb7f40782..298e0f22a1 100644 --- a/integ-test/src/test/resources/correctness/queries/select.txt +++ b/integ-test/src/test/resources/correctness/queries/select.txt @@ -1,4 +1,8 @@ SELECT 1 + 2 FROM kibana_sample_data_flights -SELECT abs(-10) FROM kibana_sample_data_flights -SELECT DistanceMiles FROM kibana_sample_data_flights +SELECT Cancelled, AvgTicketPrice, FlightDelayMin, Carrier, timestamp FROM kibana_sample_data_flights +SELECT `Cancelled`, `AvgTicketPrice` FROM kibana_sample_data_flights +SELECT ABS(DistanceMiles), (FlightDelayMin * 2) - 3 FROM kibana_sample_data_flights +SELECT abs(DistanceMiles), Abs(FlightDelayMin) FROM kibana_sample_data_flights +SELECT Cancelled AS Cancel, AvgTicketPrice AS ATP FROM kibana_sample_data_flights +SELECT Cancelled AS `Cancel`, AvgTicketPrice AS "ATP" FROM kibana_sample_data_flights SELECT AvgTicketPrice, Carrier FROM kibana_sample_data_flights WHERE AvgTicketPrice <= 500 diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java index 6f243e684b..70eb40cb8c 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSQLQueryAction.java @@ -21,10 +21,11 @@ import static org.elasticsearch.rest.RestStatus.INTERNAL_SERVER_ERROR; import static org.elasticsearch.rest.RestStatus.OK; -import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException; import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; import com.amazon.opendistroforelasticsearch.sql.elasticsearch.security.SecurityAccess; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; import com.amazon.opendistroforelasticsearch.sql.protocol.response.QueryResult; import com.amazon.opendistroforelasticsearch.sql.protocol.response.format.SimpleJsonResponseFormatter; import com.amazon.opendistroforelasticsearch.sql.sql.SQLService; @@ -89,13 +90,17 @@ public RestChannelConsumer prepareRequest(SQLQueryRequest request, NodeClient no } SQLService sqlService = createSQLService(nodeClient); - UnresolvedPlan ast; + PhysicalPlan plan; try { - ast = sqlService.parse(request.getQuery()); + // For now analyzing and planning stage may throw syntax exception as well + // which hints the fallback to legacy code is necessary here. + plan = sqlService.plan( + sqlService.analyze( + sqlService.parse(request.getQuery()))); } catch (SyntaxCheckException e) { return NOT_SUPPORTED_YET; } - return channel -> sqlService.execute(ast, createListener(channel)); + return channel -> sqlService.execute(plan, createListener(channel)); } private SQLService createSQLService(NodeClient client) { diff --git a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java index b715dc8364..073fe52efc 100644 --- a/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java +++ b/legacy/src/main/java/com/amazon/opendistroforelasticsearch/sql/legacy/plugin/RestSqlAction.java @@ -67,6 +67,7 @@ import java.util.function.Predicate; import java.util.regex.Pattern; +import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.CURSOR_ENABLED; import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_ENABLED; import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_SUGGESTION; import static com.amazon.opendistroforelasticsearch.sql.legacy.plugin.SqlSettings.QUERY_ANALYSIS_SEMANTIC_THRESHOLD; @@ -145,7 +146,7 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli Format format = SqlRequestParam.getFormat(request.params()); - if (isNewEngineEnabled()) { + if (isNewEngineEnabled() && isCursorDisabled()) { // Route request to new query engine if it's supported already SQLQueryRequest newSqlRequest = new SQLQueryRequest(sqlRequest.getJsonContent(), sqlRequest.getSql(), @@ -266,6 +267,11 @@ private boolean isNewEngineEnabled() { return LocalClusterState.state().getSettingValue(SQL_NEW_ENGINE_ENABLED); } + private boolean isCursorDisabled() { + Boolean isEnabled = LocalClusterState.state().getSettingValue(CURSOR_ENABLED); + return Boolean.FALSE.equals(isEnabled); + } + private static ColumnTypeProvider performAnalysis(String sql) { LocalClusterState clusterState = LocalClusterState.state(); SqlAnalysisConfig config = new SqlAnalysisConfig( diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java index f8c4b351e0..1834f98196 100644 --- a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java @@ -15,7 +15,6 @@ package com.amazon.opendistroforelasticsearch.sql.ppl.parser; -import static com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils.unquoteIdentifier; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.BinaryArithmeticContext; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.BooleanLiteralContext; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.CompareExprContext; @@ -193,7 +192,7 @@ public UnresolvedExpression visitQualifiedName(QualifiedNameContext ctx) { ctx.ident() .stream() .map(ParserRuleContext::getText) - .map(StringUtils::unquoteIdentifier) + .map(StringUtils::unquoteText) .collect(Collectors.toList()) ); } @@ -204,14 +203,14 @@ public UnresolvedExpression visitWcQualifiedName(WcQualifiedNameContext ctx) { ctx.wildcard() .stream() .map(ParserRuleContext::getText) - .map(StringUtils::unquoteIdentifier) + .map(StringUtils::unquoteText) .collect(Collectors.toList()) ); } @Override public UnresolvedExpression visitStringLiteral(StringLiteralContext ctx) { - return new Literal(unquoteIdentifier(ctx.getText()), DataType.STRING); + return new Literal(StringUtils.unquoteText(ctx.getText()), DataType.STRING); } @Override diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/ArgumentFactory.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/ArgumentFactory.java index aaa839d0b2..6167639ed5 100644 --- a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/ArgumentFactory.java +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/ArgumentFactory.java @@ -15,7 +15,6 @@ package com.amazon.opendistroforelasticsearch.sql.ppl.utils; -import static com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils.unquoteIdentifier; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.BooleanLiteralContext; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.DedupCommandContext; import static com.amazon.opendistroforelasticsearch.sql.ppl.antlr.parser.OpenDistroPPLParser.FieldsCommandContext; @@ -27,6 +26,7 @@ import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; import com.amazon.opendistroforelasticsearch.sql.ast.expression.DataType; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Literal; +import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -140,7 +140,7 @@ private static Literal getArgumentValue(ParserRuleContext ctx) { ? new Literal(Integer.parseInt(ctx.getText()), DataType.INTEGER) : ctx instanceof BooleanLiteralContext ? new Literal(Boolean.valueOf(ctx.getText()), DataType.BOOLEAN) - : new Literal(unquoteIdentifier(ctx.getText()), DataType.STRING); + : new Literal(StringUtils.unquoteText(ctx.getText()), DataType.STRING); } } diff --git a/sql/src/main/antlr/OpenDistroSQLIdentifierParser.g4 b/sql/src/main/antlr/OpenDistroSQLIdentifierParser.g4 index f29895e522..131722c4e4 100644 --- a/sql/src/main/antlr/OpenDistroSQLIdentifierParser.g4 +++ b/sql/src/main/antlr/OpenDistroSQLIdentifierParser.g4 @@ -34,6 +34,14 @@ tableName : qualifiedName ; +columnName + : qualifiedName + ; + +alias + : ident + ; + qualifiedName : ident (DOT ident)* ; diff --git a/sql/src/main/antlr/OpenDistroSQLLexer.g4 b/sql/src/main/antlr/OpenDistroSQLLexer.g4 index bf56a09786..0806eba699 100644 --- a/sql/src/main/antlr/OpenDistroSQLLexer.g4 +++ b/sql/src/main/antlr/OpenDistroSQLLexer.g4 @@ -320,7 +320,7 @@ BACKTICK_QUOTE_ID: BQUOTA_STRING; // Fragments for Literal primitives fragment EXPONENT_NUM_PART: 'E' [-+]? DEC_DIGIT+; -fragment ID_LITERAL: [*A-Z]+?[*A-Z_\-0-9]*; +fragment ID_LITERAL: [@*A-Z]+?[*A-Z_\-0-9]*; fragment DQUOTA_STRING: '"' ( '\\'. | '""' | ~('"'| '\\') )* '"'; fragment SQUOTA_STRING: '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\''; fragment BQUOTA_STRING: '`' ( '\\'. | '``' | ~('`'|'\\'))* '`'; diff --git a/sql/src/main/antlr/OpenDistroSQLParser.g4 b/sql/src/main/antlr/OpenDistroSQLParser.g4 index 72bc4e90df..ac72d0b233 100644 --- a/sql/src/main/antlr/OpenDistroSQLParser.g4 +++ b/sql/src/main/antlr/OpenDistroSQLParser.g4 @@ -68,11 +68,11 @@ selectClause ; selectElements - : (star=STAR | selectElement) (',' selectElement)* + : (star=STAR | selectElement) (COMMA selectElement)* ; selectElement - : expression #selectExpressionElement + : expression (AS? alias)? ; fromClause @@ -154,6 +154,7 @@ predicate expressionAtom : constant #constantExpressionAtom + | columnName #fullColumnNameExpressionAtom | functionCall #functionCallExpressionAtom | LR_BRACKET expression RR_BRACKET #nestedExpressionAtom | left=expressionAtom mathOperator right=expressionAtom #mathExpressionAtom diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java index 599d02bddb..387da42a79 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLService.java @@ -76,15 +76,13 @@ public void execute(SQLQueryRequest request, ResponseListener lis } /** - * Given AST, run the remaining steps to execute it. - * @param ast AST + * Given physical plan, execute it and listen on response. + * @param plan physical plan * @param listener callback listener */ - public void execute(UnresolvedPlan ast, ResponseListener listener) { + public void execute(PhysicalPlan plan, ResponseListener listener) { try { - executionEngine.execute( - plan( - analyze(ast)), listener); + executionEngine.execute(plan, listener); } catch (Exception e) { listener.onFailure(e); } @@ -95,7 +93,7 @@ public void execute(UnresolvedPlan ast, ResponseListener listener */ public UnresolvedPlan parse(String query) { ParseTree cst = parser.parse(query); - return cst.accept(new AstBuilder()); + return cst.accept(new AstBuilder(query)); } /** diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java index 0774bf769f..dd3da3f91f 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilder.java @@ -18,32 +18,44 @@ import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.FromClauseContext; import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.SelectClauseContext; +import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.SelectElementContext; import static com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.SimpleSelectContext; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Alias; import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Relation; import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Values; import com.amazon.opendistroforelasticsearch.sql.common.antlr.SyntaxCheckException; +import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.QuerySpecificationContext; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParserBaseVisitor; import com.google.common.collect.ImmutableList; import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; +import lombok.RequiredArgsConstructor; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.tree.ParseTree; /** * Abstract syntax tree (AST) builder. */ +@RequiredArgsConstructor public class AstBuilder extends OpenDistroSQLParserBaseVisitor { private static final Project SELECT_ALL = null; private final AstExpressionBuilder expressionBuilder = new AstExpressionBuilder(); + /** + * SQL query to get original token text. This is necessary because token.getText() returns + * text without whitespaces or other characters discarded by lexer. + */ + private final String query; + @Override public UnresolvedPlan visitSimpleSelect(SimpleSelectContext ctx) { QuerySpecificationContext query = ctx.querySpecification(); @@ -71,10 +83,9 @@ public UnresolvedPlan visitSelectClause(SelectClauseContext ctx) { return SELECT_ALL; } - List selectElements = ctx.selectElements().children; + List selectElements = ctx.selectElements().selectElement(); return new Project(selectElements.stream() - .map(this::visitAstExpression) - .filter(Objects::nonNull) + .map(this::visitSelectItem) .collect(Collectors.toList())); } @@ -92,4 +103,25 @@ private UnresolvedExpression visitAstExpression(ParseTree tree) { return expressionBuilder.visit(tree); } + private UnresolvedExpression visitSelectItem(SelectElementContext ctx) { + String name = StringUtils.unquoteIdentifier(getTextInQuery(ctx.expression())); + UnresolvedExpression expr = visitAstExpression(ctx.expression()); + + if (ctx.alias() == null) { + return new Alias(name, expr); + } else { + String alias = StringUtils.unquoteIdentifier(ctx.alias().getText()); + return new Alias(name, expr, alias); + } + } + + /** + * Get original text in query. + */ + private String getTextInQuery(ParserRuleContext ctx) { + Token start = ctx.getStart(); + Token stop = ctx.getStop(); + return query.substring(start.getStartIndex(), stop.getStopIndex() + 1); + } + } diff --git a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java index 2daf1c0822..1dadde63ff 100644 --- a/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java +++ b/sql/src/main/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstExpressionBuilder.java @@ -16,7 +16,6 @@ package com.amazon.opendistroforelasticsearch.sql.sql.parser; -import static com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils.unquoteIdentifier; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.IS_NOT_NULL; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.IS_NULL; import static com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName.LIKE; @@ -32,15 +31,19 @@ import com.amazon.opendistroforelasticsearch.sql.ast.expression.Function; import com.amazon.opendistroforelasticsearch.sql.ast.expression.QualifiedName; import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; +import com.amazon.opendistroforelasticsearch.sql.common.utils.StringUtils; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser; +import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.ColumnNameContext; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.IdentContext; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.NestedExpressionAtomContext; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.QualifiedNameContext; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParser.TableNameContext; import com.amazon.opendistroforelasticsearch.sql.sql.antlr.parser.OpenDistroSQLParserBaseVisitor; import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.stream.Collectors; -import org.antlr.v4.runtime.tree.RuleNode; +import org.antlr.v4.runtime.RuleContext; /** * Expression builder to parse text to expression in AST. @@ -49,22 +52,22 @@ public class AstExpressionBuilder extends OpenDistroSQLParserBaseVisitor identifiers) { + return new QualifiedName( + identifiers.stream() + .map(RuleContext::getText) + .map(StringUtils::unquoteIdentifier) + .collect(Collectors.toList()) + ); } } diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLServiceTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLServiceTest.java index ce563414ff..017278454f 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLServiceTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/SQLServiceTest.java @@ -22,17 +22,15 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; -import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; import com.amazon.opendistroforelasticsearch.sql.common.response.ResponseListener; import com.amazon.opendistroforelasticsearch.sql.executor.ExecutionEngine; -import com.amazon.opendistroforelasticsearch.sql.sql.antlr.SQLSyntaxParser; +import com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlan; import com.amazon.opendistroforelasticsearch.sql.sql.config.SQLServiceConfig; import com.amazon.opendistroforelasticsearch.sql.sql.domain.SQLQueryRequest; -import com.amazon.opendistroforelasticsearch.sql.sql.parser.AstBuilder; import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; import java.util.Collections; -import org.antlr.v4.runtime.tree.ParseTree; import org.json.JSONObject; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -87,17 +85,14 @@ public void onFailure(Exception e) { } @Test - public void canExecuteFromAst() { + public void canExecuteFromPhysicalPlan() { doAnswer(invocation -> { ResponseListener listener = invocation.getArgument(1); listener.onResponse(new QueryResponse(Collections.emptyList())); return null; }).when(executionEngine).execute(any(), any()); - ParseTree parseTree = new SQLSyntaxParser().parse("SELECT 123"); - UnresolvedPlan ast = parseTree.accept(new AstBuilder()); - - sqlService.execute(ast, + sqlService.execute(mock(PhysicalPlan.class), new ResponseListener() { @Override public void onResponse(QueryResponse response) { @@ -129,13 +124,10 @@ public void onFailure(Exception e) { } @Test - public void canCaptureErrorDuringExecutionFromAst() { + public void canCaptureErrorDuringExecutionFromPhysicalPlan() { doThrow(new RuntimeException()).when(executionEngine).execute(any(), any()); - ParseTree parseTree = new SQLSyntaxParser().parse("SELECT 123"); - UnresolvedPlan ast = parseTree.accept(new AstBuilder()); - - sqlService.execute(ast, + sqlService.execute(mock(PhysicalPlan.class), new ResponseListener() { @Override public void onResponse(QueryResponse response) { diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java index 94d668c4ba..f8f9df9875 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/antlr/SQLSyntaxParserTest.java @@ -36,6 +36,26 @@ public void canParseSelectLiterals() { assertNotNull(parser.parse("SELECT 123, 'hello'")); } + @Test + public void canParseSelectLiteralWithAlias() { + assertNotNull(parser.parse("SELECT (1 + 2) * 3 AS expr")); + } + + @Test + public void canParseSelectFields() { + assertNotNull(parser.parse("SELECT name, age FROM accounts")); + } + + @Test + public void canParseSelectFieldWithAlias() { + assertNotNull(parser.parse("SELECT name AS n, age AS a FROM accounts")); + } + + @Test + public void canParseSelectFieldWithQuotedAlias() { + assertNotNull(parser.parse("SELECT name AS \"n\", age AS `a` FROM accounts")); + } + @Test public void canParseIndexNameWithDate() { assertNotNull(parser.parse("SELECT * FROM logs_2020_01")); diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java index 5f047b22b5..4a80334b94 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstBuilderTest.java @@ -16,10 +16,13 @@ package com.amazon.opendistroforelasticsearch.sql.sql.parser; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.alias; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.doubleLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.function; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.intLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.project; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.qualifiedName; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.relation; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.stringLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.values; @@ -40,27 +43,37 @@ class AstBuilderTest { */ private final SQLSyntaxParser parser = new SQLSyntaxParser(); - /** - * AST builder class that being tested. - */ - private final AstBuilder astBuilder = new AstBuilder(); - @Test - public void canBuildSelectLiterals() { + public void can_build_select_literals() { assertEquals( project( values(emptyList()), - intLiteral(123), - stringLiteral("hello"), - booleanLiteral(false), - doubleLiteral(-4.567) + alias("123", intLiteral(123)), + alias("'hello'", stringLiteral("hello")), + alias("false", booleanLiteral(false)), + alias("-4.567", doubleLiteral(-4.567)) ), buildAST("SELECT 123, 'hello', false, -4.567") ); } @Test - public void canBuildSelectAllFromIndex() { + public void can_build_select_function_call_with_alias() { + assertEquals( + project( + relation("test"), + alias( + "ABS(age)", + function("ABS", qualifiedName("age")), + "a" + ) + ), + buildAST("SELECT ABS(age) AS a FROM test") + ); + } + + @Test + public void can_build_select_all_from_index() { assertEquals( relation("test"), buildAST("SELECT * FROM test") @@ -70,16 +83,54 @@ public void canBuildSelectAllFromIndex() { } @Test - public void buildSelectFieldsFromIndex() { // TODO: change to select fields later + public void can_build_select_fields_from_index() { + assertEquals( + project( + relation("test"), + alias("age", qualifiedName("age")) + ), + buildAST("SELECT age FROM test") + ); + } + + @Test + public void can_build_select_fields_with_alias() { assertEquals( - project(relation("test"), intLiteral(1)), - buildAST("SELECT 1 FROM test") + project( + relation("test"), + alias("age", qualifiedName("age"), "a") + ), + buildAST("SELECT age AS a FROM test") + ); + } + + @Test + public void can_build_select_fields_with_alias_quoted() { + assertEquals( + project( + relation("test"), + alias( + "name", + qualifiedName("name"), + "first name" + ), + alias( + "(age + 10)", + function("+", qualifiedName("age"), intLiteral(10)), + "Age_Expr" + ) + ), + buildAST("SELECT" + + " name AS \"first name\", " + + " (age + 10) AS `Age_Expr` " + + "FROM test" + ) ); } private UnresolvedPlan buildAST(String query) { ParseTree parseTree = parser.parse(query); - return parseTree.accept(astBuilder); + return parseTree.accept(new AstBuilder(query)); } } \ No newline at end of file diff --git a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstQualifiedNameBuilderTest.java b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstQualifiedNameBuilderTest.java index e8c1506e7d..7f7d5cf48a 100644 --- a/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstQualifiedNameBuilderTest.java +++ b/sql/src/test/java/com/amazon/opendistroforelasticsearch/sql/sql/parser/AstQualifiedNameBuilderTest.java @@ -42,7 +42,7 @@ public void canBuildRegularIdentifierForSQLStandard() { @Test public void canBuildRegularIdentifierForElasticsearch() { buildFromTableName(".kibana").expectQualifiedName(".kibana"); - //buildFromIdentifier("@timestamp").expectQualifiedName("@timestamp");//TODO: field name + buildFromIdentifier("@timestamp").expectQualifiedName("@timestamp"); buildFromIdentifier("logs-2020-01").expectQualifiedName("logs-2020-01"); buildFromIdentifier("*logs*").expectQualifiedName("*logs*"); }