From 8e4453f70ef15789ae61d71eac1e21c30fe12a23 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 20 May 2020 14:13:12 -0700 Subject: [PATCH] [PPL] Add support for fields command (#472) * add LogicalProject and LogicalRemove * add todo comments * add physical project and remove operator * update --- .../sql/analysis/Analyzer.java | 33 ++++ .../sql/ast/dsl/AstDSL.java | 40 ++-- .../sql/ast/expression/Argument.java | 2 +- .../sql/ast/expression/Field.java | 4 +- .../sql/ast/tree/Aggregation.java | 5 +- .../sql/ast/tree/Project.java | 5 +- .../sql/planner/logical/LogicalPlanDSL.java | 47 +++-- .../logical/LogicalPlanNodeVisitor.java | 38 ++-- .../sql/planner/logical/LogicalProject.java | 46 +++++ .../sql/planner/logical/LogicalRemove.java | 47 +++++ .../sql/planner/physical/PhysicalPlanDSL.java | 39 ++-- .../physical/PhysicalPlanNodeVisitor.java | 7 + .../sql/planner/physical/ProjectOperator.java | 65 ++++++ .../sql/planner/physical/RemoveOperator.java | 75 +++++++ .../sql/analysis/AnalyzerTest.java | 61 +++++- .../logical/LogicalPlanNodeVisitorTest.java | 15 +- .../physical/PhysicalPlanNodeVisitorTest.java | 187 +++++++++--------- .../planner/physical/ProjectOperatorTest.java | 88 +++++++++ .../planner/physical/RemoveOperatorTest.java | 117 +++++++++++ .../sql/ppl/parser/AstExpressionBuilder.java | 2 +- .../sql/ppl/utils/ArgumentFactory.java | 12 +- .../sql/ppl/parser/AstBuilderTest.java | 25 ++- 22 files changed, 776 insertions(+), 184 deletions(-) create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java create mode 100644 core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java create mode 100644 core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperatorTest.java create mode 100644 core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperatorTest.java diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java index ad897358ab..6e08e1e687 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/analysis/Analyzer.java @@ -16,13 +16,16 @@ package com.amazon.opendistroforelasticsearch.sql.analysis; import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; import com.amazon.opendistroforelasticsearch.sql.ast.expression.Field; import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Aggregation; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Filter; +import com.amazon.opendistroforelasticsearch.sql.ast.tree.Project; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Relation; import com.amazon.opendistroforelasticsearch.sql.ast.tree.Rename; import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprMissingValue; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; import com.amazon.opendistroforelasticsearch.sql.expression.Expression; @@ -31,12 +34,17 @@ import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalAggregation; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalFilter; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlan; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalProject; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRelation; +import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRemove; import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalRename; import com.amazon.opendistroforelasticsearch.sql.storage.StorageEngine; import com.amazon.opendistroforelasticsearch.sql.storage.Table; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import java.util.List; +import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; /** @@ -108,4 +116,29 @@ public LogicalPlan visitAggregation(Aggregation node, AnalysisContext context) { } return new LogicalAggregation(child, aggregatorBuilder.build(), groupbyBuilder.build()); } + + /** + * Build {@link LogicalProject} or {@link LogicalRemove} from {@link Field}. + * + *

Todo, the include/exclude fields should change the env definition. The cons of current + * implementation is even the query contain the field reference which has been excluded from fields command. There + * is no {@link SemanticCheckException} will be thrown. Instead, the during runtime evaluation, the not exist filed + * will be resolve to {@link ExprMissingValue} which will not impact the correctness. + * + * Postpone the implementation when finding more use case. + */ + @Override + public LogicalPlan visitProject(Project node, AnalysisContext context) { + LogicalPlan child = node.getChild().get(0).accept(this, context); + List referenceExpressions = node.getProjectList().stream() + .map(expr -> (ReferenceExpression) expressionAnalyzer.analyze(expr, context)) + .collect(Collectors.toList()); + Argument argument = node.getArgExprList().get(0); + Boolean exclude = (Boolean) argument.getValue().getValue(); + if (exclude) { + return new LogicalRemove(child, ImmutableSet.copyOf(referenceExpressions)); + } else { + return new LogicalProject(child, referenceExpressions); + } + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java index f77f215c9b..203e763622 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/dsl/AstDSL.java @@ -57,12 +57,12 @@ public static UnresolvedPlan project(UnresolvedPlan input, UnresolvedExpression. return new Project(Arrays.asList(projectList)).attach(input); } - public static UnresolvedPlan projectWithArg(UnresolvedPlan input, List argList, UnresolvedExpression... projectList) { + public static UnresolvedPlan projectWithArg(UnresolvedPlan input, List argList, UnresolvedExpression... projectList) { return new Project(Arrays.asList(projectList), argList).attach(input); } public static UnresolvedPlan agg(UnresolvedPlan input, List aggList, List sortList, - List groupList, List argList) { + List groupList, List argList) { return new Aggregation(aggList, sortList, groupList, argList).attach(input); } @@ -82,27 +82,27 @@ public static UnresolvedExpression unresolvedAttr(String attr) { return new UnresolvedAttribute(attr); } - private static UnresolvedExpression literal(Object value, DataType type) { + private static Literal literal(Object value, DataType type) { return new Literal(value, type); } - public static UnresolvedExpression intLiteral(Integer value) { + public static Literal intLiteral(Integer value) { return literal(value, DataType.INTEGER); } - public static UnresolvedExpression doubleLiteral(Double value) { + public static Literal doubleLiteral(Double value) { return literal(value, DataType.DOUBLE); } - public static UnresolvedExpression stringLiteral(String value) { + public static Literal stringLiteral(String value) { return literal(value, DataType.STRING); } - public static UnresolvedExpression booleanLiteral(Boolean value) { + public static Literal booleanLiteral(Boolean value) { return literal(value, DataType.BOOLEAN); } - public static UnresolvedExpression nullLiteral() { + public static Literal nullLiteral() { return literal(null, DataType.NULL); } @@ -146,7 +146,7 @@ public static UnresolvedExpression compare(String operator, UnresolvedExpression return new Compare(operator, left, right); } - public static UnresolvedExpression argument(String argName, UnresolvedExpression argValue) { + public static Argument argument(String argName, Literal argValue) { return new Argument(argName, argValue); } @@ -158,19 +158,19 @@ public static UnresolvedExpression field(String field) { return new Field(field); } - public static UnresolvedExpression field(UnresolvedExpression field, UnresolvedExpression... fieldArgs) { + public static UnresolvedExpression field(UnresolvedExpression field, Argument... fieldArgs) { return new Field((QualifiedName) field, Arrays.asList(fieldArgs)); } - public static UnresolvedExpression field(String field, UnresolvedExpression... fieldArgs) { + public static UnresolvedExpression field(String field, Argument... fieldArgs) { return new Field(field, Arrays.asList(fieldArgs)); } - public static UnresolvedExpression field(UnresolvedExpression field, List fieldArgs) { + public static UnresolvedExpression field(UnresolvedExpression field, List fieldArgs) { return new Field((QualifiedName) field, fieldArgs); } - public static UnresolvedExpression field(String field, List fieldArgs) { + public static UnresolvedExpression field(String field, List fieldArgs) { return new Field(field, fieldArgs); } @@ -178,13 +178,17 @@ public static List exprList(UnresolvedExpression... exprLi return Arrays.asList(exprList); } - public static List defaultFieldsArgs() { + public static List exprList(Argument... exprList) { + return Arrays.asList(exprList); + } + + public static List defaultFieldsArgs() { return exprList( argument("exclude", booleanLiteral(false)) ); } - public static List defaultStatsArgs() { + public static List defaultStatsArgs() { return exprList( argument("partitions", intLiteral(1)), argument("allnum", booleanLiteral(false)), @@ -193,7 +197,7 @@ public static List defaultStatsArgs() { ); } - public static List defaultDedupArgs() { + public static List defaultDedupArgs() { return exprList( argument("number", intLiteral(1)), argument("keepevents", booleanLiteral(false)), @@ -202,14 +206,14 @@ public static List defaultDedupArgs() { ); } - public static List defaultSortArgs() { + public static List defaultSortArgs() { return exprList( argument("count", intLiteral(1000)), argument("desc", booleanLiteral(false)) ); } - public static List defaultSortFieldArgs() { + public static List defaultSortFieldArgs() { return exprList( argument("exclude", booleanLiteral(false)), argument("type", nullLiteral()) diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Argument.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Argument.java index f676e5f29a..31da0acb44 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Argument.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Argument.java @@ -29,7 +29,7 @@ @EqualsAndHashCode(callSuper = false) public class Argument extends UnresolvedExpression { private final String argName; - private final UnresolvedExpression value; + private final Literal value; // private final DataType valueType; @Override public List getChild() { diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Field.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Field.java index d9d0ff79e8..3dbb498c61 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Field.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/expression/Field.java @@ -29,7 +29,7 @@ @AllArgsConstructor public class Field extends UnresolvedExpression { private QualifiedName field; - private List fieldArgs; + private List fieldArgs; public Field(QualifiedName field) { this.field = field; @@ -40,7 +40,7 @@ public Field(String field) { this.field = new QualifiedName(field); } - public Field(String field, List fieldArgs) { + public Field(String field, List fieldArgs) { this.field = new QualifiedName(field); this.fieldArgs = fieldArgs; } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Aggregation.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Aggregation.java index 833e1a0bc9..556b93ee7d 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Aggregation.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Aggregation.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.sql.ast.tree; import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; import com.google.common.collect.ImmutableList; import java.util.Collections; @@ -36,7 +37,7 @@ public class Aggregation extends UnresolvedPlan { private List aggExprList; private List sortExprList; private List groupExprList; - private List argExprList; + private List argExprList; private UnresolvedPlan child; public Aggregation(List aggExprList, @@ -51,7 +52,7 @@ public Aggregation(List aggExprList, public Aggregation(List aggExprList, List sortExprList, List groupExprList, - List argExprList) { + List argExprList) { this.aggExprList = aggExprList; this.sortExprList = sortExprList; this.groupExprList = groupExprList; diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Project.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Project.java index 58da207022..4e9dc0aa50 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Project.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/ast/tree/Project.java @@ -16,6 +16,7 @@ package com.amazon.opendistroforelasticsearch.sql.ast.tree; import com.amazon.opendistroforelasticsearch.sql.ast.AbstractNodeVisitor; +import com.amazon.opendistroforelasticsearch.sql.ast.expression.Argument; import com.amazon.opendistroforelasticsearch.sql.ast.expression.UnresolvedExpression; import com.google.common.collect.ImmutableList; import java.util.Collections; @@ -34,7 +35,7 @@ public class Project extends UnresolvedPlan { @Setter private List projectList; - private List argExprList; + private List argExprList; private UnresolvedPlan child; public Project(List projectList) { @@ -42,7 +43,7 @@ public Project(List projectList) { this.argExprList = Collections.emptyList(); } - public Project(List projectList, List argExprList) { + public Project(List projectList, List argExprList) { this.projectList = projectList; this.argExprList = argExprList; } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java index 458c4c3c74..a8facc1444 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanDSL.java @@ -18,30 +18,39 @@ import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; +import com.google.common.collect.ImmutableSet; +import java.util.Arrays; import lombok.experimental.UtilityClass; import java.util.List; import java.util.Map; -/** - * Logical Plan DSL. - */ +/** Logical Plan DSL. */ @UtilityClass public class LogicalPlanDSL { - public static LogicalPlan aggregation(LogicalPlan input, List aggregatorList, - List groupByList) { - return new LogicalAggregation(input, aggregatorList, groupByList); - } - - public static LogicalPlan filter(LogicalPlan input, Expression expression) { - return new LogicalFilter(input, expression); - } - - public static LogicalPlan relation(String tableName) { - return new LogicalRelation(tableName); - } - - public static LogicalPlan rename(LogicalPlan input, Map renameMap) { - return new LogicalRename(input, renameMap); - } + public static LogicalPlan aggregation( + LogicalPlan input, List aggregatorList, List groupByList) { + return new LogicalAggregation(input, aggregatorList, groupByList); + } + + public static LogicalPlan filter(LogicalPlan input, Expression expression) { + return new LogicalFilter(input, expression); + } + + public static LogicalPlan relation(String tableName) { + return new LogicalRelation(tableName); + } + + public static LogicalPlan rename( + LogicalPlan input, Map renameMap) { + return new LogicalRename(input, renameMap); + } + + public static LogicalPlan project(LogicalPlan input, ReferenceExpression... fields) { + return new LogicalProject(input, Arrays.asList(fields)); + } + + public static LogicalPlan remove(LogicalPlan input, ReferenceExpression... fields) { + return new LogicalRemove(input, ImmutableSet.copyOf(fields)); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java index 26364bb903..d145aa6c3b 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitor.java @@ -22,23 +22,31 @@ * @param context type. */ public abstract class LogicalPlanNodeVisitor { - protected R visitNode(LogicalPlan plan, C context) { - return null; - } + protected R visitNode(LogicalPlan plan, C context) { + return null; + } - public R visitRelation(LogicalRelation plan, C context) { - return visitNode(plan, context); - } + public R visitRelation(LogicalRelation plan, C context) { + return visitNode(plan, context); + } - public R visitFilter(LogicalFilter plan, C context) { - return visitNode(plan, context); - } + public R visitFilter(LogicalFilter plan, C context) { + return visitNode(plan, context); + } - public R visitAggregation(LogicalAggregation plan, C context) { - return visitNode(plan, context); - } + public R visitAggregation(LogicalAggregation plan, C context) { + return visitNode(plan, context); + } - public R visitRename(LogicalRename plan, C context) { - return visitNode(plan, context); - } + public R visitRename(LogicalRename plan, C context) { + return visitNode(plan, context); + } + + public R visitProject(LogicalProject plan, C context) { + return visitNode(plan, context); + } + + public R visitRemove(LogicalRemove plan, C context) { + return visitNode(plan, context); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java new file mode 100644 index 0000000000..a8ab31523b --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalProject.java @@ -0,0 +1,46 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import java.util.Arrays; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Project field specified by the {@link LogicalProject#projectList}. + */ +@ToString +@EqualsAndHashCode +@RequiredArgsConstructor +public class LogicalProject extends LogicalPlan { + private final LogicalPlan child; + @Getter + private final List projectList; + + @Override + public List getChild() { + return Arrays.asList(child); + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitProject(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java new file mode 100644 index 0000000000..4b73048c64 --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalRemove.java @@ -0,0 +1,47 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.logical; + +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** + * Remove field specified by the {@link LogicalRemove#removeList}. + */ +@ToString +@EqualsAndHashCode +@RequiredArgsConstructor +public class LogicalRemove extends LogicalPlan { + private final LogicalPlan child; + @Getter + private final Set removeList; + + @Override + public List getChild() { + return Arrays.asList(child); + } + + @Override + public R accept(LogicalPlanNodeVisitor visitor, C context) { + return visitor.visitRemove(this, context); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java index 9bbf604784..d59b123e10 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanDSL.java @@ -18,26 +18,35 @@ import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; -import lombok.experimental.UtilityClass; - +import com.google.common.collect.ImmutableSet; +import java.util.Arrays; import java.util.List; import java.util.Map; +import lombok.experimental.UtilityClass; -/** - * Physical Plan DSL. - */ +/** Physical Plan DSL. */ @UtilityClass public class PhysicalPlanDSL { - public static AggregationOperator agg(PhysicalPlan input, List aggregators, List groups) { - return new AggregationOperator(input, aggregators, groups); - } + public static AggregationOperator agg( + PhysicalPlan input, List aggregators, List groups) { + return new AggregationOperator(input, aggregators, groups); + } + + public static FilterOperator filter(PhysicalPlan input, Expression condition) { + return new FilterOperator(input, condition); + } + + public static RenameOperator rename( + PhysicalPlan input, Map renameMap) { + return new RenameOperator(input, renameMap); + } - public static FilterOperator filter(PhysicalPlan input, Expression condition) { - return new FilterOperator(input, condition); - } + public static ProjectOperator project(PhysicalPlan input, ReferenceExpression... fields) { + return new ProjectOperator(input, Arrays.asList(fields)); + } - public static RenameOperator rename(PhysicalPlan input, Map renameMap) { - return new RenameOperator(input, renameMap); - } -} \ No newline at end of file + public static RemoveOperator remove(PhysicalPlan input, ReferenceExpression... fields) { + return new RemoveOperator(input, ImmutableSet.copyOf(fields)); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitor.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitor.java index 6e000d1b6a..a0fec17d92 100644 --- a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitor.java +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitor.java @@ -38,4 +38,11 @@ public R visitAggregation(AggregationOperator node, C context) { public R visitRename(RenameOperator node, C context) { return visitNode(node, context); } + + public R visitProject(ProjectOperator node, C context) { + return visitNode(node, context); + } + public R visitRemove(RemoveOperator node, C context) { + return visitNode(node, context); + } } diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java new file mode 100644 index 0000000000..30928bd1ab --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperator.java @@ -0,0 +1,65 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import java.util.Collections; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** Project the fields specified in {@link ProjectOperator#projectList} from input. */ +@ToString +@EqualsAndHashCode +@RequiredArgsConstructor +public class ProjectOperator extends PhysicalPlan { + private final PhysicalPlan input; + private final List projectList; + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitProject(this, context); + } + + @Override + public List getChild() { + return Collections.singletonList(input); + } + + @Override + public boolean hasNext() { + return input.hasNext(); + } + + @Override + public ExprValue next() { + ExprValue inputValue = input.next(); + ImmutableMap.Builder mapBuilder = new Builder<>(); + for (ReferenceExpression ref : projectList) { + ExprValue exprValue = ref.valueOf(inputValue.bindingTuples()); + // missing value is ignored. + if (!exprValue.isMissing()) { + mapBuilder.put(ref.toString(), exprValue); + } + } + return ExprTupleValue.fromExprValueMap(mapBuilder.build()); + } +} diff --git a/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java new file mode 100644 index 0000000000..20f151c35e --- /dev/null +++ b/core/src/main/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperator.java @@ -0,0 +1,75 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprType.STRUCT; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprTupleValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMap.Builder; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import lombok.EqualsAndHashCode; +import lombok.RequiredArgsConstructor; +import lombok.ToString; + +/** Remove the fields specified in {@link RemoveOperator#removeList} from input. */ +@ToString +@EqualsAndHashCode +@RequiredArgsConstructor +public class RemoveOperator extends PhysicalPlan { + private final PhysicalPlan input; + private final Set removeList; + + @Override + public R accept(PhysicalPlanNodeVisitor visitor, C context) { + return visitor.visitRemove(this, context); + } + + @Override + public List getChild() { + return Collections.singletonList(input); + } + + @Override + public boolean hasNext() { + return input.hasNext(); + } + + @Override + public ExprValue next() { + ExprValue inputValue = input.next(); + if (STRUCT == inputValue.type()) { + ImmutableMap.Builder mapBuilder = new Builder<>(); + Map tupleValue = ExprValueUtils.getTupleValue(inputValue); + for (Entry valueEntry : tupleValue.entrySet()) { + if (!removeList.contains(DSL.ref(valueEntry.getKey()))) { + mapBuilder.put(valueEntry); + } + } + return ExprTupleValue.fromExprValueMap(mapBuilder.build()); + } else { + return inputValue; + } + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java index 084788d5dc..c6d97fa8cb 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/analysis/AnalyzerTest.java @@ -15,6 +15,13 @@ package com.amazon.opendistroforelasticsearch.sql.analysis; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.argument; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.field; +import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + import com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL; import com.amazon.opendistroforelasticsearch.sql.ast.tree.UnresolvedPlan; import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException; @@ -23,13 +30,10 @@ import com.amazon.opendistroforelasticsearch.sql.planner.logical.LogicalPlanDSL; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.util.Collections; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.field; -import static com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils.integerValue; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - class AnalyzerTest extends AnalyzerTestBase { @Test public void filter_relation() { @@ -45,7 +49,6 @@ public void filter_relation() { ); } - @Test public void rename_relation() { assertAnalyzeEqual( @@ -126,6 +129,52 @@ public void rename_to_invalid_expression() { assertEquals("the target expected to be field, but is avg(Field(field=integer_value, fieldArgs=null))", exception.getMessage()); } + @Test + public void project_source() { + assertAnalyzeEqual( + LogicalPlanDSL.project( + LogicalPlanDSL.relation("schema"), + DSL.ref("integer_value"), DSL.ref("double_value") + ), + AstDSL.projectWithArg( + AstDSL.relation("schema"), + AstDSL.defaultFieldsArgs(), + AstDSL.field("integer_value"), AstDSL.field("double_value") + ) + ); + } + + @Test + public void remove_source() { + assertAnalyzeEqual( + LogicalPlanDSL.remove( + LogicalPlanDSL.relation("schema"), + DSL.ref("integer_value"), DSL.ref("double_value") + ), + AstDSL.projectWithArg( + AstDSL.relation("schema"), + Collections.singletonList(argument("exclude", booleanLiteral(true))), + AstDSL.field("integer_value"), AstDSL.field("double_value") + ) + ); + } + + @Disabled("the project/remove command should shrink the type env") + @Test + public void project_source_change_type_env() { + SemanticCheckException exception = assertThrows(SemanticCheckException.class, () -> analyze( + AstDSL.projectWithArg( + AstDSL.projectWithArg( + AstDSL.relation("schema"), + AstDSL.defaultFieldsArgs(), + AstDSL.field("integer_value"), AstDSL.field("double_value") + ), + AstDSL.defaultFieldsArgs(), + AstDSL.field("float_value") + ) + )); + } + protected void assertAnalyzeEqual(LogicalPlan expected, UnresolvedPlan unresolvedPlan) { assertEquals(expected, analyze(unresolvedPlan)); } diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java index 4f8a53cbf5..a77b180e2e 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/logical/LogicalPlanNodeVisitorTest.java @@ -15,21 +15,20 @@ package com.amazon.opendistroforelasticsearch.sql.planner.logical; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + import com.amazon.opendistroforelasticsearch.sql.expression.Expression; import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import java.util.stream.Collectors; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.stream.Collectors; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; - /** * Todo. Temporary added for UT coverage, Will be removed. */ @@ -73,6 +72,8 @@ public void testAbstractPlanNodeVisitorShouldReturnNull() { aggregation, ImmutableMap.of(ref, ref) ); + LogicalPlan project = LogicalPlanDSL.project(relation, ref); + LogicalPlan remove = LogicalPlanDSL.remove(relation, ref); assertNull(relation.accept(new LogicalPlanNodeVisitor() { }, null)); @@ -82,6 +83,10 @@ public void testAbstractPlanNodeVisitorShouldReturnNull() { }, null)); assertNull(rename.accept(new LogicalPlanNodeVisitor() { }, null)); + assertNull(project.accept(new LogicalPlanNodeVisitor() { + }, null)); + assertNull(remove.accept(new LogicalPlanNodeVisitor() { + }, null)); } private static class NodesCount extends LogicalPlanNodeVisitor { diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java index 26e9f69b07..b1fc3131ac 100644 --- a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/PhysicalPlanNodeVisitorTest.java @@ -16,110 +16,115 @@ package com.amazon.opendistroforelasticsearch.sql.planner.physical; import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.amazon.opendistroforelasticsearch.sql.expression.Expression; +import com.amazon.opendistroforelasticsearch.sql.expression.ReferenceExpression; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; -/** - * Todo, testing purpose, delete later. - */ +/** Todo, testing purpose, delete later. */ +@ExtendWith(MockitoExtension.class) class PhysicalPlanNodeVisitorTest extends PhysicalPlanTestBase { - @Test - public void print_physical_plan() { - PhysicalPlan plan = PhysicalPlanDSL.rename( - PhysicalPlanDSL.agg( + @Mock PhysicalPlan plan; + @Mock ReferenceExpression ref; + + @Test + public void print_physical_plan() { + PhysicalPlan plan = + PhysicalPlanDSL.remove( + PhysicalPlanDSL.project( + PhysicalPlanDSL.rename( + PhysicalPlanDSL.agg( PhysicalPlanDSL.filter( - new TestScan(), - dsl.equal(typeEnv(), DSL.ref("response"), DSL.literal(10)) - ), + new TestScan(), + dsl.equal(typeEnv(), DSL.ref("response"), DSL.literal(10))), ImmutableList.of(dsl.avg(typeEnv(), DSL.ref("response"))), - ImmutableList.of() - ), - ImmutableMap.of(DSL.ref("ivalue"), DSL.ref("avg(response)")) - ); - - PhysicalPlanPrinter printer = new PhysicalPlanPrinter(); - assertEquals("Rename->\n" + - "\tAggregation->\n" + - "\t\tFilter->", printer.print(plan)); + ImmutableList.of()), + ImmutableMap.of(DSL.ref("ivalue"), DSL.ref("avg(response)"))), + ref), + ref); + + PhysicalPlanPrinter printer = new PhysicalPlanPrinter(); + assertEquals( + "Remove->\n" + + "\tProject->\n" + + "\t\tRename->\n" + + "\t\t\tAggregation->\n" + + "\t\t\t\tFilter->", + printer.print(plan)); + } + + @Test + public void test_PhysicalPlanVisitor_should_return_null() { + PhysicalPlan filter = + PhysicalPlanDSL.filter( + new TestScan(), dsl.equal(typeEnv(), DSL.ref("response"), DSL.literal(10))); + PhysicalPlan aggregation = + PhysicalPlanDSL.agg( + filter, ImmutableList.of(dsl.avg(typeEnv(), DSL.ref("response"))), ImmutableList.of()); + PhysicalPlan rename = + PhysicalPlanDSL.rename( + aggregation, ImmutableMap.of(DSL.ref("ivalue"), DSL.ref("avg(response)"))); + PhysicalPlan project = PhysicalPlanDSL.project(plan, ref); + PhysicalPlan remove = PhysicalPlanDSL.remove(plan, ref); + + assertNull(filter.accept(new PhysicalPlanNodeVisitor() {}, null)); + assertNull(aggregation.accept(new PhysicalPlanNodeVisitor() {}, null)); + assertNull(rename.accept(new PhysicalPlanNodeVisitor() {}, null)); + assertNull(project.accept(new PhysicalPlanNodeVisitor() {}, null)); + assertNull(remove.accept(new PhysicalPlanNodeVisitor() {}, null)); + } + + public static class PhysicalPlanPrinter extends PhysicalPlanNodeVisitor { + + public String print(PhysicalPlan node) { + return node.accept(this, 0); + } + + @Override + public String visitFilter(FilterOperator node, Integer tabs) { + return name(node, "Filter->", tabs); + } + + @Override + public String visitAggregation(AggregationOperator node, Integer tabs) { + return name(node, "Aggregation->", tabs); + } + + @Override + public String visitRename(RenameOperator node, Integer tabs) { + return name(node, "Rename->", tabs); + } + + @Override + public String visitProject(ProjectOperator node, Integer tabs) { + return name(node, "Project->", tabs); } - @Test - public void test_PhysicalPlanVisitor_should_return_null() { - PhysicalPlan filter = PhysicalPlanDSL.filter( - new TestScan(), - dsl.equal(typeEnv(), DSL.ref("response"), DSL.literal(10)) - ); - PhysicalPlan aggregation = PhysicalPlanDSL.agg( - filter, - ImmutableList.of(dsl.avg(typeEnv(), DSL.ref("response"))), - ImmutableList.of() - ); - PhysicalPlan rename = PhysicalPlanDSL.rename( - aggregation, - ImmutableMap.of(DSL.ref("ivalue"), DSL.ref("avg(response)")) - ); - assertNull(filter.accept(new PhysicalPlanNodeVisitor() { - }, null)); - assertNull(aggregation.accept(new PhysicalPlanNodeVisitor() { - }, null)); - assertNull(rename.accept(new PhysicalPlanNodeVisitor() { - }, null)); + @Override + public String visitRemove(RemoveOperator node, Integer tabs) { + return name(node, "Remove->", tabs); } - public static class PhysicalPlanPrinter extends PhysicalPlanNodeVisitor { - - public String print(PhysicalPlan node) { - return node.accept(this, 0); - } - - @Override - public String visitFilter(FilterOperator node, Integer tabs) { - String child = node.getChild().get(0).accept(this, tabs + 1); - StringBuilder sb = new StringBuilder(); - for (Integer i = 0; i < tabs; i++) { - sb.append("\t"); - } - sb.append("Filter->"); - if (!Strings.isNullOrEmpty(child)) { - sb.append("\n"); - sb.append(child); - } - return sb.toString(); - } - - @Override - public String visitAggregation(AggregationOperator node, Integer tabs) { - String child = node.getChild().get(0).accept(this, tabs + 1); - StringBuilder sb = new StringBuilder(); - for (Integer i = 0; i < tabs; i++) { - sb.append("\t"); - } - sb.append("Aggregation->"); - if (!Strings.isNullOrEmpty(child)) { - sb.append("\n"); - sb.append(child); - } - return sb.toString(); - } - - @Override - public String visitRename(RenameOperator node, Integer tabs) { - String child = node.getChild().get(0).accept(this, tabs + 1); - StringBuilder sb = new StringBuilder(); - for (Integer i = 0; i < tabs; i++) { - sb.append("\t"); - } - sb.append("Rename->"); - if (!Strings.isNullOrEmpty(child)) { - sb.append("\n"); - sb.append(child); - } - return sb.toString(); - } + private String name(PhysicalPlan node, String current, int tabs) { + String child = node.getChild().get(0).accept(this, tabs + 1); + StringBuilder sb = new StringBuilder(); + for (Integer i = 0; i < tabs; i++) { + sb.append("\t"); + } + sb.append(current); + if (!Strings.isNullOrEmpty(child)) { + sb.append("\n"); + sb.append(child); + } + return sb.toString(); } -} \ No newline at end of file + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperatorTest.java new file mode 100644 index 0000000000..ba961fd89d --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/ProjectOperatorTest.java @@ -0,0 +1,88 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL.project; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.iterableWithSize; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class ProjectOperatorTest extends PhysicalPlanTestBase { + + @Mock private PhysicalPlan inputPlan; + + @Test + public void project_one_field() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", 200))); + PhysicalPlan plan = project(inputPlan, DSL.ref("action")); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET"))))); + } + + @Test + public void project_two_field_follow_the_project_order() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", 200))); + PhysicalPlan plan = project(inputPlan, DSL.ref("response"), DSL.ref("action")); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems( + ExprValueUtils.tupleValue(ImmutableMap.of("response", 200, "action", "GET"))))); + } + + @Test + public void project_ignore_missing_value() { + when(inputPlan.hasNext()).thenReturn(true, true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", 200))) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "POST"))); + PhysicalPlan plan = project(inputPlan, DSL.ref("response"), DSL.ref("action")); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(2), + hasItems( + ExprValueUtils.tupleValue(ImmutableMap.of("response", 200, "action", "GET")), + ExprValueUtils.tupleValue(ImmutableMap.of("action", "POST"))))); + } +} diff --git a/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperatorTest.java b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperatorTest.java new file mode 100644 index 0000000000..9a706c4091 --- /dev/null +++ b/core/src/test/java/com/amazon/opendistroforelasticsearch/sql/planner/physical/RemoveOperatorTest.java @@ -0,0 +1,117 @@ +/* + * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.amazon.opendistroforelasticsearch.sql.planner.physical; + +import static com.amazon.opendistroforelasticsearch.sql.planner.physical.PhysicalPlanDSL.remove; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.iterableWithSize; +import static org.mockito.Mockito.when; + +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue; +import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils; +import com.amazon.opendistroforelasticsearch.sql.expression.DSL; +import com.google.common.collect.ImmutableMap; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +class RemoveOperatorTest extends PhysicalPlanTestBase { + @Mock private PhysicalPlan inputPlan; + + @Test + public void remove_one_field() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", 200))); + PhysicalPlan plan = remove(inputPlan, DSL.ref("action")); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems(ExprValueUtils.tupleValue(ImmutableMap.of("response", 200))))); + } + + @Test + public void remove_one_field_follow_the_input_order() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue( + ImmutableMap.of("action", "GET", "response", 200, "referer", "www.amazon.com"))); + PhysicalPlan plan = remove(inputPlan, DSL.ref("response")); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems( + ExprValueUtils.tupleValue( + ImmutableMap.of("action", "GET", "referer", "www.amazon.com"))))); + } + + @Test + public void remove_two_field() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue( + ImmutableMap.of("action", "GET", "response", 200, "referer", "www.amazon.com"))); + PhysicalPlan plan = remove(inputPlan, DSL.ref("response"), DSL.ref("referer")); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET"))))); + } + + @Test + public void project_ignore_missing_value() { + when(inputPlan.hasNext()).thenReturn(true, true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", 200))) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "POST"))); + PhysicalPlan plan = remove(inputPlan, DSL.ref("response")); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(2), + hasItems( + ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET")), + ExprValueUtils.tupleValue(ImmutableMap.of("action", "POST"))))); + } + + @Test + public void remove_nothing_with_none_tuple_value() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()).thenReturn(ExprValueUtils.integerValue(1)); + PhysicalPlan plan = remove(inputPlan, DSL.ref("response"), DSL.ref("referer")); + List result = execute(plan); + + assertThat(result, allOf(iterableWithSize(1), hasItems(ExprValueUtils.integerValue(1)))); + } +} diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java index 81e94dded4..416a2a0f54 100644 --- a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstExpressionBuilder.java @@ -145,7 +145,7 @@ public UnresolvedExpression visitStatsFunctionCall(StatsFunctionCallContext ctx) @Override public UnresolvedExpression visitPercentileAggFunction(PercentileAggFunctionContext ctx) { return new AggregateFunction(ctx.PERCENTILE().getText(), visit(ctx.aggField), - Collections.singletonList(new Argument("rank", visit(ctx.value)))); + Collections.singletonList(new Argument("rank", (Literal) visit(ctx.value)))); } /** Eval function */ diff --git a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/ArgumentFactory.java b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/ArgumentFactory.java index 1de1b491fe..6ac7ac14a6 100644 --- a/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/ArgumentFactory.java +++ b/ppl/src/main/java/com/amazon/opendistroforelasticsearch/sql/ppl/utils/ArgumentFactory.java @@ -43,7 +43,7 @@ public class ArgumentFactory { * @param ctx FieldsCommandContext instance * @return the list of arguments fetched from the fields command */ - public static List getArgumentList(FieldsCommandContext ctx) { + public static List getArgumentList(FieldsCommandContext ctx) { return Collections.singletonList( ctx.MINUS() != null ? new Argument("exclude", new Literal(true, DataType.BOOLEAN)) @@ -55,7 +55,7 @@ public static List getArgumentList(FieldsCommandContext ct * @param ctx StatsCommandContext instance * @return the list of arguments fetched from the stats command */ - public static List getArgumentList(StatsCommandContext ctx) { + public static List getArgumentList(StatsCommandContext ctx) { return Arrays.asList( ctx.partitions != null ? new Argument("partitions", getArgumentValue(ctx.partitions)) @@ -76,7 +76,7 @@ public static List getArgumentList(StatsCommandContext ctx * @param ctx DedupCommandContext instance * @return the list of arguments fetched from the dedup command */ - public static List getArgumentList(DedupCommandContext ctx) { + public static List getArgumentList(DedupCommandContext ctx) { return Arrays.asList( ctx.number != null ? new Argument("number", getArgumentValue(ctx.number)) @@ -97,7 +97,7 @@ public static List getArgumentList(DedupCommandContext ctx * @param ctx SortCommandContext instance * @return the list of arguments fetched from the sort command */ - public static List getArgumentList(SortCommandContext ctx) { + public static List getArgumentList(SortCommandContext ctx) { return Arrays.asList( ctx.count != null ? new Argument("count", getArgumentValue(ctx.count)) @@ -112,7 +112,7 @@ public static List getArgumentList(SortCommandContext ctx) * @param ctx SortFieldContext instance * @return the list of arguments fetched from the sort field in sort command */ - public static List getArgumentList(OpenDistroPPLParser.SortFieldContext ctx) { + public static List getArgumentList(OpenDistroPPLParser.SortFieldContext ctx) { return Arrays.asList( ctx.MINUS() != null ? new Argument("exclude", new Literal(true, DataType.BOOLEAN)) @@ -129,7 +129,7 @@ public static List getArgumentList(OpenDistroPPLParser.Sor ); } - private static UnresolvedExpression getArgumentValue(ParserRuleContext ctx) { + private static Literal getArgumentValue(ParserRuleContext ctx) { return ctx instanceof IntegerLiteralContext ? new Literal(Integer.parseInt(ctx.getText()), DataType.INTEGER) : ctx instanceof BooleanLiteralContext ? new Literal(Boolean.valueOf(ctx.getText()), DataType.BOOLEAN) : new Literal(unquoteIdentifier(ctx.getText()), DataType.STRING); diff --git a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilderTest.java index fb832ac060..6c5439e712 100644 --- a/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/com/amazon/opendistroforelasticsearch/sql/ppl/parser/AstBuilderTest.java @@ -17,10 +17,13 @@ import com.amazon.opendistroforelasticsearch.sql.ppl.antlr.PPLSyntaxParser; import com.amazon.opendistroforelasticsearch.sql.ast.Node; +import java.util.Collections; import org.junit.Test; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.agg; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.aggregate; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.argument; +import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.booleanLiteral; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.compare; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultDedupArgs; import static com.amazon.opendistroforelasticsearch.sql.ast.dsl.AstDSL.defaultFieldsArgs; @@ -93,7 +96,7 @@ public void testWhereCommand() { } @Test - public void testFieldsCommand() { + public void testFieldsCommandWithoutArguments() { assertEqual("source=t | fields f, g", projectWithArg( relation("t"), @@ -102,6 +105,26 @@ public void testFieldsCommand() { )); } + @Test + public void testFieldsCommandWithIncludeArguments() { + assertEqual("source=t | fields + f, g", + projectWithArg( + relation("t"), + defaultFieldsArgs(), + field("f"), field("g") + )); + } + + @Test + public void testFieldsCommandWithExcludeArguments() { + assertEqual("source=t | fields - f, g", + projectWithArg( + relation("t"), + Collections.singletonList(argument("exclude", booleanLiteral(true))), + field("f"), field("g") + )); + } + @Test public void testRenameCommand() { assertEqual("source=t | rename f as g",